43 Commits

Author SHA1 Message Date
enricobuehler a9dc6efe55 fix(windows): drop the orphaned touch_last_used re-export
ci / web (push) Successful in 44s
ci / docs-site (push) Successful in 1m0s
ci / bench (push) Successful in 6m1s
windows / build (aarch64-pc-windows-msvc) (push) Successful in 5m2s
windows / build (x86_64-pc-windows-msvc) (push) Successful in 1m29s
ci / rust (push) Successful in 22m28s
docker / build-push (--build-arg FEDORA_VERSION=44, ci, ci/fedora-rpm.Dockerfile, punktfunk-fedora44-rpm) (push) Successful in 13s
docker / build-push (., web/Dockerfile, punktfunk-web) (push) Successful in 12s
decky / build-publish (push) Successful in 19s
docker / build-push (ci, ci/fedora-rpm.Dockerfile, punktfunk-fedora-rpm) (push) Successful in 10s
docker / build-push (ci, ci/rust-ci.Dockerfile, punktfunk-rust-ci) (push) Successful in 11s
docker / build-push (docs-site, docs-site/Dockerfile, punktfunk-docs) (push) Successful in 10s
android-screenshots / screenshots (push) Successful in 3m6s
flatpak / build-publish (push) Successful in 7m12s
web-screenshots / screenshots (push) Successful in 2m53s
linux-client-screenshots / screenshots (push) Successful in 10m8s
docker / deploy-docs (push) Successful in 23s
arch / build-publish (push) Successful in 11m54s
deb / build-publish (push) Successful in 12m53s
windows-host / package (push) Successful in 15m26s
android / android (push) Successful in 15m45s
rpm / build-publish (43, bazzite, punktfunk-fedora-rpm) (push) Successful in 17m25s
rpm / build-publish (44, fedora-44, punktfunk-fedora44-rpm) (push) Successful in 16m16s
windows-msix / package (arm64, C:\Users\Public\ffmpeg-arm64, --no-default-features, aarch64-pc-windows-msvc, C:\t-a64) (push) Successful in 3m59s
windows-msix / package (x64, C:\Users\Public\ffmpeg, , x86_64-pc-windows-msvc, C:\t) (push) Successful in 3m59s
apple / swift (push) Successful in 4m56s
release / apple (push) Successful in 24m44s
apple / screenshots (push) Successful in 20m44s
More builtin-removal fallout: trust.rs re-exported pf_client_core::trust::
touch_last_used, whose only consumer was the deleted in-process session pump. In
a binary crate an unused pub-use is a hard -D warnings error (it surfaced only
after the gamepad dead-code errors were cleared, which had suppressed the
unused_imports pass). Drop it; every other re-export still has a user.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-07-13 08:37:24 +02:00
enricobuehler 9822fc3b1c fix(windows): drop the orphaned in-process gamepad forwarding hooks
ci / web (push) Successful in 1m7s
ci / docs-site (push) Successful in 1m0s
ci / bench (push) Successful in 5m49s
android-screenshots / screenshots (push) Successful in 3m17s
ci / rust (push) Successful in 17m50s
decky / build-publish (push) Successful in 18s
windows / build (aarch64-pc-windows-msvc) (push) Failing after 59s
apple / swift (push) Successful in 4m25s
windows / build (x86_64-pc-windows-msvc) (push) Failing after 5m22s
android / android (push) Successful in 14m55s
arch / build-publish (push) Successful in 13m12s
deb / build-publish (push) Successful in 11m29s
web-screenshots / screenshots (push) Successful in 2m42s
docker / deploy-docs (push) Successful in 12s
docker / build-push (--build-arg FEDORA_VERSION=44, ci, ci/fedora-rpm.Dockerfile, punktfunk-fedora44-rpm) (push) Successful in 10s
docker / build-push (docs-site, docs-site/Dockerfile, punktfunk-docs) (push) Successful in 9s
docker / build-push (ci, ci/fedora-rpm.Dockerfile, punktfunk-fedora-rpm) (push) Successful in 6m10s
docker / build-push (., web/Dockerfile, punktfunk-web) (push) Successful in 9s
docker / build-push (ci, ci/rust-ci.Dockerfile, punktfunk-rust-ci) (push) Successful in 11s
linux-client-screenshots / screenshots (push) Successful in 7m16s
rpm / build-publish (43, bazzite, punktfunk-fedora-rpm) (push) Successful in 12m58s
rpm / build-publish (44, fedora-44, punktfunk-fedora44-rpm) (push) Successful in 13m36s
windows-host / package (push) Successful in 15m17s
windows-msix / package (arm64, C:\Users\Public\ffmpeg-arm64, --no-default-features, aarch64-pc-windows-msvc, C:\t-a64) (push) Successful in 4m10s
release / apple (push) Successful in 22m45s
windows-msix / package (x64, C:\Users\Public\ffmpeg, , x86_64-pc-windows-msvc, C:\t) (push) Successful in 3m59s
flatpak / build-publish (push) Successful in 5m41s
apple / screenshots (push) Successful in 19m29s
Removing the builtin stream path (ef580825) left GamepadService's attach/detach/
active/auto_pref + the Ctl::Attach/Detach variants with no callers — the spawned
punktfunk-session binary owns pad forwarding now. The client still compiled, but
clippy -D warnings tripped on the dead code. Drop the forwarding hooks + the
active-pad mirror; the service keeps pads() (Settings list) and set_pinned()
(persist the forwarded-pad selection the session child reads).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-07-13 07:49:38 +02:00
enricobuehler cdb43f00fe style: rustfmt the freeze-until-reanchor client wiring
ci / web (push) Successful in 57s
ci / docs-site (push) Successful in 1m7s
ci / bench (push) Successful in 5m11s
decky / build-publish (push) Successful in 28s
docker / build-push (--build-arg FEDORA_VERSION=44, ci, ci/fedora-rpm.Dockerfile, punktfunk-fedora44-rpm) (push) Successful in 8s
docker / build-push (., web/Dockerfile, punktfunk-web) (push) Successful in 40s
deb / build-publish (push) Successful in 11m37s
docker / build-push (docs-site, docs-site/Dockerfile, punktfunk-docs) (push) Successful in 8s
arch / build-publish (push) Successful in 15m30s
apple / swift (push) Successful in 4m19s
android / android (push) Successful in 16m52s
docker / build-push (ci, ci/rust-ci.Dockerfile, punktfunk-rust-ci) (push) Successful in 8m34s
docker / build-push (ci, ci/fedora-rpm.Dockerfile, punktfunk-fedora-rpm) (push) Successful in 10m14s
windows-host / package (push) Successful in 13m57s
flatpak / build-publish (push) Successful in 5m48s
windows-msix / package (arm64, C:\Users\Public\ffmpeg-arm64, --no-default-features, aarch64-pc-windows-msvc, C:\t-a64) (push) Successful in 3m46s
ci / rust (push) Successful in 22m54s
windows-msix / package (x64, C:\Users\Public\ffmpeg, , x86_64-pc-windows-msvc, C:\t) (push) Successful in 2m7s
windows / build (aarch64-pc-windows-msvc) (push) Failing after 5m1s
rpm / build-publish (43, bazzite, punktfunk-fedora-rpm) (push) Successful in 15m27s
rpm / build-publish (44, fedora-44, punktfunk-fedora44-rpm) (push) Successful in 15m1s
windows / build (x86_64-pc-windows-msvc) (push) Failing after 5m30s
release / apple (push) Successful in 21m2s
docker / deploy-docs (push) Successful in 21s
apple / screenshots (push) Successful in 19m51s
cargo fmt --all --check flagged the reanchor gate wiring (decode.rs / session.rs /
abi.rs / reanchor.rs): wrapped signatures + comparisons, and two multi-line comments
that followed a trailing-comment line were restructured to their own lines so
rustfmt keeps them at normal indentation instead of deep-aligning them.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-07-13 07:36:43 +02:00
enricobuehler 644c035a34 feat(encode/amf): accept AMF runtime >=1.4.34 (graceful degrade) + log loaded amfrt64.dll identity
ci / rust (push) Failing after 47s
ci / web (push) Successful in 55s
ci / docs-site (push) Successful in 56s
decky / build-publish (push) Successful in 18s
docker / build-push (--build-arg FEDORA_VERSION=44, ci, ci/fedora-rpm.Dockerfile, punktfunk-fedora44-rpm) (push) Successful in 8s
docker / build-push (., web/Dockerfile, punktfunk-web) (push) Successful in 8s
docker / build-push (ci, ci/fedora-rpm.Dockerfile, punktfunk-fedora-rpm) (push) Successful in 7s
docker / build-push (ci, ci/rust-ci.Dockerfile, punktfunk-rust-ci) (push) Successful in 8s
docker / build-push (docs-site, docs-site/Dockerfile, punktfunk-docs) (push) Successful in 8s
docker / deploy-docs (push) Successful in 11s
ci / bench (push) Successful in 5m8s
android / android (push) Successful in 14m4s
arch / build-publish (push) Successful in 12m44s
deb / build-publish (push) Successful in 13m48s
rpm / build-publish (44, fedora-44, punktfunk-fedora44-rpm) (push) Successful in 12m4s
rpm / build-publish (43, bazzite, punktfunk-fedora-rpm) (push) Successful in 14m19s
windows-host / package (push) Successful in 9m36s
apple / swift (push) Successful in 4m40s
apple / screenshots (push) Successful in 19m47s
The Windows AMF encoder hard-rejected any runtime <1.4.36 — a Jan-2025
(Adrenalin 25.1.1) driver floor. Every AMD host on an older driver failed the
session with "update the AMD driver" after 8 retries, notably Boot Camp Macs
whose bundled amfrt64.dll lags far behind.

Split the single pin:
- AMF_MIN_VERSION (1.4.34): the ABI floor accepted at load. Every vtable slot
  the FFI mirrors is a base-interface slot stable since well before 1.4.34; the
  1.4.35/1.4.36-only features are string-keyed encoder properties already applied
  via set_prop(required=false), which log-and-continue — so an older driver
  degrades those features individually instead of failing.
- AMF_HEADER_VERSION (1.4.36): the header the mirror targets, now passed to
  AMFInit capped at min(header, runtime) so claiming a version newer than the
  runtime can't make AMFInit reject an otherwise-usable older driver.

No functionality removed: a >=1.4.36 runtime behaves exactly as before.

Also logs, once per process, the AMF runtime version AND the loaded amfrt64.dll's
full path + file-version resource (via GetModuleFileNameW + VerQueryValueW). This
surfaces the Boot Camp failure mode where the display driver reads 25.x but the
System32 amfrt64.dll is a stale build reporting an old AMF version; the too-old
decline now names the DLL path/build and points at reboot + DDU reinstall.

Not compile-verified: amf.rs is Windows-only and this Linux box can't cross-build
it (a dependency's C build fails for the msvc target). Needs cargo check/clippy on
the Windows build box / CI. rustfmt-clean; the windows-crate FFI signatures were
verified against the on-disk 0.62.2 bindings.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-07-13 01:32:51 +02:00
enricobuehler 3c16c1dd30 chore(release): bump workspace version to 0.10.0
audit / bun-audit (push) Successful in 12s
ci / rust (push) Failing after 42s
ci / docs-site (push) Successful in 48s
ci / web (push) Successful in 1m17s
audit / cargo-audit (push) Successful in 2m35s
ci / bench (push) Successful in 5m56s
windows / build (aarch64-pc-windows-msvc) (push) Failing after 5m9s
apple / swift (push) Successful in 4m44s
windows / build (x86_64-pc-windows-msvc) (push) Failing after 5m31s
apple / screenshots (push) Successful in 19m30s
decky / build-publish (push) Successful in 24s
docker / build-push (--build-arg FEDORA_VERSION=44, ci, ci/fedora-rpm.Dockerfile, punktfunk-fedora44-rpm) (push) Successful in 27s
docker / build-push (., web/Dockerfile, punktfunk-web) (push) Successful in 9s
docker / build-push (ci, ci/rust-ci.Dockerfile, punktfunk-rust-ci) (push) Successful in 8s
docker / build-push (docs-site, docs-site/Dockerfile, punktfunk-docs) (push) Successful in 11s
android-screenshots / screenshots (push) Successful in 3m19s
docker / build-push (ci, ci/fedora-rpm.Dockerfile, punktfunk-fedora-rpm) (push) Successful in 6m58s
windows-host / package (push) Successful in 9m32s
linux-client-screenshots / screenshots (push) Successful in 8m5s
docker / deploy-docs (push) Successful in 22s
web-screenshots / screenshots (push) Successful in 2m44s
windows-msix / package (arm64, C:\Users\Public\ffmpeg-arm64, --no-default-features, aarch64-pc-windows-msvc, C:\t-a64) (push) Successful in 2m36s
arch / build-publish (push) Successful in 12m38s
deb / build-publish (push) Successful in 12m44s
android / android (push) Successful in 13m10s
windows-msix / package (x64, C:\Users\Public\ffmpeg, , x86_64-pc-windows-msvc, C:\t) (push) Successful in 4m28s
rpm / build-publish (43, bazzite, punktfunk-fedora-rpm) (push) Successful in 13m59s
release / apple (push) Successful in 24m1s
rpm / build-publish (44, fedora-44, punktfunk-fedora44-rpm) (push) Successful in 19m20s
flatpak / build-publish (push) Failing after 8m2s
Bumps [workspace.package] version 0.9.2 -> 0.10.0 (14 workspace crates). Apple
MARKETING_VERSION / Android versionName are set from the release tag by CI, so no
client manifest changes; the nested Windows-driver workspace keeps its independent
0.0.1 version.

Also syncs Cargo.lock: the version bump for the 14 members, plus dropping the
now-orphaned crossbeam-channel entry the Windows builtin removal (ef580825) left
behind (it dropped the dep from the manifest but not the lock), so
`cargo build --locked` (ci.yml / deb.yml) stays green.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-07-13 01:32:26 +02:00
enricobuehler c0fc2d8ee8 feat(apple): iPad ⌃⌥⇧Q release chord + click-to-recapture, pixel-grid snap, match-window opt-in
ci / docs-site (push) Successful in 49s
ci / rust (push) Failing after 51s
ci / web (push) Successful in 52s
docker / build-push (--build-arg FEDORA_VERSION=44, ci, ci/fedora-rpm.Dockerfile, punktfunk-fedora44-rpm) (push) Successful in 8s
docker / build-push (., web/Dockerfile, punktfunk-web) (push) Successful in 9s
decky / build-publish (push) Successful in 16s
docker / build-push (ci, ci/fedora-rpm.Dockerfile, punktfunk-fedora-rpm) (push) Successful in 9s
docker / build-push (docs-site, docs-site/Dockerfile, punktfunk-docs) (push) Successful in 9s
deb / build-publish (push) Failing after 4m20s
apple / swift (push) Successful in 4m43s
arch / build-publish (push) Failing after 4m56s
rpm / build-publish (43, bazzite, punktfunk-fedora-rpm) (push) Failing after 4m8s
ci / bench (push) Successful in 5m43s
docker / build-push (ci, ci/rust-ci.Dockerfile, punktfunk-rust-ci) (push) Successful in 6m21s
docker / deploy-docs (push) Successful in 21s
rpm / build-publish (44, fedora-44, punktfunk-fedora44-rpm) (push) Failing after 3m33s
windows-host / package (push) Successful in 9m2s
flatpak / build-publish (push) Failing after 8m11s
android / android (push) Successful in 12m4s
windows-msix / package (arm64, C:\Users\Public\ffmpeg-arm64, --no-default-features, aarch64-pc-windows-msvc, C:\t-a64) (push) Successful in 3m54s
windows-msix / package (x64, C:\Users\Public\ffmpeg, , x86_64-pc-windows-msvc, C:\t) (push) Successful in 3m39s
windows / build (aarch64-pc-windows-msvc) (push) Failing after 5m3s
release / apple (push) Successful in 21m56s
windows / build (x86_64-pc-windows-msvc) (push) Failing after 5m31s
apple / screenshots (push) Successful in 19m11s
- InputCapture / StreamViewIOS: iPad ⌃⌥⇧Q un-capture chord, recognized from the
  GCKeyboard HID stream (no NSEvent monitor on iOS) for cross-client parity with
  the macOS/Windows/Linux combo; and a click into the video re-engages capture —
  the iPad analogue of macOS mouseDown → engageCapture(fromClick:), with the
  engaging click suppressed toward the host.
- SessionPresenter: snap the aspect-fit sublayer frame to the backing pixel grid.
  AVMakeRect centers the fit rect at fractional points, so the compositor
  resampled the layer — a uniform "everything soft" blur even when the drawable
  was pixel-exact 1:1. Rounding origin + size to device pixels makes the composite
  a true 1:1 blit; idempotent when already aligned.
- MetalVideoPresenter: PUNKTFUNK_BILINEAR_LUMA=1 A/B lever — compiles the shader
  with Catmull-Rom luma off (plain bilinear) to isolate bicubic overshoot from
  upstream fringing.
- SettingsView / StreamView / StreamViewIOS: match-window reverted to opt-in
  (default OFF) — the explicit mode is used and never auto-resized unless enabled.
2026-07-13 01:22:09 +02:00
enricobuehler ef5808254a refactor(windows): remove the legacy in-process builtin stream path
The real Windows client is the spawned punktfunk-session Vulkan binary
(pf-client-core); the in-process builtin GUI stream — reachable only via
PUNKTFUNK_BUILTIN_STREAM=1 — was dead weight kept alive by nothing and a
recurring source of wasted effort. Remove it: delete present/render/input/
audio.rs and the builtin remainder of session/video.rs, rip all the builtin
wiring (app/mod, connect, stream), and make connect always spawn.

Preserve the two shipped keepers that happened to live in those files by
relocating them to a new probe.rs: run_speed_probe (the per-host network speed
test used by the Settings speed page and --headless --speed-test) and
decodable_codecs (the codec-capability advert on the probe connect). Trim gpu.rs
to just the Settings adapter picker (adapter_names + helpers). --headless now
supports only --speed-test — the in-process decode/frame-counter went with the
pump.

Drops the now-orphaned deps opus, wasapi, crossbeam-channel, anyhow; keeps
ffmpeg-next (probe::decodable_codecs still needs it). Net 4432 deletions.
Statically verified (module wiring, imports, orphaned symbols/deps all clean);
the type-level compile runs on the windows-amd64 CI runner, which has the
toolchain this non-Windows host lacks.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-07-13 01:22:09 +02:00
enricobuehler 8a18e130a2 feat(client): freeze-until-reanchor loss recovery on Android + Apple via shared core gate
After unrecoverable loss the host keeps sending delta frames that reference a
picture the client never received; hardware decoders conceal these as gray/
garbage with a success status. Linux already withheld them and held the last
good frame until a proven clean re-anchor — this brings that behavior to the
Android and Apple clients.

Extract the Linux pump's freeze state machine into a shared `ReanchorGate` in
punktfunk-core (reanchor.rs, 18 tests) exposed over the C ABI (ABI v6, additive —
no wire change) for the Swift clients. Migrate the Linux/Deck pump
(pf-client-core) onto it as the parity proof (no-op refactor). Then wire:

- Android (decode.rs, both sync + async loops): arm on the frame-index gap, a
  pts-keyed flag map carries the wire flags to the output-buffer release, fold
  the gate per drained output, gate.poll replaces the dropped-climb block.
- Apple Stage2Pipeline (default): arm on a gap (new noteFrameIndexGap), withhold
  at the ring-submit seam (CAMetalLayer holds its last drawable), poll
  framesDropped, fold VT decode errors through the no-output streak.
- Apple StreamPump (stage-1): fold at enqueue, withhold via
  kCMSampleAttachmentKey_DoNotDisplay so the layer keeps decoding (reference
  chain intact) but holds the last displayed frame.
- Apple VideoDecoder: thread the AU's wire flags to the async decode callback via
  a retained FrameContext refcon (replaces the receivedNs bit-pattern scalar).

Lifts only on a proven re-anchor (IDR / RFI anchor / 2nd recovery mark) with a
500 ms backstop so a lost re-anchor can never freeze forever. Apple: swift build
clean, 123/123 tests pass (incl. VideoToolboxRoundTripTests). On-glass
loss-injection validation still owed.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-07-13 01:22:09 +02:00
enricobuehler cd701a9594 fix(vdisplay): preserve FramePublisher across swap-chain reassign (sibling-join freeze)
windows-drivers / probe-and-proto (push) Successful in 28s
ci / web (push) Successful in 54s
ci / docs-site (push) Successful in 1m0s
docker / build-push (--build-arg FEDORA_VERSION=44, ci, ci/fedora-rpm.Dockerfile, punktfunk-fedora44-rpm) (push) Successful in 8s
decky / build-publish (push) Successful in 19s
docker / build-push (., web/Dockerfile, punktfunk-web) (push) Successful in 8s
docker / build-push (ci, ci/rust-ci.Dockerfile, punktfunk-rust-ci) (push) Successful in 9s
docker / build-push (docs-site, docs-site/Dockerfile, punktfunk-docs) (push) Successful in 7s
windows-drivers / driver-build (push) Successful in 1m35s
apple / swift (push) Successful in 4m34s
ci / bench (push) Successful in 5m39s
docker / build-push (ci, ci/fedora-rpm.Dockerfile, punktfunk-fedora-rpm) (push) Successful in 6m50s
docker / deploy-docs (push) Successful in 33s
windows-host / package (push) Successful in 10m8s
android / android (push) Successful in 12m24s
arch / build-publish (push) Successful in 16m41s
deb / build-publish (push) Successful in 17m40s
rpm / build-publish (44, fedora-44, punktfunk-fedora44-rpm) (push) Successful in 12m13s
rpm / build-publish (43, bazzite, punktfunk-fedora-rpm) (push) Successful in 18m21s
ci / rust (push) Successful in 23m52s
apple / screenshots (push) Successful in 20m9s
When a second client got its own virtual display mid-stream, the FIRST
client's IDD-push stream froze (video only; `new_fps=0 repeat_fps=240`
forever). Adding/removing/resizing a sibling display re-commits the CCD
topology, which makes the OS unassign+reassign the first monitor's IddCx
swap chain. `unassign_swap_chain` dropped the SwapChainProcessor, dropping
`run_core`'s local FramePublisher and closing the sealed-channel handles.
The fresh worker then polled the frame-channel stash — but that stash is
consumed once at session open, and the host only re-delivers on a ring
recreate (a descriptor change). The first monitor's descriptor didn't
change and WUDFHost stayed alive, so no watchdog fired: the driver drained
the swap chain without publishing and the host repeated its last frame
indefinitely. Confirmed twice on the .173 box (host.log 21:12 & 21:15).

Preserve the live FramePublisher across the flap instead of dropping it:
the host-owned ring (header/event/textures) it holds stays valid — only
the swap chain died.

- frame_transport.rs: FramePublisher records its render-adapter LUID +
  exposes render_adapter().
- monitor.rs: MonitorObject.preserved_publisher + preserve_publisher()
  (mirrors set_frame_channel) + take_preserved_publisher() (mirrors
  take_frame_channel). Monitor teardown drops the stashed publisher and
  closes its ring handles, so nothing leaks.
- swap_chain_processor.rs run_core: after SetDevice OK, re-adopt a
  preserved publisher ONLY when the new swap chain renders on the same
  LUID (same pooled Direct3DDevice → its context + opened textures are
  valid); on loop exit, stash the publisher back on the monitor.

Safe: the old worker is fully joined (drop-outside-lock discipline)
before the new one runs, so no concurrent context use; a stale re-adopted
publisher is superseded by the existing is_stale() + has_frame_channel()
newest-wins checks at the loop top.

Verified clippy -D warnings clean on rustc 1.96.0 via a faithful mock
crate (the real crate needs the WDK to compile). Needs a driver rebuild +
reinstall on the host to take effect; not yet hardware-validated.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-07-13 00:19:20 +02:00
enricobuehler 05868ef634 fix(encode): Vulkan-HEVC full-RPS reference retention + AV1 feature gate (RFI review)
ci / web (push) Successful in 56s
ci / docs-site (push) Successful in 1m0s
docker / build-push (--build-arg FEDORA_VERSION=44, ci, ci/fedora-rpm.Dockerfile, punktfunk-fedora44-rpm) (push) Successful in 14s
decky / build-publish (push) Successful in 30s
docker / build-push (ci, ci/fedora-rpm.Dockerfile, punktfunk-fedora-rpm) (push) Successful in 11s
docker / build-push (ci, ci/rust-ci.Dockerfile, punktfunk-rust-ci) (push) Successful in 8s
docker / build-push (docs-site, docs-site/Dockerfile, punktfunk-docs) (push) Successful in 10s
docker / build-push (., web/Dockerfile, punktfunk-web) (push) Successful in 52s
apple / swift (push) Successful in 4m47s
ci / bench (push) Successful in 6m2s
docker / deploy-docs (push) Successful in 24s
windows-host / package (push) Successful in 8m6s
arch / build-publish (push) Successful in 11m4s
rpm / build-publish (44, fedora-44, punktfunk-fedora44-rpm) (push) Successful in 13m19s
android / android (push) Successful in 16m58s
deb / build-publish (push) Successful in 17m45s
rpm / build-publish (43, bazzite, punktfunk-fedora-rpm) (push) Successful in 18m38s
ci / rust (push) Successful in 24m8s
apple / screenshots (push) Successful in 20m38s
2026-07-12 review of the host encoders / client decoders / RFI plane.
NVENC (both), AMF-LTR, the session glue, and the client RfiTracker came
out clean; every fix lands in the Vulkan Video backend + dispatch:

1. HEVC: author each P-frame's short-term RPS to retain ALL resident DPB
   pictures (minus the setup slot), not just its one reference. HEVC
   8.3.2 evicts unlisted pictures, and clients keep FEEDING the decoder
   while frozen — so with the old single-pic RPS, a conforming parser
   (FFmpeg = the Linux VAAPI/Vulkan and Windows D3D11VA clients) had
   already discarded the picture an RFI recovery anchor references
   whenever a fed post-loss frame preceded it: generate_missing_ref, and
   the "clean" anchor plus everything chained after it decodes as
   garbage. Pure builder (`build_h265_rps_s0`) + unit tests; AV1 needs
   nothing (slot-based retention). The smoke test now encodes a fed
   post-loss frame between loss@4 and anchor@6 so an ffmpeg decode of
   the dropped dump exercises exactly this (expect ONE POC-4 complaint,
   never POC 3) — revalidate on the AMD box; this NVIDIA dev box fails
   the backend earlier at HEVC header retrieval (pre-existing).

2. AV1: chain PhysicalDeviceVideoEncodeAV1FeaturesKHR (videoEncodeAV1 =
   TRUE, stype 1000513004) into device creation — spec-required for the
   ENCODE_AV1 codec op; RADV tolerated the omission, validation layers
   and stricter drivers do not.

3. RFI decline no longer self-arms force_kf — that bypassed the session
   glue's 750 ms IDR cooldown, turning a storm of hopeless RFI requests
   into one full IDR each. Decline like NVENC/AMF and let the caller's
   coalesced keyframe path own the fallback; add the missing
   first>last guard for parity.

4. open_video_backend now returns the label of the branch that ACTUALLY
   opened, so the mgmt API / web console reports "vulkan" instead of
   "vaapi" for the default-on Vulkan sessions (the old dispatch-mirror
   resolved_backend_label went stale when the backend gained its VAAPI
   fallback; deleted).

Structure: the ~230-line inline HEVC coding block moves to
record_coding_h265 (symmetric with record_coding_av1) and the duplicated
pre-encode barriers dedupe into begin_encode_cmd.

Follow-up plan (separate, punktfunk-planning): bring the post-loss
freeze + RECOVERY_ANCHOR/POINT lift to the Android/Windows/Apple clients
via a shared ReanchorGate (design/client-reanchor-freeze-parity.md).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-12 23:12:12 +02:00
enricobuehler 2d37835545 feat(encode): AV1 on the Linux Vulkan Video encoder (real RFI)
ci / web (push) Successful in 1m12s
ci / docs-site (push) Successful in 1m22s
decky / build-publish (push) Successful in 19s
ci / bench (push) Successful in 5m36s
docker / build-push (--build-arg FEDORA_VERSION=44, ci, ci/fedora-rpm.Dockerfile, punktfunk-fedora44-rpm) (push) Successful in 9s
docker / build-push (., web/Dockerfile, punktfunk-web) (push) Successful in 8s
docker / build-push (ci, ci/fedora-rpm.Dockerfile, punktfunk-fedora-rpm) (push) Successful in 8s
docker / build-push (ci, ci/rust-ci.Dockerfile, punktfunk-rust-ci) (push) Successful in 8s
docker / build-push (docs-site, docs-site/Dockerfile, punktfunk-docs) (push) Successful in 8s
arch / build-publish (push) Successful in 12m32s
docker / deploy-docs (push) Successful in 21s
android / android (push) Successful in 13m27s
ci / rust (push) Successful in 17m43s
deb / build-publish (push) Successful in 13m40s
rpm / build-publish (44, fedora-44, punktfunk-fedora44-rpm) (push) Successful in 15m19s
rpm / build-publish (43, bazzite, punktfunk-fedora-rpm) (push) Successful in 15m36s
windows-host / package (push) Successful in 14m43s
apple / swift (push) Successful in 4m42s
apple / screenshots (push) Successful in 19m20s
Extend the raw Vulkan Video backend to AV1 (`VK_KHR_video_encode_av1`)
alongside HEVC, so AMD/Intel Linux hosts get the same clean-P-frame loss
recovery for AV1 that HEVC already has — no full IDR on packet loss.

ash 0.38.0+1.3.281 predates the AV1-encode extension (finalized in Vulkan
1.3.290) and bumping ash breaks the SDL/Vulkan client (it drops the
lifetime on AllocationCallbacks, which sdl3-sys still generates). So the
AV1-encode structs/flags/enums are vendored host-only in
`encode/linux/vk_av1_encode.rs`, copied verbatim from ash-master's
generated code and chained into ash's generic video-encode calls via raw
p_next — the common StdVideoAV1* types (from AV1 decode) are reused from
ash 1.3.281.

`vulkan_video.rs` gains a parallel AV1 path: AV1 Main profile/caps/session
(+ max-level session-create), a bit-packed sequence-header OBU + per-frame
temporal-delimiter framing (Vulkan AV1 encode, unlike H26x, emits only the
frame OBU), and per-frame StdVideoEncodeAV1PictureInfo with the RFI
reference model — a normal P inherits CDF context from its reference for
compression, while an IDR or recovery anchor sets primary_ref_frame=NONE +
error_resilient_mode so it decodes independent of the (possibly lost)
frames since its reference. HEVC recording is unchanged; the shared CSC /
ring / DPB-barrier pipeline is reused as-is. Codec routing in
`open_video_backend` extends the HEVC arm to HEVC|AV1.

The seq header enables only order-hint (+128px superblocks per caps),
matching FFmpeg's proven Vulkan AV1 config — enabling CDEF/restoration made
VCN emit frame-header sections our seq header didn't match, desyncing every
inter frame.

Headless-validated on real RADV (780M): open + 6-frame encode (I P P P P P)
decodes 0-error on both dav1d and ffmpeg/cbs; the RFI recovery anchor at
frame 4 is a clean P (not IDR), and dropping the "lost" frame 3 still
decodes clean (re-anchored to frame 2). HEVC smoke unchanged (no
regression). `cargo clippy --features vulkan-encode -- -D warnings` and the
no-feature build both green.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-07-12 22:06:24 +02:00
enricobuehler 9514a8c0e2 fix(client): correct Linux/Windows "Forwarded controller" copy for multi-controller
ci / web (push) Successful in 51s
ci / docs-site (push) Successful in 56s
docker / build-push (--build-arg FEDORA_VERSION=44, ci, ci/fedora-rpm.Dockerfile, punktfunk-fedora44-rpm) (push) Successful in 8s
docker / build-push (., web/Dockerfile, punktfunk-web) (push) Successful in 8s
docker / build-push (ci, ci/fedora-rpm.Dockerfile, punktfunk-fedora-rpm) (push) Successful in 11s
docker / build-push (ci, ci/rust-ci.Dockerfile, punktfunk-rust-ci) (push) Successful in 10s
decky / build-publish (push) Successful in 52s
docker / build-push (docs-site, docs-site/Dockerfile, punktfunk-docs) (push) Successful in 8s
apple / swift (push) Successful in 4m39s
ci / bench (push) Successful in 6m27s
windows-host / package (push) Successful in 8m34s
flatpak / build-publish (push) Successful in 7m15s
docker / deploy-docs (push) Successful in 24s
android / android (push) Successful in 12m49s
windows-msix / package (arm64, C:\Users\Public\ffmpeg-arm64, --no-default-features, aarch64-pc-windows-msvc, C:\t-a64) (push) Successful in 4m16s
deb / build-publish (push) Successful in 12m50s
arch / build-publish (push) Successful in 16m32s
windows-msix / package (x64, C:\Users\Public\ffmpeg, , x86_64-pc-windows-msvc, C:\t) (push) Successful in 3m53s
rpm / build-publish (43, bazzite, punktfunk-fedora-rpm) (push) Successful in 19m44s
windows / build (aarch64-pc-windows-msvc) (push) Successful in 5m14s
rpm / build-publish (44, fedora-44, punktfunk-fedora44-rpm) (push) Successful in 17m53s
ci / rust (push) Successful in 25m18s
release / apple (push) Successful in 22m3s
windows / build (x86_64-pc-windows-msvc) (push) Successful in 5m57s
apple / screenshots (push) Successful in 18m56s
The relm4 (Linux) and native (Windows) shells spawn punktfunk-session, which
since the native-plane rework forwards ALL controllers by default — but the
"Forwarded controller" settings dropdowns still described the pre-rework world
("Automatic (most recent)", "Exactly one controller is forwarded to the host").

The dropdown already lists every detected pad and wires set_pinned(None)=all /
set_pinned(key)=single-player; this fixes only the misleading labels, subtitle,
tooltip, and stale leading comments to match: Automatic forwards every real
controller (each its own player); pick one to force single-player.

cargo check -p punktfunk-client-linux green; Windows is windows-gated (pure
string edits, CI windows.yml).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-07-12 21:53:07 +02:00
enricobuehler 97c67b2692 feat(apple): multi-controller support
Roll the pf-client-core slot pattern to the Apple client (Swift):

- GamepadManager tracks all connected GCControllers, assigning each a stable
  lowest-free wire pad index + concrete type, emitting GamepadArrival on
  connect and GamepadRemove on disconnect (index freed for reuse on re-plug).
- GamepadCapture binds every controller with per-controller Slot state
  (buttons/axes/fingers/motion), threading the pad index into flags on every
  event; GamepadWire/InputEvents carry the pad + the two new events.
- GamepadFeedback + RumbleRenderer go per-pad (rumbleByPad, slots[pad]),
  routing rumble/HID back to the correct controller by wire index.
- ContentView/Settings surface every forwarded controller.

pad 0 => flags 0, so single-controller wire is byte-identical. Cannot build
on the Linux dev box (no Swift toolchain / Apple frameworks); wire bytes
hand-checked against input.rs and GamepadWireTests extended for multi-pad.
CI apple.yml (swift build/test on macOS) is the compile gate.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-07-12 21:53:07 +02:00
enricobuehler 0ad4e6eff7 feat(android): multi-controller support
Roll the pf-client-core slot pattern to the Android client (Kotlin + JNI):

- New kit/GamepadRouter.kt: the Android analogue of the client-core Slot
  model — a deviceId→Slot map assigning each InputDevice a stable lowest-free
  wire pad index held for its lifetime, GamepadArrival(pref) before a pad's
  first input, GamepadRemove on onInputDeviceRemoved, per-slot AxisMapper +
  held-bitmask so two pads never clobber each other. The isForwardable gate
  (excludes DualSense/DS4 all-zero sensor sibling nodes) is centralized in
  slotFor so no entry point can open a phantom slot.
- native/src/session/input.rs: JNI shims take a pad arg -> flags=pad
  (nativeSendGamepadButton/Axis, plus nativeSendGamepadArrival/Remove).
- native/src/feedback.rs: pad carried in rumble bits 49..52 + a leading
  hidout pad byte; GamepadFeedback.kt routes rumble/lightbar/LED back to the
  originating device by pad via deviceForPad.
- MainActivity.kt routes key/motion events by device; ControllersScreen.kt
  badges every forwarded pad (was hardcoded i==0), reading getControllerNumber.

A lone controller lands on wire index 0, so its per-transition datagrams stay
byte-identical to the old single-pad path. gradle :app:assembleDebug green
(Rust cross-compiled via cargo-ndk); JNI signatures hand-verified 1:1.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-07-12 21:53:07 +02:00
enricobuehler 76be4c3e12 feat(gamepad): multi-controller support on the native plane
Host was already built for 16 pads; the blocker was every client
hard-coding pad 0. This lands the host-side + reference-client contract:

- input.rs: new wire kinds GamepadArrival=14 (declares a pad's type:
  code=GamepadPref byte, flags=pad) and GamepadRemove=13 (flags=seq<<24|pad,
  shares the snapshot seq space via encode/decode_gamepad_remove).
- pf-client-core/gamepad.rs: reworked from a single `open` pad to a
  slots: Vec<Slot> model — every forwarded controller gets a stable
  lowest-free wire index held for its lifetime, per-slot held/axis/touch/
  rumble state, GamepadArrival on open + GamepadRemove on close, and
  feedback routed back per wire index. Automatic forwards all real pads;
  a pin forces single-player.
- punktfunk1.rs: replaced the single-session PadBackend enum with a Pads
  router — per-pad kinds[]/owner[] arrays, lazily-created per-kind managers,
  pure route_decision keeping a live device in its manager across a kind
  change (no ghost/dup). Input thread seq-gates GamepadRemove (clears the
  pad_mask bit, resets rumble) and applies GamepadArrival kinds.
- inject linux/windows backends: add the two new no-op InputKind arms.

Native/session + default-Windows clients (both spawn punktfunk-session)
inherit this. 57 core + 33 client-core + 272 host tests green; clippy clean.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-07-12 21:53:07 +02:00
enricobuehler 84329205eb feat(encode): default-on the Linux Vulkan Video HEVC backend
ci / docs-site (push) Successful in 59s
ci / web (push) Successful in 1m7s
decky / build-publish (push) Successful in 18s
docker / build-push (., web/Dockerfile, punktfunk-web) (push) Successful in 8s
docker / build-push (ci, ci/fedora-rpm.Dockerfile, punktfunk-fedora-rpm) (push) Successful in 8s
docker / build-push (ci, ci/rust-ci.Dockerfile, punktfunk-rust-ci) (push) Successful in 8s
docker / build-push (docs-site, docs-site/Dockerfile, punktfunk-docs) (push) Successful in 44s
apple / swift (push) Successful in 4m13s
ci / bench (push) Successful in 7m13s
windows-host / package (push) Successful in 9m43s
docker / build-push (--build-arg FEDORA_VERSION=44, ci, ci/fedora-rpm.Dockerfile, punktfunk-fedora44-rpm) (push) Successful in 9m26s
docker / deploy-docs (push) Successful in 28s
android / android (push) Successful in 12m35s
deb / build-publish (push) Successful in 12m46s
arch / build-publish (push) Successful in 16m31s
rpm / build-publish (43, bazzite, punktfunk-fedora-rpm) (push) Successful in 19m32s
apple / screenshots (push) Successful in 18m49s
ci / rust (push) Successful in 25m10s
rpm / build-publish (44, fedora-44, punktfunk-fedora44-rpm) (push) Successful in 18m0s
On-glass validated 2026-07-12 on an AMD RADV 780M with a real Deck-class
client: the pipelined raw-Vulkan HEVC encoder ran a rock-solid 1080p@240
session and healed loss with clean P-frame recovery anchors (real RFI the
libav VAAPI path can't express). Ship it on by default, mirroring the NVENC
default-on.

- vulkan_encode_enabled() defaults ON; PUNKTFUNK_VULKAN_ENCODE=0 is the libav
  VAAPI escape hatch. A failed open still falls back to VAAPI, so a device
  without h265 Vulkan encode (or an untested Intel/ANV that misbehaves at open)
  degrades gracefully instead of breaking the stream.
- Ring depth defaults to 2 (one frame of overlap, lowest added latency — the
  on-glass-validated real-time setting); PUNKTFUNK_VULKAN_INFLIGHT still tunes it.
- Compile --features punktfunk-host/vulkan-encode into the arch/deb/rpm host
  builds (pure-Rust ash, no new lib / no link-time dep), alongside nvenc.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-07-12 20:20:31 +02:00
enricobuehler e9d8f2bc04 perf(encode): pipeline the Vulkan Video encoder (frames in flight)
The Linux raw-Vulkan HEVC backend blocked on two GPU fences per frame, so
CPU readback and the next capture could not overlap the GPU encode. Refactor
into a small ring of per-in-flight-frame resources (own command buffers,
CSC descriptor set + Y/UV/NV12 scratch, bitstream buffer, feedback query and
sync objects) so submit() records into a free slot and returns without
waiting, and poll() reads back the oldest slot once its fence signals. The
pump's non-blocking poll then overlaps a frame's CSC+encode with the next
capture — the throughput win — with no capturer/pump change (VAAPI untouched).

- New `Frame` struct + `make_frame`; encoder holds `frames`/`ring`/`in_flight`.
- `record_submit` (non-blocking) + `read_slot` (fence-gated readback) replace
  the synchronous `encode_frame`; `enqueue` applies backpressure by draining
  the oldest slot when the ring is full.
- DPB self-barrier between consecutive encodes: orders frame N's reconstruct
  write before N+1's reference read now that they can be in flight together.
- flush() drains all in-flight slots in order; reset() waits idle + discards.
- Ring depth defaults to 3, overridable via PUNKTFUNK_VULKAN_INFLIGHT (2..=6).
- Smoke test drains via poll-loop + flush (async breaks one-AU-per-submit).

Headless-validated on real RADV 780M: cargo check (feature on/off) + clippy
-D warnings + rustfmt clean; smoke test at ring depth 2/3/6 all ffmpeg-decode
clean (I P P P P P) and drop-heal (I P P P P) with 0 errors.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-07-12 19:51:21 +02:00
enricobuehler bbbb7f5723 fix+perf(encode): clamp Vulkan CSC to source edge + cache dmabuf imports
Two refinements after the initial on-glass validation on RADV (780M):

- Green padding bar at non-16-aligned heights (e.g. 1080 → coded 1088): the CSC
  compute shader read past the edge of the shorter source dmabuf for the 8
  alignment-padding rows, producing undefined/green garbage that showed on a
  client rendering the coded frame. Clamp every source fetch to `textureSize-1`
  so padding rows duplicate the last real row (invisible, and the SPS
  conformance window still crops it for a compliant decoder). BT.709 conversion
  is byte-identical for in-bounds pixels. 5120x1440 (exactly aligned) was never
  affected.

- Per-frame dmabuf import churn: the backend created + imported + destroyed a
  VkImage every frame (allocation jitter → stutter). PipeWire cycles a small
  fixed pool, so import each underlying buffer ONCE (keyed by st_dev/st_ino —
  each frame's fd is a fresh dup of the same buffer) and reuse it, matching the
  CUDA-path VkBridge. First import acquires from the foreign producer; cached
  re-reads keep queue ownership and use a plain visibility barrier. On-glass:
  ~3-6 imports per session then silent (was ~one per frame at 240 Hz), stutter
  gone at resolutions with headroom.

Also adds a PF_SMOKE_W/H override to the headless smoke test to exercise the
conformance-window crop path (ffprobe confirms coded 1088 → displayed 1080).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-07-12 19:24:12 +02:00
enricobuehler 5ab6daa694 feat(encode): raw Vulkan Video HEVC backend on Linux (AMD/Intel) with real RFI
Add `VulkanVideoEncoder` (`VK_KHR_video_encode_h265` via ash) — the open-stack
twin of the direct-NVENC RFI path, giving AMD/Intel Linux hosts real
reference-frame invalidation loss recovery: a clean P-frame recovery anchor
that re-references a known-good older frame instead of a full IDR. The app owns
the DPB, so recovery = pointing the P-frame's single L0 reference at a resident
slot strictly older than the loss (never a concealed frame).

The backend owns its own ash instance/device with encode + compute queues,
authors VPS/SPS/PPS (Main, conformance-window crop for non-16-aligned heights
like 1080->1088), runs a DPB-ring reference-slot state machine with monotonic
POC and CBR rate control, and does an on-GPU RGB->NV12 BT.709 compute CSC
(embedded rgb2yuv.spv) since capture delivers packed-RGB dmabufs — importing
each frame's dmabuf (explicit DRM modifier) or uploading a CPU-RGB fallback,
CSC on the compute queue, then encode on the encode queue, ordered by a
semaphore.

Wired into `open_video_backend`: an AMD/Intel HEVC session opens this instead
of libav VAAPI when `PUNKTFUNK_VULKAN_ENCODE=1` (VAAPI fallback on any open
error, so it can only improve recovery, never break a stream); `PUNKTFUNK_
ENCODER=vulkan` forces it. Gated behind the new `vulkan-encode` Cargo feature,
which pulls no new dependency (reuses the `ash` bindings already carried for
the dmabuf zero-copy bridge). Opt-in until on-glass validated, mirroring how
the direct-NVENC path shipped.

Headless-validated on real RADV (RDNA3 780M, Mesa 26): open + multi-frame
encode + `invalidate_ref_frames` all run through the real struct and ffmpeg
decodes the output `I P P P P P` with 0 errors; the recovery frame is a clean
P-frame (not an IDR); and dropping the "lost" AU still decodes cleanly because
the recovery re-anchored to an older frame — the RFI heal, proven on real
hardware. `cargo check`/`clippy -D warnings` green with the feature on and off.

Design: design/linux-vulkan-video-encode.md. Harness: design/vkenc-probe-harness/.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-07-12 17:08:15 +02:00
enricobuehler 76594f27c1 feat(nvenc): default-on the Linux direct-SDK NVENC path
ci / web (push) Successful in 45s
ci / docs-site (push) Successful in 1m5s
decky / build-publish (push) Successful in 20s
docker / build-push (--build-arg FEDORA_VERSION=44, ci, ci/fedora-rpm.Dockerfile, punktfunk-fedora44-rpm) (push) Successful in 9s
docker / build-push (., web/Dockerfile, punktfunk-web) (push) Successful in 7s
docker / build-push (ci, ci/fedora-rpm.Dockerfile, punktfunk-fedora-rpm) (push) Successful in 7s
docker / build-push (ci, ci/rust-ci.Dockerfile, punktfunk-rust-ci) (push) Successful in 8s
docker / build-push (docs-site, docs-site/Dockerfile, punktfunk-docs) (push) Successful in 8s
apple / swift (push) Successful in 4m49s
ci / bench (push) Successful in 6m13s
docker / deploy-docs (push) Successful in 20s
windows-host / package (push) Successful in 14m19s
arch / build-publish (push) Successful in 20m25s
rpm / build-publish (43, bazzite, punktfunk-fedora-rpm) (push) Successful in 18m28s
android / android (push) Successful in 21m8s
deb / build-publish (push) Successful in 20m53s
rpm / build-publish (44, fedora-44, punktfunk-fedora44-rpm) (push) Successful in 20m25s
ci / rust (push) Successful in 24m14s
apple / screenshots (push) Successful in 21m0s
On-glass validated 2026-07-12 on an RTX 5070 Ti host with a real Steam Deck
client (NV12 4:2:0, 1280x800@90): real nvEncInvalidateRefFrames landed 73/73
as clean P-frame recovery anchors (never IDR), losses consistently ~2 frames
deep — well inside the 5-frame DPB. That is the loss recovery the libav
hevc_nvenc path cannot express, so make the direct path the default on NVIDIA.

PUNKTFUNK_NVENC_DIRECT=0 (also false/no/off) is now the libav escape hatch.
Still gated on a CUDA capture payload — the `cuda` check in open_nvenc_probed
keeps AMD/Intel on VAAPI regardless, and the NVENC/CUDA entry points stay
dlopen'd at runtime (no new DT_NEEDED), so non-NVIDIA hosts are unaffected.
Packaging comments updated to match.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-07-12 13:34:53 +02:00
enricobuehler e89b2f60eb build(linux): enable --features nvenc in the arch/deb/rpm host builds
ci / web (push) Successful in 53s
ci / docs-site (push) Successful in 1m3s
decky / build-publish (push) Successful in 20s
docker / build-push (--build-arg FEDORA_VERSION=44, ci, ci/fedora-rpm.Dockerfile, punktfunk-fedora44-rpm) (push) Successful in 8s
docker / build-push (., web/Dockerfile, punktfunk-web) (push) Successful in 8s
docker / build-push (ci, ci/fedora-rpm.Dockerfile, punktfunk-fedora-rpm) (push) Successful in 7s
docker / build-push (docs-site, docs-site/Dockerfile, punktfunk-docs) (push) Successful in 6s
ci / bench (push) Successful in 6m36s
docker / build-push (ci, ci/rust-ci.Dockerfile, punktfunk-rust-ci) (push) Successful in 6m7s
docker / deploy-docs (push) Successful in 20s
deb / build-publish (push) Successful in 12m12s
apple / swift (push) Successful in 5m5s
rpm / build-publish (43, bazzite, punktfunk-fedora-rpm) (push) Successful in 12m55s
arch / build-publish (push) Successful in 15m6s
android / android (push) Successful in 20m8s
ci / rust (push) Successful in 23m44s
rpm / build-publish (44, fedora-44, punktfunk-fedora44-rpm) (push) Successful in 18m36s
apple / screenshots (push) Successful in 20m26s
The Phase 5.2 direct-SDK NVENC Linux backend (encode/linux/nvenc_cuda.rs) is
gated `#[cfg(all(target_os = "linux", feature = "nvenc"))]`, but no Linux
packager passed `--features nvenc` (it was historically a Windows-only feature
for the D3D11 NVENC path). So the module was compiled OUT of every arch/deb/rpm
canary regardless of commit — PUNKTFUNK_NVENC_DIRECT was a silent no-op on the
shipped Linux host. Add `--features punktfunk-host/nvenc` to all three package
builds so the code actually ships.

AMD/Intel-SAFE (verified): this is NOT the old Windows link-import crash. The
NVENC/CUDA entry points are dlopen'd at RUNTIME (libloading) — `objdump -p` shows
the feature build's DT_NEEDED is byte-identical to a plain build (no libcuda /
libnvidia-encode), so the binary starts fine driver-less. We use only the crate's
`sys::nvEncodeAPI` types (no code-running safe wrapper / lazy statics), cudarc
stays on `ci-check`/dynamic-loading (no CUDA toolkit at build), and the encoder is
only constructed for a CUDA capture frame + PUNKTFUNK_NVENC_DIRECT — never on a
VAAPI (AMD/Intel) host. The sysext images repackage these outputs, so they inherit
it; no other Linux host build compiles the binary.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-07-12 12:17:19 +02:00
enricobuehler 63bc2bb10f docs(web-console): fix stale http:// URLs — the console serves HTTPS on :47992
ci / docs-site (push) Successful in 48s
ci / web (push) Successful in 53s
docker / build-push (--build-arg FEDORA_VERSION=44, ci, ci/fedora-rpm.Dockerfile, punktfunk-fedora44-rpm) (push) Successful in 10s
decky / build-publish (push) Successful in 17s
docker / build-push (ci, ci/fedora-rpm.Dockerfile, punktfunk-fedora-rpm) (push) Successful in 9s
docker / build-push (., web/Dockerfile, punktfunk-web) (push) Successful in 14s
docker / build-push (ci, ci/rust-ci.Dockerfile, punktfunk-rust-ci) (push) Successful in 7s
docker / build-push (docs-site, docs-site/Dockerfile, punktfunk-docs) (push) Successful in 52s
arch / build-publish (push) Has been cancelled
ci / bench (push) Has been cancelled
ci / rust (push) Has been cancelled
deb / build-publish (push) Has been cancelled
android / android (push) Has been cancelled
apple / swift (push) Has been cancelled
apple / screenshots (push) Has been cancelled
docker / deploy-docs (push) Has been cancelled
rpm / build-publish (43, bazzite, punktfunk-fedora-rpm) (push) Has been cancelled
rpm / build-publish (44, fedora-44, punktfunk-fedora44-rpm) (push) Has been cancelled
windows-host / package (push) Has been cancelled
The punktfunk-web unit serves HTTP/1.1 over TLS with the host's own
self-signed identity cert, but several guides still told users to open
http://<host>:47992, which fails. Correct the scheme everywhere and note
the one-time browser cert warning in the canonical + SteamOS docs.

The RPM %post web hint was doubly wrong (http://<host-ip>:3000): wrong
scheme and wrong port — the service listens on :47992, not the :3000 dev
default. Also fixes scripts/web-init.sh, so the URL the SteamOS/Linux
installer prints at the end of setup is correct too.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-07-12 12:14:52 +02:00
enricobuehler ad532b08a0 style(encode): rustfmt the direct-SDK NVENC Linux backend
ci / web (push) Successful in 49s
ci / docs-site (push) Successful in 52s
decky / build-publish (push) Successful in 17s
docker / build-push (--build-arg FEDORA_VERSION=44, ci, ci/fedora-rpm.Dockerfile, punktfunk-fedora44-rpm) (push) Successful in 8s
docker / build-push (., web/Dockerfile, punktfunk-web) (push) Successful in 7s
docker / build-push (ci, ci/fedora-rpm.Dockerfile, punktfunk-fedora-rpm) (push) Successful in 6s
docker / build-push (ci, ci/rust-ci.Dockerfile, punktfunk-rust-ci) (push) Successful in 7s
docker / build-push (docs-site, docs-site/Dockerfile, punktfunk-docs) (push) Successful in 6s
ci / bench (push) Successful in 6m1s
docker / deploy-docs (push) Successful in 20s
arch / build-publish (push) Successful in 12m23s
android / android (push) Successful in 13m26s
deb / build-publish (push) Successful in 14m27s
rpm / build-publish (43, bazzite, punktfunk-fedora-rpm) (push) Successful in 15m22s
ci / rust (push) Successful in 18m25s
rpm / build-publish (44, fedora-44, punktfunk-fedora44-rpm) (push) Successful in 15m14s
windows-host / package (push) Successful in 14m15s
apple / swift (push) Successful in 4m49s
apple / screenshots (push) Successful in 20m11s
`cargo fmt` was not run on the Phase 5.2 additions (nvenc_cuda.rs + the encode.rs
dispatcher fork), failing the ci.yml rust fmt job. Whitespace/wrapping only — no
behavior change.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-07-12 11:35:07 +02:00
enricobuehler 93093f3cf9 feat(encode): direct-SDK NVENC on Linux (CUDA input) with real RFI
ci / rust (push) Failing after 1m1s
ci / web (push) Successful in 1m3s
ci / docs-site (push) Successful in 1m9s
docker / build-push (--build-arg FEDORA_VERSION=44, ci, ci/fedora-rpm.Dockerfile, punktfunk-fedora44-rpm) (push) Successful in 10s
decky / build-publish (push) Successful in 17s
docker / build-push (., web/Dockerfile, punktfunk-web) (push) Successful in 8s
docker / build-push (ci, ci/fedora-rpm.Dockerfile, punktfunk-fedora-rpm) (push) Successful in 8s
docker / build-push (ci, ci/rust-ci.Dockerfile, punktfunk-rust-ci) (push) Successful in 8s
docker / build-push (docs-site, docs-site/Dockerfile, punktfunk-docs) (push) Successful in 57s
apple / swift (push) Successful in 4m42s
ci / bench (push) Successful in 7m37s
docker / deploy-docs (push) Successful in 22s
flatpak / build-publish (push) Successful in 6m51s
windows-host / package (push) Successful in 14m26s
rpm / build-publish (44, fedora-44, punktfunk-fedora44-rpm) (push) Successful in 12m59s
arch / build-publish (push) Successful in 16m48s
deb / build-publish (push) Successful in 17m14s
windows-msix / package (arm64, C:\Users\Public\ffmpeg-arm64, --no-default-features, aarch64-pc-windows-msvc, C:\t-a64) (push) Successful in 3m49s
android / android (push) Successful in 18m24s
rpm / build-publish (43, bazzite, punktfunk-fedora-rpm) (push) Successful in 18m28s
windows-msix / package (x64, C:\Users\Public\ffmpeg, , x86_64-pc-windows-msvc, C:\t) (push) Successful in 3m36s
windows / build (aarch64-pc-windows-msvc) (push) Successful in 5m1s
release / apple (push) Successful in 22m56s
windows / build (x86_64-pc-windows-msvc) (push) Successful in 5m40s
apple / screenshots (push) Successful in 18m53s
Phase 5.2 of design/encoder-recovery-hardening.md (design/linux-direct-nvenc.md).
The Linux NVIDIA host encodes through libavcodec `hevc_nvenc`, which structurally
cannot express `nvEncInvalidateRefFrames` — so every FEC-unrecoverable loss is a
full IDR and, since the client freezes-until-reanchor, a per-loss freeze for
RTT+IDR-encode. This ports the Windows raw-NVENC backend to
NV_ENC_DEVICE_TYPE_CUDA over the shared CUcontext so Linux NVIDIA gets the same
real RFI + F2 recovery-anchor + reset() stall lever + HDR-SEI/Main10 plumbing.

New `encode/linux/nvenc_cuda.rs` (`NvencCudaEncoder`):
- runtime-loaded entry table via `dlopen libnvidia-encode.so.1` (never link-time,
  mirroring the zerocopy::cuda libcuda loader) — one binary still starts on
  AMD/Intel Linux boxes and falls through to VAAPI/software;
- session on the shared CUcontext (zerocopy::cuda::context());
- an encoder-owned ring of registered CUDADEVICEPTR input surfaces
  (zerocopy::cuda::InputSurface + a contiguous-NV12 allocator), each captured
  DeviceBuffer device→device copied in via the existing copy_* helpers — mirrors
  the libav recycled-hwframe-pool copy, so zero regression vs today;
- config/RFI/anchor/reset ported from the Windows backend; sync-only (NVENC async
  is Windows-only, so that whole subsystem is dropped);
- Main10/HDR-SEI wired but inert until a Linux P010 capture path (Phase 5.1).

Wired behind PUNKTFUNK_NVENC_DIRECT (default OFF) in open_nvenc_probed; the Windows
path is untouched (no shared extraction in v1). Two on-hardware `#[ignore]` smokes
added.

Validated on .21 (RTX 5070 Ti, driver 610.43.03): builds on Linux under ci-check,
clippy-clean, full host suite 272/0, NV12 smoke (8 AUs, real invalidate_ref_frames
+ recovery_anchor on a P-frame) and YUV444 FREXT smoke (6 AUs, chroma_444) green;
Windows compile unaffected. Owed: the client-in-the-loop matrix (RFI-survives-ABR,
reset() heal, A/B vs libav) and the default flip.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-07-12 11:18:21 +02:00
enricobuehler fdda7144ed fix(encode): harden loss-recovery correctness across host encoders (F1–F7)
Phases 1–4 of design/encoder-recovery-hardening.md — make the shipped RFI/
freeze-until-reanchor recovery honest and rebuild-safe across every backend.

F1 — frame-index domain desync: the encode loop now owns a session-lifetime
`au_seq`; `Encoder::submit_indexed(au_seq + inflight)` pins NVENC inputTimeStamp
and AMF LTR slots to the WIRE frame index, so `invalidate_ref_frames` compares
client frame numbers in the same domain and survives adaptive-bitrate rebuilds
(an internal counter desynced on the first rebuild → RFI silently dead / an AMF
force-ref onto a never-decoded frame). `FrameMsg.frame_index` →
`Session::seal_frame_at`; GameStream gets the same via `VideoPacketizer::
packetize(.., Some(idx))`.

F2 — Windows NVENC left the client frozen ~1s per loss: NVENC RFI was
transparent (no anchor tag) while the session glue armed the 750ms IDR cooldown,
so the freeze only lifted on the ~1s keyframe re-ask. NVENC now mirrors AMF —
`pending_anchor` tags the first post-invalidate AU (the clean re-anchor
P-frame) `recovery_anchor`, incl. the covering-range dedupe re-arm; the client
lifts at ~RTT.

F3 — speed-test probe filler burned video frame indexes: moved to its own index
space (`Packetizer::alloc_probe_index` + `Session::submit_probe_frame`) with a
second client reassembly window routed on FLAG_PROBE, gated on the new
VIDEO_CAP_PROBE_SEQ Hello bit (mid-session probes declined for older clients).

F4 — RFI range sanity cap: forward gaps wider than `packet::RFI_MAX_RANGE` (256)
resync via keyframe instead of an out-of-range RFI, host- and client-side
(client huge-gap → keyframe in `RfiRecovery::observe` + the pf-client-core pump).

F5 — reset() parity: Windows NVENC (teardown + lazy re-init), Linux VAAPI
(drop-inner), Linux NVENC (reopen from stored OpenArgs) now give the stall
watchdog a heal lever instead of ending the session.

F6 — sw.rs `pending: VecDeque` (was `Option`), killing the silent AU drop at
capturer pipeline depth > 1. F7 — doc sweep on the RFI/anchor comments.

Verified: punktfunk-core lib tests (macOS + Linux), full punktfunk-host suite on
Linux (RTX 5070 Ti), Windows compile. Owed: the on-glass client matrix (F2
freeze A/B, AMF LTR spike across a bitrate rebuild).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-07-12 11:17:19 +02:00
enricobuehler 0dc414f197 docs: guide for embedding the C ABI (webOS, Xbox, Tizen examples)
ci / web (push) Successful in 1m4s
ci / docs-site (push) Successful in 1m19s
decky / build-publish (push) Successful in 33s
docker / build-push (., web/Dockerfile, punktfunk-web) (push) Successful in 40s
apple / swift (push) Successful in 3m38s
ci / bench (push) Successful in 7m5s
docker / build-push (ci, ci/rust-ci.Dockerfile, punktfunk-rust-ci) (push) Successful in 9s
docker / build-push (docs-site, docs-site/Dockerfile, punktfunk-docs) (push) Successful in 9s
docker / build-push (--build-arg FEDORA_VERSION=44, ci, ci/fedora-rpm.Dockerfile, punktfunk-fedora44-rpm) (push) Successful in 9m21s
docker / build-push (ci, ci/fedora-rpm.Dockerfile, punktfunk-fedora-rpm) (push) Successful in 8m25s
docker / deploy-docs (push) Successful in 35s
android / android (push) Successful in 12m34s
arch / build-publish (push) Successful in 15m3s
deb / build-publish (push) Successful in 16m18s
rpm / build-publish (43, bazzite, punktfunk-fedora-rpm) (push) Successful in 15m7s
apple / screenshots (push) Successful in 19m20s
rpm / build-publish (44, fedora-44, punktfunk-fedora44-rpm) (push) Successful in 12m17s
ci / rust (push) Successful in 23m12s
Developer integration guide for building a punktfunk client on any platform
by linking punktfunk-core through its stable C ABI: what the core does vs.
what the embedder supplies, build/link/cross-compile, the full client
lifecycle (identity/pairing, connect ladder, video+recovery loop, audio,
input, feedback planes, teardown), plus worked blueprints for webOS, Xbox
(GDK), and Tizen.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-07-12 10:49:13 +02:00
enricobuehler a95b518ef3 fix(windows): show app version on About screen + capitalize Punktfunk on licenses
ci / docs-site (push) Successful in 51s
ci / web (push) Successful in 57s
docker / build-push (--build-arg FEDORA_VERSION=44, ci, ci/fedora-rpm.Dockerfile, punktfunk-fedora44-rpm) (push) Successful in 9s
decky / build-publish (push) Successful in 19s
docker / build-push (ci, ci/fedora-rpm.Dockerfile, punktfunk-fedora-rpm) (push) Successful in 7s
docker / build-push (ci, ci/rust-ci.Dockerfile, punktfunk-rust-ci) (push) Successful in 7s
docker / build-push (docs-site, docs-site/Dockerfile, punktfunk-docs) (push) Successful in 7s
docker / build-push (., web/Dockerfile, punktfunk-web) (push) Successful in 38s
windows-msix / package (arm64, C:\Users\Public\ffmpeg-arm64, --no-default-features, aarch64-pc-windows-msvc, C:\t-a64) (push) Successful in 2m1s
windows-msix / package (x64, C:\Users\Public\ffmpeg, , x86_64-pc-windows-msvc, C:\t) (push) Successful in 2m3s
apple / swift (push) Successful in 5m1s
windows / build (aarch64-pc-windows-msvc) (push) Successful in 1m4s
windows / build (x86_64-pc-windows-msvc) (push) Successful in 1m25s
ci / bench (push) Successful in 8m10s
docker / deploy-docs (push) Successful in 20s
arch / build-publish (push) Successful in 11m11s
rpm / build-publish (44, fedora-44, punktfunk-fedora44-rpm) (push) Successful in 12m47s
android / android (push) Successful in 16m10s
deb / build-publish (push) Successful in 16m57s
rpm / build-publish (43, bazzite, punktfunk-fedora-rpm) (push) Successful in 17m47s
ci / rust (push) Successful in 23m26s
apple / screenshots (push) Successful in 19m57s
The About settings card had no version at all — add an identity block (app name +
"Version <CARGO_PKG_VERSION>", the workspace version) at the top, the WinUI
Settings convention and matching the Apple client's "Version X" wording. Also
capitalize the brand name on the licenses screen (was lowercase "punktfunk").

Verified against the pinned windows-reactor source + cargo fmt --check; full
Windows link left to CI (Windows-only crate).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-07-12 02:46:45 +02:00
enricobuehler f77fdee3e9 style(host): fix clippy doc_lazy_continuation in reconfig_allowed
ci / web (push) Successful in 49s
ci / docs-site (push) Successful in 55s
docker / build-push (--build-arg FEDORA_VERSION=44, ci, ci/fedora-rpm.Dockerfile, punktfunk-fedora44-rpm) (push) Successful in 9s
decky / build-publish (push) Successful in 17s
docker / build-push (., web/Dockerfile, punktfunk-web) (push) Successful in 7s
docker / build-push (ci, ci/fedora-rpm.Dockerfile, punktfunk-fedora-rpm) (push) Successful in 7s
docker / build-push (ci, ci/rust-ci.Dockerfile, punktfunk-rust-ci) (push) Successful in 8s
docker / build-push (docs-site, docs-site/Dockerfile, punktfunk-docs) (push) Successful in 7s
ci / bench (push) Successful in 5m57s
docker / deploy-docs (push) Successful in 21s
windows-host / package (push) Successful in 8m52s
android / android (push) Successful in 12m58s
arch / build-publish (push) Successful in 13m33s
deb / build-publish (push) Successful in 16m27s
rpm / build-publish (44, fedora-44, punktfunk-fedora44-rpm) (push) Successful in 16m10s
rpm / build-publish (43, bazzite, punktfunk-fedora-rpm) (push) Successful in 16m26s
apple / swift (push) Successful in 5m21s
ci / rust (push) Successful in 19m47s
apple / screenshots (push) Successful in 19m47s
New clippy (rust-1.96) flags the summary line after the 'Gated OFF for' bullet
list as a lazy list-continuation under -D warnings. Add a blank /// line so it
reads as its own paragraph (clippy's suggested fix), matching intent. Comment
only; no behavior change.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-07-12 02:05:25 +02:00
enricobuehler a85be8e467 feat(displays): clearer virtual-display preset names + descriptions
ci / web (push) Successful in 1m0s
ci / docs-site (push) Successful in 1m15s
decky / build-publish (push) Successful in 18s
apple / swift (push) Successful in 4m55s
docker / build-push (., web/Dockerfile, punktfunk-web) (push) Successful in 31s
docker / build-push (ci, ci/fedora-rpm.Dockerfile, punktfunk-fedora-rpm) (push) Successful in 7s
docker / build-push (ci, ci/rust-ci.Dockerfile, punktfunk-rust-ci) (push) Successful in 7s
docker / build-push (docs-site, docs-site/Dockerfile, punktfunk-docs) (push) Successful in 1m2s
ci / rust (push) Failing after 7m8s
docker / build-push (--build-arg FEDORA_VERSION=44, ci, ci/fedora-rpm.Dockerfile, punktfunk-fedora44-rpm) (push) Successful in 6m21s
docker / deploy-docs (push) Successful in 24s
windows-host / package (push) Failing after 8m23s
ci / bench (push) Successful in 8m44s
arch / build-publish (push) Successful in 12m26s
android / android (push) Successful in 13m37s
deb / build-publish (push) Successful in 14m23s
rpm / build-publish (44, fedora-44, punktfunk-fedora44-rpm) (push) Successful in 12m34s
rpm / build-publish (43, bazzite, punktfunk-fedora-rpm) (push) Successful in 13m17s
release / apple (push) Successful in 22m53s
apple / screenshots (push) Successful in 20m47s
"Gaming rig" actually meant a dedicated/headless box you only ever stream from,
which confused users — rename it to "Headless box" and rewrite all five preset
summaries to be scenario-first and shorter (the console cards already show the
mechanics as badges). Updated across the host API summaries (mgmt.rs), the web
console labels (en/de), and the docs table + prose. The internal preset id
`gaming-rig` is unchanged (stable API / stored-policy / test contract).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-07-12 01:29:18 +02:00
enricobuehler 1dfcb0b2f6 feat(android): default-UI connect/wake modal (Material dialog)
Mirror the Apple client: the connect/wake overlay was showing the full-screen
aurora takeover in the default touch UI too. Make ConnectOverlay mode-aware —
gamepad/console keeps the aurora ConnectTakeover, the default UI now renders a
Material 3 AlertDialog over the host grid (inert scrim; Back/buttons cancel),
matching the app's other touch dialogs. Extract a shared connectCopy() so both
presentations read identically; ConnectTakeover is now console-only.

Screenshot scenes updated (touch phases -> modal over the host grid via
shootScreen; console stays a root capture); record-mode tests green.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-07-12 01:29:18 +02:00
enricobuehler e87fd42cee feat(apple): default-UI connect/wake modal, auto-wake toggle, pixel-exact windowed streaming
Three client-UX changes that share the connect/present path (and settings files):

- Connect/wake overlay is now mode-aware: the console/gamepad UI keeps the
  full-screen aurora takeover, while the default (touch/desktop) UI shows a
  Liquid Glass modal over the host grid — the takeover looked out of place there.
- Add an auto-wake toggle (DefaultsKey.autoWake, default on) across macOS/iOS/tvOS
  Settings + the gamepad settings view; gate startSession/prepareWake and the
  gamepad "Wake & Connect" label on it. MAC-address learning stays always-on.
- Windowed sessions now stream at the window's native pixels (Match-window
  default-on) so the picture is 1:1 pixel-exact instead of the presenter
  resampling a fixed-mode frame; fullscreen reports full-display px, also 1:1.
  Also lands the mid-resize aspect-fit tracking (decoded contentSize) that keeps
  the picture undistorted after a resize.

swift build + swift test (121 tests) green; screenshot scenes verified.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-07-12 01:29:12 +02:00
enricobuehler dd02e1f402 feat(clients): unified full-screen connect/wake takeover + iOS/tvOS Wake-on-LAN
ci / web (push) Successful in 54s
ci / docs-site (push) Successful in 1m8s
decky / build-publish (push) Successful in 20s
docker / build-push (., web/Dockerfile, punktfunk-web) (push) Successful in 7s
docker / build-push (ci, ci/fedora-rpm.Dockerfile, punktfunk-fedora-rpm) (push) Successful in 11s
android / android (push) Successful in 12m23s
docker / build-push (docs-site, docs-site/Dockerfile, punktfunk-docs) (push) Successful in 8s
ci / bench (push) Successful in 6m3s
arch / build-publish (push) Successful in 12m57s
docker / build-push (--build-arg FEDORA_VERSION=44, ci, ci/fedora-rpm.Dockerfile, punktfunk-fedora44-rpm) (push) Successful in 9m40s
docker / build-push (ci, ci/rust-ci.Dockerfile, punktfunk-rust-ci) (push) Successful in 7m4s
docker / deploy-docs (push) Successful in 11s
flatpak / build-publish (push) Successful in 5m57s
deb / build-publish (push) Successful in 13m26s
windows-msix / package (arm64, C:\Users\Public\ffmpeg-arm64, --no-default-features, aarch64-pc-windows-msvc, C:\t-a64) (push) Successful in 3m50s
apple / swift (push) Successful in 4m31s
rpm / build-publish (43, bazzite, punktfunk-fedora-rpm) (push) Successful in 14m8s
rpm / build-publish (44, fedora-44, punktfunk-fedora44-rpm) (push) Successful in 13m44s
windows-msix / package (x64, C:\Users\Public\ffmpeg, , x86_64-pc-windows-msvc, C:\t) (push) Successful in 4m7s
windows / build (aarch64-pc-windows-msvc) (push) Successful in 5m9s
windows / build (x86_64-pc-windows-msvc) (push) Successful in 5m45s
release / apple (push) Successful in 21m27s
ci / rust (push) Failing after 6m47s
apple / screenshots (push) Successful in 20m18s
Give instant feedback the moment a host is picked, and make the wake wait a
full-screen takeover instead of a modal card — unified into one ConnectOverlay
across every client:

- android: new ConnectOverlay (aurora backdrop; Connecting / Waking / timed-out
  phases) replaces the tiny inline "Connecting…" row and the WakeOverlay card.
  The dial phase is now cancelable and hands off to the wake wait in one frame.
- console (pf-console-ui): the connect/wake overlays become a full-screen aurora
  takeover (draw_takeover) instead of a centered card over a dim scrim; the
  Waking → Connecting handoff no longer blinks.
- apple: new ConnectOverlay mirrors it (macOS / iOS / tvOS), replacing the
  per-tile connecting spinner + the WakeOverlay card; instant "Connecting…" from
  model.phase, and the carousel is gated inactive during the dial.

Also enable Wake-on-LAN on iOS/tvOS now that the multicast entitlement is
approved: enable com.apple.developer.networking.multicast and flip
wakeOnLANAvailable to true on every platform (MACs were already learned from
mDNS, so wake works immediately).

Verified: Android compileDebugKotlin + screenshot renders; console clippy +
36 tests + rendered phases on Linux; Apple swift build + 121 tests + rendered
phases.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-07-11 23:29:35 +02:00
enricobuehler 2271f67202 style: rustfmt the recovery + resize changes (Windows CI fmt check)
ci / web (push) Successful in 59s
ci / docs-site (push) Successful in 1m1s
docker / build-push (--build-arg FEDORA_VERSION=44, ci, ci/fedora-rpm.Dockerfile, punktfunk-fedora44-rpm) (push) Successful in 7s
decky / build-publish (push) Successful in 18s
docker / build-push (., web/Dockerfile, punktfunk-web) (push) Successful in 8s
docker / build-push (ci, ci/fedora-rpm.Dockerfile, punktfunk-fedora-rpm) (push) Successful in 8s
docker / build-push (ci, ci/rust-ci.Dockerfile, punktfunk-rust-ci) (push) Successful in 7s
docker / build-push (docs-site, docs-site/Dockerfile, punktfunk-docs) (push) Successful in 59s
apple / swift (push) Successful in 4m26s
ci / bench (push) Successful in 5m53s
ci / rust (push) Failing after 7m35s
windows-host / package (push) Failing after 8m56s
flatpak / build-publish (push) Failing after 8m12s
arch / build-publish (push) Successful in 11m41s
windows-msix / package (arm64, C:\Users\Public\ffmpeg-arm64, --no-default-features, aarch64-pc-windows-msvc, C:\t-a64) (push) Successful in 3m54s
android / android (push) Successful in 13m28s
deb / build-publish (push) Successful in 13m42s
rpm / build-publish (43, bazzite, punktfunk-fedora-rpm) (push) Successful in 13m16s
windows-msix / package (x64, C:\Users\Public\ffmpeg, , x86_64-pc-windows-msvc, C:\t) (push) Successful in 3m42s
rpm / build-publish (44, fedora-44, punktfunk-fedora44-rpm) (push) Successful in 15m33s
windows / build (aarch64-pc-windows-msvc) (push) Successful in 5m0s
docker / deploy-docs (push) Successful in 20s
windows / build (x86_64-pc-windows-msvc) (push) Successful in 5m48s
release / apple (push) Successful in 25m4s
apple / screenshots (push) Successful in 19m32s
The `cargo fmt --check` step on the x86_64-pc-windows-msvc job was
failing: the mid-stream loss-recovery and resize-overlay commits landed
with unformatted wraps across pf-presenter, pf-client-core, punktfunk-core,
pf-console-ui, and a few host files.

Applied `cargo fmt`, and hand-relocated two trailing comments in
session.rs (a decoded-frame note and the wrap-counter note) to their own
lines so rustfmt no longer column-aligns the following comment block to
a deep indent.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-07-11 23:25:06 +02:00
enricobuehler 89aa6767f9 feat(resize): scrim+spinner resize overlay in the shared presenter (Windows + GTK4)
ci / docs-site (push) Failing after 29s
ci / web (push) Successful in 43s
docker / build-push (--build-arg FEDORA_VERSION=44, ci, ci/fedora-rpm.Dockerfile, punktfunk-fedora44-rpm) (push) Successful in 9s
docker / build-push (., web/Dockerfile, punktfunk-web) (push) Successful in 8s
docker / build-push (ci, ci/fedora-rpm.Dockerfile, punktfunk-fedora-rpm) (push) Successful in 9s
decky / build-publish (push) Successful in 33s
docker / build-push (ci, ci/rust-ci.Dockerfile, punktfunk-rust-ci) (push) Successful in 9s
docker / build-push (docs-site, docs-site/Dockerfile, punktfunk-docs) (push) Successful in 7s
docker / deploy-docs (push) Successful in 20s
windows-msix / package (arm64, C:\Users\Public\ffmpeg-arm64, --no-default-features, aarch64-pc-windows-msvc, C:\t-a64) (push) Successful in 3m52s
apple / swift (push) Successful in 4m55s
ci / bench (push) Successful in 6m37s
windows-msix / package (x64, C:\Users\Public\ffmpeg, , x86_64-pc-windows-msvc, C:\t) (push) Successful in 4m2s
arch / build-publish (push) Successful in 12m42s
android / android (push) Successful in 12m51s
windows / build (aarch64-pc-windows-msvc) (push) Successful in 5m7s
deb / build-publish (push) Successful in 13m4s
rpm / build-publish (44, fedora-44, punktfunk-fedora44-rpm) (push) Successful in 13m24s
rpm / build-publish (43, bazzite, punktfunk-fedora-rpm) (push) Successful in 13m56s
apple / screenshots (push) Successful in 19m31s
ci / rust (push) Failing after 39s
windows / build (x86_64-pc-windows-msvc) (push) Failing after 1m4s
flatpak / build-publish (push) Failing after 8m4s
The mid-stream Match-window trigger + Resolution tri-state already shipped on BOTH
desktop clients via the session-always punktfunk-session binary (pf-presenter D1/D2,
C1). This ports the last Apple-parity piece — the resize-in-progress indicator
(clients/apple ResizeIndicator/ResizeIndicatorView) — into the SHARED pf-presenter
overlay, so one implementation covers both the Windows and GTK4 session windows.

- ResizeIndicator (run.rs): the Apple state machine in Rust — `steering` (a switch was
  requested) shows it, `decoded` (a frame reached the target size) clears it, `tick`
  times it out after 2.5 s for a switch the host rejected/capped. The live drag stays
  sharp; only the host's 0.3-2 s virtual-display + encoder rebuild gap is covered. A
  present-while-resizing path keeps the spinner animating through that frame-less gap.
- DecodedImage::dimensions() (pf-client-core): the END signal — a decoded frame at the
  target size means the sharp new-mode picture is on glass (the accept ack alone lands
  ahead of the host's rebuild). Mirrors is_keyframe()'s cfg arms.
- FrameCtx.resizing (pf-presenter/overlay.rs) + Skia draw (pf-console-ui): a full-screen
  55% scrim + the shared rotating theme::spinner + "Resizing…" label. The overlay
  composites its own RGBA quad and can't sample the video to blur it as SwiftUI does, so
  a scrim stands in for the blur — same intent, one draw. resizing_since clocks the
  spinner; Drawn.resize_step defeats the damage gate so it redraws each frame.

Verified on Linux: pf-presenter/pf-console-ui/session/linux-client build + clippy
-D warnings clean; 8 pf-presenter tests green incl. 2 new ResizeIndicator tests.
Windows session-binary compile (cfg-symmetric) + live on-glass both pending.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-07-11 20:40:03 +02:00
enricobuehler 7cea893db5 feat(recovery): wire LTR-RFI loss recovery into every client
ci / web (push) Successful in 48s
ci / rust (push) Failing after 48s
docker / build-push (--build-arg FEDORA_VERSION=44, ci, ci/fedora-rpm.Dockerfile, punktfunk-fedora44-rpm) (push) Successful in 7s
ci / docs-site (push) Successful in 1m0s
docker / build-push (., web/Dockerfile, punktfunk-web) (push) Successful in 7s
decky / build-publish (push) Successful in 17s
docker / build-push (docs-site, docs-site/Dockerfile, punktfunk-docs) (push) Successful in 9s
docker / build-push (ci, ci/fedora-rpm.Dockerfile, punktfunk-fedora-rpm) (push) Successful in 11s
docker / build-push (ci, ci/rust-ci.Dockerfile, punktfunk-rust-ci) (push) Successful in 11s
apple / swift (push) Successful in 4m51s
ci / bench (push) Successful in 6m13s
docker / deploy-docs (push) Successful in 20s
windows-host / package (push) Failing after 8m22s
flatpak / build-publish (push) Failing after 8m4s
arch / build-publish (push) Successful in 11m53s
windows-msix / package (arm64, C:\Users\Public\ffmpeg-arm64, --no-default-features, aarch64-pc-windows-msvc, C:\t-a64) (push) Successful in 3m45s
android / android (push) Successful in 12m53s
deb / build-publish (push) Successful in 13m4s
rpm / build-publish (44, fedora-44, punktfunk-fedora44-rpm) (push) Successful in 13m16s
rpm / build-publish (43, bazzite, punktfunk-fedora-rpm) (push) Successful in 13m29s
windows-msix / package (x64, C:\Users\Public\ffmpeg, , x86_64-pc-windows-msvc, C:\t) (push) Successful in 3m42s
windows / build (aarch64-pc-windows-msvc) (push) Successful in 5m2s
windows / build (x86_64-pc-windows-msvc) (push) Failing after 5m29s
release / apple (push) Successful in 25m48s
apple / screenshots (push) Successful in 19m31s
Centralize the client-side loss-range detector in punktfunk-core so every
embedder shares one implementation instead of re-deriving the wrapping
frame-index arithmetic:

- NativeClient::note_frame_index(frame_index) folds each received AU (in
  receive order) through RfiRecovery::observe, firing a throttled RFI request
  for the exact lost span [first_missing, frame_index-1] on a forward gap. A
  host that can RFI (AMD LTR / NVENC) re-references a known-good frame instead
  of paying a 20-40x IDR spike; the frames_dropped-driven keyframe path stays
  the backstop for when the recovery frame itself is lost.
- Export request_rfi + note_frame_index over the C ABI (Apple client).
- Call it from the Android (hw+sw pumps), Apple (StreamPump + Stage2Pipeline
  via PunktfunkConnection.noteFrameIndex), and Windows in-process pumps.
  Linux/Deck inherit it through pf-client-core's session pump.
- Split the decision into a pure RfiRecovery::observe(frame_index, now) and add
  8 unit tests: arming, contiguous runs, exact lost-range, single-frame drop,
  the 100ms throttle (burst-suppress then re-open), reorder stragglers, and
  u32 wraparound (contiguous + gap-range).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-07-11 19:11:01 +02:00
enricobuehler e55ff1bb28 feat(recovery): clean mid-stream loss recovery — freeze-until-reanchor + AMD LTR-RFI
ci / rust (push) Failing after 53s
ci / docs-site (push) Successful in 54s
ci / web (push) Successful in 58s
docker / build-push (--build-arg FEDORA_VERSION=44, ci, ci/fedora-rpm.Dockerfile, punktfunk-fedora44-rpm) (push) Successful in 8s
docker / build-push (., web/Dockerfile, punktfunk-web) (push) Successful in 9s
decky / build-publish (push) Successful in 18s
docker / build-push (ci, ci/fedora-rpm.Dockerfile, punktfunk-fedora-rpm) (push) Successful in 8s
docker / build-push (docs-site, docs-site/Dockerfile, punktfunk-docs) (push) Successful in 7s
apple / swift (push) Successful in 4m57s
ci / bench (push) Successful in 5m43s
docker / build-push (ci, ci/rust-ci.Dockerfile, punktfunk-rust-ci) (push) Successful in 6m4s
docker / deploy-docs (push) Successful in 21s
windows-host / package (push) Failing after 8m42s
flatpak / build-publish (push) Failing after 8m6s
android / android (push) Successful in 11m59s
windows-msix / package (arm64, C:\Users\Public\ffmpeg-arm64, --no-default-features, aarch64-pc-windows-msvc, C:\t-a64) (push) Successful in 3m59s
arch / build-publish (push) Successful in 13m20s
deb / build-publish (push) Successful in 14m36s
rpm / build-publish (43, bazzite, punktfunk-fedora-rpm) (push) Successful in 14m45s
windows-msix / package (x64, C:\Users\Public\ffmpeg, , x86_64-pc-windows-msvc, C:\t) (push) Successful in 3m44s
rpm / build-publish (44, fedora-44, punktfunk-fedora44-rpm) (push) Successful in 13m21s
windows / build (aarch64-pc-windows-msvc) (push) Successful in 5m1s
windows / build (x86_64-pc-windows-msvc) (push) Failing after 5m28s
release / apple (push) Successful in 25m54s
apple / screenshots (push) Successful in 19m35s
Removes the "gray frames with motion" artifact on Vulkan-Video clients and lets
AMD/NVENC hosts re-anchor after loss WITHOUT a 20-40x IDR spike.

Client (pf-client-core): after a reference loss the hardware decoder conceals the
missing-reference deltas (on RADV, a gray plate with new motion painted over) and
returns Ok. The pump now freezes on the last good picture until a clean re-anchor
instead of showing the concealment — lifting on a real IDR, an intra-refresh
recovery mark (2nd wave boundary), or an LTR-RFI recovery anchor (1st). The
frame_index gap is the early, precise loss signal and drives an RFI request.

Host recovery signals (inert unless the backend supports them):
- USER_FLAG_RECOVERY_POINT — intra-refresh wave boundary (NVENC constrained GDR).
- USER_FLAG_RECOVERY_ANCHOR — AMD LTR reference-frame-invalidation recovery frame.

AMD LTR-RFI (encode/windows/amf.rs) — the AMD twin of NVENC RFI. AMF's AVC/HEVC API
has no constrained-intra property (intra-refresh cannot heal; PSNR-proven), so the
only clean-recovery lever is user LTR: mark frames as long-term references, and on
loss force the next frame to re-reference the newest known-good one — a clean
P-frame, not an IDR. Two rotating LTR slots, ~0.5s mark cadence, on by default for
AVC/HEVC (PUNKTFUNK_NO_AMF_LTR disables). invalidate_ref_frames picks the newest LTR
before the loss; a range older than the live slots falls back to a keyframe.

Protocol (punktfunk-core): RfiRequest control message + NativeClient::request_rfi().
Host: RfiRequest dispatch -> invalidate_ref_frames (IDR fallback); an RFI success
anchors the keyframe cooldown so the client's frames_dropped echo of the same loss
is coalesced away rather than emitting a redundant IDR.

Spike: synthetic NV12 GPU source for headless AMF encoder testing.

Validated: core rfi_request_roundtrip; pf-client-core 31 unit tests
(incl. an_rfi_anchor_lifts_immediately); punktfunk-host builds + 271 tests on Linux;
punktfunk-host builds clean on Windows; real AMD iGPU spike (invalidate at frame 90
forced re-reference to LTR frame 60 — 180 frames, keyframes=1, no recovery IDR).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-07-11 17:31:17 +02:00
enricobuehler 890c7531d8 Merge branch 'midstream-resize': mid-stream resolution resize
apple / swift (push) Successful in 4m19s
ci / rust (push) Failing after 46s
ci / web (push) Successful in 51s
ci / docs-site (push) Successful in 1m2s
decky / build-publish (push) Successful in 20s
docker / build-push (--build-arg FEDORA_VERSION=44, ci, ci/fedora-rpm.Dockerfile, punktfunk-fedora44-rpm) (push) Successful in 9s
docker / build-push (., web/Dockerfile, punktfunk-web) (push) Successful in 9s
docker / build-push (ci, ci/fedora-rpm.Dockerfile, punktfunk-fedora-rpm) (push) Successful in 8s
docker / build-push (ci, ci/rust-ci.Dockerfile, punktfunk-rust-ci) (push) Successful in 8s
docker / build-push (docs-site, docs-site/Dockerfile, punktfunk-docs) (push) Successful in 8s
ci / bench (push) Successful in 5m17s
arch / build-publish (push) Successful in 10m44s
docker / deploy-docs (push) Successful in 11s
android / android (push) Successful in 16m43s
windows-host / package (push) Failing after 8m29s
flatpak / build-publish (push) Failing after 8m13s
deb / build-publish (push) Successful in 11m24s
windows-msix / package (arm64, C:\Users\Public\ffmpeg-arm64, --no-default-features, aarch64-pc-windows-msvc, C:\t-a64) (push) Successful in 3m51s
windows-msix / package (x64, C:\Users\Public\ffmpeg, , x86_64-pc-windows-msvc, C:\t) (push) Successful in 4m1s
rpm / build-publish (43, bazzite, punktfunk-fedora-rpm) (push) Successful in 12m24s
release / apple (push) Successful in 24m35s
rpm / build-publish (44, fedora-44, punktfunk-fedora44-rpm) (push) Successful in 12m24s
windows / build (aarch64-pc-windows-msvc) (push) Successful in 5m2s
windows / build (x86_64-pc-windows-msvc) (push) Failing after 5m25s
apple / screenshots (push) Successful in 19m55s
Lands the mid-stream resolution resize feature (client-driven Reconfigure so
the host's virtual display + encoder follow a resized client window without a
reconnect), all paths default OFF:

- host hardening H1-H5 + session-binary Match window (C1)
- Apple macOS/iPadOS Match-window trigger + settings (C3) and the resize
  overlay (blur + spinner) client UX
- Windows on-glass fixes: corrective-ack actual resolution + pf-vdisplay
  monitor re-arrival for out-of-list mid-stream modes
- Linux backend matrix + the live-reconfigure gate unit tests

Validated on-glass: Windows IDD-push (.173), Linux Mutter + KWin. Android
(C4) deferred; Apple full build pending on a Mac.
2026-07-11 15:59:07 +02:00
enricobuehler e6fbcecdb9 fix(clients/windows): GUI text inputs read the live value, not a stale render snapshot
ci / web (push) Successful in 50s
ci / docs-site (push) Successful in 1m5s
decky / build-publish (push) Successful in 23s
docker / build-push (--build-arg FEDORA_VERSION=44, ci, ci/fedora-rpm.Dockerfile, punktfunk-fedora44-rpm) (push) Successful in 16s
docker / build-push (., web/Dockerfile, punktfunk-web) (push) Successful in 14s
docker / build-push (ci, ci/fedora-rpm.Dockerfile, punktfunk-fedora-rpm) (push) Successful in 14s
docker / build-push (ci, ci/rust-ci.Dockerfile, punktfunk-rust-ci) (push) Successful in 9s
docker / build-push (docs-site, docs-site/Dockerfile, punktfunk-docs) (push) Successful in 1m0s
windows-msix / package (arm64, C:\Users\Public\ffmpeg-arm64, --no-default-features, aarch64-pc-windows-msvc, C:\t-a64) (push) Successful in 2m33s
apple / swift (push) Successful in 4m24s
windows-msix / package (x64, C:\Users\Public\ffmpeg, , x86_64-pc-windows-msvc, C:\t) (push) Successful in 2m18s
ci / bench (push) Successful in 7m10s
windows / build (aarch64-pc-windows-msvc) (push) Successful in 5m17s
arch / build-publish (push) Successful in 11m38s
deb / build-publish (push) Successful in 12m6s
rpm / build-publish (44, fedora-44, punktfunk-fedora44-rpm) (push) Successful in 13m9s
windows / build (x86_64-pc-windows-msvc) (push) Successful in 5m46s
android / android (push) Successful in 17m7s
rpm / build-publish (43, bazzite, punktfunk-fedora-rpm) (push) Successful in 19m48s
docker / deploy-docs (push) Successful in 22s
ci / rust (push) Successful in 26m33s
apple / screenshots (push) Successful in 19m29s
A component() page re-renders reliably only when its props change: root wraps
every screen in a stable animated border, so once the entrance tween settles the
reconciler skips that unchanged-props subtree and a page's own use_state writes
never force a re-render. Three text fields read their value at click time from
that stranded local state:

- PIN pairing sent an empty PIN, so pairing always failed with "wrong PIN, or not
  armed?" — the reported bug. The CLI --pair path bypasses the reactor and worked.
- "Add host" Connect captured the empty mount-time address and silently did
  nothing (you open the modal precisely when the host isn't being discovered, so
  no discovery tick re-renders the page while you type).
- Rename round-tripped the draft through an always-deferred AsyncSetState into a
  controlled text box, fighting the caret on fast typing and dropping the last
  character when Save was clicked before the write landed.

Fix: hold each field's live value in a use_ref cell written on every keystroke
and read at commit time (uncontrolled input), instead of a render-time snapshot.
Rename is seeded when its target changes and no longer re-renders the whole page
per keystroke. Reviewed the rest of the app (settings, speed test, library,
stream, connect/request-access/waking, forget) — all driven by root-state props
and wired correctly.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-07-11 15:58:25 +02:00
enricobuehler f01c5e210c feat(resize/apple): resize overlay — blur + spinner during mid-stream resize
Make a Match-window resize deliberate instead of a stutter: blur the live
stream and show a spinner while the host rebuilds its virtual display +
encoder and VideoToolbox re-inits on the new-mode IDR. No new protocol —
driven entirely by existing client signals.

- ResizeIndicator (pure core, unit-tested): START = follower steering,
  END = a decoded frame at the target size, TIMEOUT = 2.5s safety net for a
  rejected/capped switch that never yields a new-size frame; re-arms only on
  a CHANGED target, not a repeated same-size drag.
- MatchWindowFollower.onResizeTarget fires the instant the window differs
  from the live mode (deduped via lastSteered); a new onDecodedSize callback
  threads each new-mode IDR's coded dims through StreamPump/Stage2Pipeline →
  SessionPresenter → both stream views.
- SessionModel gains @Published resizing (+ resizeTargeted/resizeDecoded, a
  tick on the 1 Hz stats timer, reset on disconnect); ContentView blurs the
  stream 16px and overlays ResizeIndicatorView while resizing (the 32px
  trust-prompt blur is unchanged and takes precedence).

tvOS declares the props but never fires the follower (it drives modes via
AVDisplayManager), so the overlay stays dormant there. Pure core verified on
the Linux toolchain; full AppKit/UIKit build pending on a Mac.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-07-11 15:58:14 +02:00
enricobuehler d294b3923c test(resize): extract + unit-test the live-reconfigure gate (H1 gamescope / H5 per-client-mode)
The Linux §6 on-glass matrix can validate the gamescope must-REJECT behavior only on
native-gamescope hardware (the NVIDIA dev box fails headless GBM allocation — a nested
Hyprland/sway/gamescope output comes up 0×0), so pin the gate down deterministically
instead: extract the inline `live_reconfig_ok` decision into a pure
`reconfig_allowed(compositor, per_client_mode)` and test it — gamescope rejects in every
identity mode, a per-client-mode policy rejects on every backend, and all other
compositors (plus the synthetic protocol-test source) with the default identity accept.

Also fmt-normalizes the re_add block from the prior commit (whitespace only).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-07-11 14:56:59 +02:00
enricobuehler 4c18bb80ca feat(resize/win): mid-stream resize on-glass fixes — corrective-ack actual res + monitor re-arrival
On-glass validation on the .173 Windows IDD-push host confirmed the Reconfigure
protocol + host rebuild work end-to-end and genuinely change pixels for an
advertised mode (1920x1080 -> 1280x720: two SPS/IDR sets, ffprobe both res). It
also surfaced two gaps for out-of-EDID-list target modes, both fixed here.

Fix 2 (corrective ack carries the ACTUAL resolution): the H2/H3 corrective ack
recovered only the achieved REFRESH (interval_hz), taking width/height straight
from the request — so when a backend delivered a different RESOLUTION (Windows
pf-vdisplay falling back to its advertised mode) the client was told it got a
size it never received, and by the D2 discipline never re-asked. New
`delivered_mode(frame.{w,h}, interval)` derives the ack from the captured frame's
real dims (what the encoder opened at / the client decodes) in both the success
and rollback branches. Unit-tested.

Fix 1 (reach arbitrary mid-stream modes via monitor RE-ARRIVAL): the pf-vdisplay
driver freezes a monitor's advertised mode list at IOCTL_ADD, and IddCx exposes
no live update-modes DDI, so an in-place ChangeDisplaySettingsExW to a mode not
advertised at arrival returns DISP_CHANGE_BADMODE. The manager's mid-stream
reconfigure now REMOVEs + re-ADDs the driver monitor at the exact new mode,
reusing the slot's stable per-client id (EDID serial / ContainerId) so the OS
keeps identity + saved DPI. The rebuilt Monitor PRESERVES gen (lease/refcount
continuity) and the group restore snapshot; reisolate_after_swap re-isolates the
new target without recapturing it. Host-only — no driver change. One monitor
hotplug per switch (the design's accepted "re-arrival for everything").

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-07-11 14:34:22 +02:00
enricobuehler 89ff326ebf feat(resize/apple): Match-window mid-stream resize trigger + settings (C3)
design/midstream-resolution-resize.md Phase 2, Apple client. The stream mode
follows the session window/scene: a windowed macOS window resize or an iPad
Stage Manager / Split View scene change renegotiates the host's virtual
display + encoder via the existing PunktfunkConnection.requestMode, so a
windowed session streams native-resolution pixels instead of scaling.
Decode/present need nothing — VideoToolbox recreates its session on the
keyframe-derived format-description change (§1 table).

- MatchWindowFollower (new): the shared D2 trigger discipline — physical
  pixels even-floored + clamped ≥320×200, debounce to resize-end, ≥1 s
  between requests, skip a size equal to the live mode, request each distinct
  size at most once (stops re-asking a rejected size / looping on a host
  rollback). Pure normalize/request core is unit-tested (MatchWindowTests).
- macOS StreamLayerView: fed from layoutPresenter() (bounds → convertToBacking),
  guarded to once-in-a-window.
- iOS StreamViewController: fed from viewDidLayoutSubviews (bounds × render
  scale); iOS-only (iPhone fullscreen no-ops, tvOS uses AVDisplayManager).
- Settings: "Match window" toggle in the Stream mode section (iOS + macOS),
  DefaultsKey.matchWindow, read per session by the follower.

Verified on a Linux Swift 6.1.2 toolchain (the app target needs AppKit/UIKit,
unavailable here): the real MatchWindowFollower.swift type-checks in Swift 5
mode against a connection stub, and the pure discipline + the follower's
decision path pass a standalone harness (drag-settle + grow → exactly two
switches, refresh preserved, no re-request loop). A full build + on-device run
(macOS window, iPad Stage Manager) remains for a Mac.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-07-11 13:36:03 +02:00
enricobuehler d0d9bd5bfb feat(resize): mid-stream resolution resize — host hardening (H1–H5) + session-binary Match window (C1)
design/midstream-resolution-resize.md Phase 0 + Phase 1.

Host (Phase 0):
- H1/H5: per-backend Reconfigure acceptance gate — reject for gamescope
  (all sub-modes; a resize must never relaunch the title) and under the
  per-client-mode identity policy (a resize would resolve a different
  display slot). Synthetic stays reconfigurable on purpose (the protocol
  test source; the C-ABI roundtrip test rides it). Plus a 500 ms host-side
  min-interval backstop against Reconfigure spam.
- H2: rollback/corrective ack — the data plane reports the mode actually
  live after a failed rebuild (or a refresh the backend capped) through a
  reconfig_result channel; the control task forwards it as a second
  accepted Reconfigured so the client's mode slot self-corrects.
- H3: live stats mode — SendStats reads a packed AtomicU64 (w|h|hz)
  updated on every switch instead of latching the session-start mode.
- H4: registry::retire(gen) — a mode-switch rebuild force-releases the
  superseded Linux display, so linger/forever keep-alive policies don't
  accumulate kept monitors at stale modes. VirtualOutput carries pool_gen
  (fresh AND reused) and the Pipeline tuple threads it to the switch arm.

Client (Phase 1, default off):
- Settings: match_window policy + persisted last window size; exposed as
  the Resolution tri-state (Native / Match window / explicit) in the Skia
  console, GTK and WinUI settings pages.
- pf-presenter: window opens at the persisted size; Hello mode follows the
  window's pixel size; D2 trigger discipline (400 ms debounce to
  resize-end, ≥1 s spacing, even-floor + ≥320×200 clamp, each distinct
  size requested at most once — covers rejects and host rollbacks) as a
  pure, unit-tested decision; HUD line + title refresh on a switch.
- Session binary wires both --connect and --browse paths; the WinUI shell
  is session-always, so this covers Windows too.

Verified: workspace tests + clippy green; synthetic --remode end-to-end;
live session-binary run (window at persisted 1000×600 → Hello 1000×600@60).
On-glass per-backend matrix (Mutter/KWin/gamescope-reject, keep-alive
accumulation) still pending before any default flip.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-11 12:53:47 +02:00
151 changed files with 14280 additions and 6432 deletions
+8 -1
View File
@@ -90,7 +90,14 @@ jobs:
git config --global --add safe.directory "$PWD" git config --global --add safe.directory "$PWD"
# punktfunk-client-session is the Vulkan/Skia streamer the shell execs for a connect — # punktfunk-client-session is the Vulkan/Skia streamer the shell execs for a connect —
# both client binaries must ship (build-client-deb.sh installs both). # both client binaries must ship (build-client-deb.sh installs both).
cargo build --release --locked \ # --features punktfunk-host/nvenc: the direct-SDK NVENC path (real RFI + recovery anchor on
# Linux NVIDIA; design/linux-direct-nvenc.md). AMD/Intel-safe — NVENC/CUDA is dlopen'd at
# runtime (no link-time dep; identical DT_NEEDED to a plain build), and the encoder is only
# constructed for a CUDA capture frame + PUNKTFUNK_NVENC_DIRECT, never on VAAPI hosts.
# --features punktfunk-host/vulkan-encode: the AMD/Intel twin — raw VK_KHR_video_encode_h265
# with real RFI (design/linux-vulkan-video-encode.md). Pure Rust ash (no new lib / link dep);
# default on for HEVC (PUNKTFUNK_VULKAN_ENCODE=0 → libav VAAPI), failed open falls back to VAAPI.
cargo build --release --locked --features punktfunk-host/nvenc,punktfunk-host/vulkan-encode \
-p punktfunk-host -p punktfunk-client-linux -p punktfunk-client-session -p punktfunk-host -p punktfunk-client-linux -p punktfunk-client-session
- name: Build + smoke-boot web console (bun preset) - name: Build + smoke-boot web console (bun preset)
Generated
+14 -27
View File
@@ -870,15 +870,6 @@ dependencies = [
"itertools 0.10.5", "itertools 0.10.5",
] ]
[[package]]
name = "crossbeam-channel"
version = "0.5.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2"
dependencies = [
"crossbeam-utils",
]
[[package]] [[package]]
name = "crossbeam-deque" name = "crossbeam-deque"
version = "0.8.6" version = "0.8.6"
@@ -2154,7 +2145,7 @@ dependencies = [
[[package]] [[package]]
name = "latency-probe" name = "latency-probe"
version = "0.9.2" version = "0.10.0"
[[package]] [[package]]
name = "lazy_static" name = "lazy_static"
@@ -2286,7 +2277,7 @@ checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad"
[[package]] [[package]]
name = "loss-harness" name = "loss-harness"
version = "0.9.2" version = "0.10.0"
dependencies = [ dependencies = [
"punktfunk-core", "punktfunk-core",
] ]
@@ -2765,7 +2756,7 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220"
[[package]] [[package]]
name = "pf-client-core" name = "pf-client-core"
version = "0.9.2" version = "0.10.0"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"async-channel", "async-channel",
@@ -2787,7 +2778,7 @@ dependencies = [
[[package]] [[package]]
name = "pf-console-ui" name = "pf-console-ui"
version = "0.9.2" version = "0.10.0"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"ash", "ash",
@@ -2808,7 +2799,7 @@ dependencies = [
[[package]] [[package]]
name = "pf-ffvk" name = "pf-ffvk"
version = "0.9.2" version = "0.10.0"
dependencies = [ dependencies = [
"ash", "ash",
"bindgen", "bindgen",
@@ -2817,7 +2808,7 @@ dependencies = [
[[package]] [[package]]
name = "pf-presenter" name = "pf-presenter"
version = "0.9.2" version = "0.10.0"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"ash", "ash",
@@ -3001,7 +2992,7 @@ dependencies = [
[[package]] [[package]]
name = "punktfunk-client-android" name = "punktfunk-client-android"
version = "0.9.2" version = "0.10.0"
dependencies = [ dependencies = [
"android_logger", "android_logger",
"jni", "jni",
@@ -3017,7 +3008,7 @@ dependencies = [
[[package]] [[package]]
name = "punktfunk-client-linux" name = "punktfunk-client-linux"
version = "0.9.2" version = "0.10.0"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"async-channel", "async-channel",
@@ -3033,7 +3024,7 @@ dependencies = [
[[package]] [[package]]
name = "punktfunk-client-session" name = "punktfunk-client-session"
version = "0.9.2" version = "0.10.0"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"pf-client-core", "pf-client-core",
@@ -3048,14 +3039,11 @@ dependencies = [
[[package]] [[package]]
name = "punktfunk-client-windows" name = "punktfunk-client-windows"
version = "0.9.2" version = "0.10.0"
dependencies = [ dependencies = [
"anyhow",
"async-channel", "async-channel",
"crossbeam-channel",
"ffmpeg-next", "ffmpeg-next",
"mdns-sd", "mdns-sd",
"opus",
"pf-client-core", "pf-client-core",
"punktfunk-core", "punktfunk-core",
"sdl3", "sdl3",
@@ -3063,7 +3051,6 @@ dependencies = [
"serde_json", "serde_json",
"tracing", "tracing",
"tracing-subscriber", "tracing-subscriber",
"wasapi",
"windows 0.62.2 (git+https://github.com/microsoft/windows-rs?rev=a4f7b2cb7c63c6bb7fc77a2affe57145be1d8c4f)", "windows 0.62.2 (git+https://github.com/microsoft/windows-rs?rev=a4f7b2cb7c63c6bb7fc77a2affe57145be1d8c4f)",
"windows-reactor", "windows-reactor",
"windows-reactor-setup", "windows-reactor-setup",
@@ -3072,7 +3059,7 @@ dependencies = [
[[package]] [[package]]
name = "punktfunk-core" name = "punktfunk-core"
version = "0.9.2" version = "0.10.0"
dependencies = [ dependencies = [
"aes-gcm", "aes-gcm",
"bytes", "bytes",
@@ -3103,7 +3090,7 @@ dependencies = [
[[package]] [[package]]
name = "punktfunk-host" name = "punktfunk-host"
version = "0.9.2" version = "0.10.0"
dependencies = [ dependencies = [
"aes", "aes",
"aes-gcm", "aes-gcm",
@@ -3175,7 +3162,7 @@ dependencies = [
[[package]] [[package]]
name = "punktfunk-probe" name = "punktfunk-probe"
version = "0.9.2" version = "0.10.0"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"mdns-sd", "mdns-sd",
@@ -3189,7 +3176,7 @@ dependencies = [
[[package]] [[package]]
name = "punktfunk-tray" name = "punktfunk-tray"
version = "0.9.2" version = "0.10.0"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"ksni", "ksni",
+1 -1
View File
@@ -35,7 +35,7 @@ exclude = [
ndk = { path = "clients/android/native/vendor/ndk" } ndk = { path = "clients/android/native/vendor/ndk" }
[workspace.package] [workspace.package]
version = "0.9.2" version = "0.10.0"
edition = "2021" edition = "2021"
rust-version = "1.82" rust-version = "1.82"
license = "MIT OR Apache-2.0" license = "MIT OR Apache-2.0"
@@ -0,0 +1,277 @@
package io.unom.punktfunk
import androidx.activity.compose.BackHandler
import androidx.compose.animation.core.LinearEasing
import androidx.compose.animation.core.RepeatMode
import androidx.compose.animation.core.animateFloat
import androidx.compose.animation.core.infiniteRepeatable
import androidx.compose.animation.core.rememberInfiniteTransition
import androidx.compose.animation.core.tween
import androidx.compose.foundation.Canvas
import androidx.compose.foundation.clickable
import androidx.compose.foundation.interaction.MutableInteractionSource
import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.Box
import androidx.compose.foundation.layout.Column
import androidx.compose.foundation.layout.fillMaxSize
import androidx.compose.foundation.layout.padding
import androidx.compose.foundation.layout.size
import androidx.compose.foundation.layout.widthIn
import androidx.compose.material.icons.Icons
import androidx.compose.material.icons.filled.Bedtime
import androidx.compose.material3.AlertDialog
import androidx.compose.material3.CircularProgressIndicator
import androidx.compose.material3.Icon
import androidx.compose.material3.Text
import androidx.compose.material3.TextButton
import androidx.compose.runtime.Composable
import androidx.compose.runtime.getValue
import androidx.compose.runtime.remember
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.graphics.Color
import androidx.compose.ui.graphics.drawscope.Stroke
import androidx.compose.ui.text.font.FontFamily
import androidx.compose.ui.text.font.FontWeight
import androidx.compose.ui.text.style.TextAlign
import androidx.compose.ui.unit.dp
import androidx.compose.ui.unit.sp
import androidx.compose.ui.window.DialogProperties
/**
* Which phase of the connect flow to draw — the pure view model [ConnectOverlay] resolves from the
* live dial/wake state, so [ConnectTakeover] / [ConnectModal] can render (and be screenshot-tested)
* statelessly.
*/
internal sealed interface ConnectPhase {
val hostName: String
/** The dial is in flight (shown the instant a host is picked). */
data class Connecting(override val hostName: String) : ConnectPhase
/** A sleeping host is being Wake-on-LAN'd and we're waiting for it to advertise again. */
data class Waking(override val hostName: String, val seconds: Int, val connectsAfter: Boolean) : ConnectPhase
/** The wake wait ran out — offer retry / cancel. */
data class WakeTimedOut(override val hostName: String) : ConnectPhase
}
/** Per-phase copy, shared by the console takeover and the touch modal so both read identically. */
private data class ConnectCopy(
val title: String,
val subtitle: String,
/** Monospace the subtitle so a ticking seconds counter doesn't jitter its width. */
val monoSubtitle: Boolean,
val cancelLabel: String,
)
private fun connectCopy(phase: ConnectPhase): ConnectCopy = when (phase) {
is ConnectPhase.Connecting -> ConnectCopy(
"Connecting to ${phase.hostName}", "Establishing a secure connection…", false, "Cancel",
)
is ConnectPhase.Waking -> ConnectCopy(
"Waking ${phase.hostName}",
"Waiting for it to come online · ${phase.seconds}s",
true,
// A wake-only wait (no dial after) says "Stop Waiting"; a wake that will connect says "Cancel".
if (!phase.connectsAfter) "Stop Waiting" else "Cancel",
)
is ConnectPhase.WakeTimedOut -> ConnectCopy(
"${phase.hostName} didn't wake",
"It may still be booting, or it's powered off / off this network.",
false,
"Cancel",
)
}
/**
* The unified "getting you connected" feedback — one flow for BOTH phases of reaching a host, so the
* user gets feedback the instant they pick one and it flows seamlessly into a wake if the host turns
* out to be asleep:
*
* - **Connecting** ([connectingHostName] non-null): the dial is in flight. Shown immediately on tap,
* so a host that takes a beat to answer no longer looks like nothing happened.
* - **Waking** ([WakeController.waking] non-null): the dial failed on a sleeping host, so we're firing
* Wake-on-LAN and waiting for it to advertise again, escalating to a retry/cancel prompt on timeout.
*
* Presentation is mode-aware (mirrors the Apple client): in the **console / gamepad** UI it's a
* full-screen aurora [ConnectTakeover] — the same signature backdrop the console home uses, driven by
* the pad (B cancels, A retries once timed out) with a hint bar. In the **default touch** UI it's a
* Material [ConnectModal] over the host grid, matching the app's other dialogs — the aurora takeover
* looked out of place there.
*
* The two phases hand off within a single Compose frame (see ConnectScreen's `doConnectDirect` →
* `waker.start` → redial), so nothing blinks between them.
*/
@Composable
fun ConnectOverlay(
connectingHostName: String?,
waker: WakeController,
gamepadUi: Boolean,
onCancelConnect: () -> Unit,
) {
val waking = waker.waking
// Waking takes precedence (it only exists after a dial has failed) so a stray overlap can't strand
// the "Connecting…" phase over a wake in progress.
val phase = when {
waking != null && waking.timedOut -> ConnectPhase.WakeTimedOut(waking.hostName)
waking != null -> ConnectPhase.Waking(waking.hostName, waking.seconds, waking.connectsAfter)
connectingHostName != null -> ConnectPhase.Connecting(connectingHostName)
else -> return
}
// System Back / pad B (remapped) cancels whatever's in flight — a plain dial or the wake wait.
val cancel = { if (waking != null) waker.cancel() else onCancelConnect() }
if (gamepadUi) {
BackHandler { cancel() }
// A retries once a wake has timed out; B falls through to the BackHandler above.
GamepadNavEffect2D(
active = true,
onDirection = {},
onActivate = { if (phase is ConnectPhase.WakeTimedOut) waker.retry() },
)
ConnectTakeover(phase = phase, onCancel = cancel, onRetry = { waker.retry() })
} else {
// The AlertDialog owns its own scrim + system-Back handling (routed to cancel).
ConnectModal(phase = phase, onCancel = cancel, onRetry = { waker.retry() })
}
}
/**
* The default-UI presentation: a Material dialog over the host grid, matching the app's other touch
* dialogs. A spinner (or the sleep glyph once timed out) sits above the title; the scrim is inert so a
* stray tap can't drop a connect in flight — only the buttons or system Back cancel.
*/
@Composable
internal fun ConnectModal(
phase: ConnectPhase,
onCancel: () -> Unit,
onRetry: () -> Unit,
) {
val copy = connectCopy(phase)
val timedOut = phase is ConnectPhase.WakeTimedOut
AlertDialog(
onDismissRequest = onCancel,
properties = DialogProperties(dismissOnClickOutside = false),
icon = {
if (timedOut) {
Icon(Icons.Filled.Bedtime, contentDescription = null)
} else {
CircularProgressIndicator(modifier = Modifier.size(28.dp), strokeWidth = 3.dp)
}
},
title = { Text(copy.title, textAlign = TextAlign.Center) },
text = {
Text(
copy.subtitle,
textAlign = TextAlign.Center,
fontFamily = if (copy.monoSubtitle) FontFamily.Monospace else FontFamily.Default,
)
},
// No confirm action until the wake times out; then "Try Again" is the primary button.
confirmButton = {
if (timedOut) TextButton(onClick = onRetry) { Text("Try Again") }
},
dismissButton = {
TextButton(onClick = onCancel) { Text(copy.cancelLabel) }
},
)
}
/**
* The console / gamepad presentation: an opaque aurora backdrop with a centred spinner/title/subtitle
* for [phase], plus a bottom hint bar spelling out the pad actions (B cancels, A retries once timed
* out) — glyph-driven like every other console screen. onClick keeps the hints tappable too, so a
* user without a working pad can still get out.
*/
@Composable
internal fun ConnectTakeover(
phase: ConnectPhase,
onCancel: () -> Unit,
onRetry: () -> Unit,
) {
val copy = connectCopy(phase)
val timedOut = phase is ConnectPhase.WakeTimedOut
Box(
Modifier
.fillMaxSize()
// Swallow taps so the screen behind can't be touched through the takeover.
.clickable(interactionSource = remember { MutableInteractionSource() }, indication = null) {},
contentAlignment = Alignment.Center,
) {
GamepadAuroraBackground(Modifier.fillMaxSize())
Column(
Modifier.padding(horizontal = 40.dp).widthIn(max = 460.dp),
horizontalAlignment = Alignment.CenterHorizontally,
verticalArrangement = Arrangement.spacedBy(18.dp),
) {
if (timedOut) {
Box(Modifier.size(120.dp), contentAlignment = Alignment.Center) {
Icon(
Icons.Filled.Bedtime,
contentDescription = null,
tint = Color.White.copy(alpha = 0.9f),
modifier = Modifier.size(46.dp),
)
}
} else {
PulsingSpinner()
}
Text(
copy.title,
color = Color.White,
fontWeight = FontWeight.Bold,
fontSize = 24.sp,
textAlign = TextAlign.Center,
)
Text(
copy.subtitle,
color = Color.White.copy(alpha = 0.65f),
fontSize = 14.sp,
textAlign = TextAlign.Center,
fontFamily = if (copy.monoSubtitle) FontFamily.Monospace else FontFamily.Default,
)
}
val hints = buildList {
add(PadGlyph.hint('B', copy.cancelLabel, onClick = onCancel))
if (timedOut) add(PadGlyph.hint('A', "Try Again", onClick = onRetry))
}
GamepadHintBar(hints, Modifier.align(Alignment.BottomCenter).padding(bottom = 28.dp))
}
}
/**
* The connecting/waking indicator: a white progress ring inside two brand-violet halo rings that
* expand and fade on a staggered loop — a small sign of life so the takeover reads as working, not
* stalled.
*/
@Composable
private fun PulsingSpinner() {
val transition = rememberInfiniteTransition(label = "connectPulse")
val pulse by transition.animateFloat(
initialValue = 0f,
targetValue = 1f,
animationSpec = infiniteRepeatable(tween(1600, easing = LinearEasing), RepeatMode.Restart),
label = "pulse",
)
Box(Modifier.size(120.dp), contentAlignment = Alignment.Center) {
Canvas(Modifier.fillMaxSize()) {
val maxR = size.minDimension / 2f
for (i in 0..1) {
val p = (pulse + i * 0.5f) % 1f
drawCircle(
color = Color(0xFF8678F5).copy(alpha = (1f - p) * 0.35f),
radius = maxR * (0.42f + p * 0.58f),
style = Stroke(width = 2.dp.toPx()),
)
}
}
CircularProgressIndicator(
color = Color.White,
strokeWidth = 3.dp,
modifier = Modifier.size(54.dp),
)
}
}
@@ -88,6 +88,16 @@ private class RequestAccessState(val target: PendingTrust) {
val cancelled = AtomicBoolean(false) val cancelled = AtomicBoolean(false)
} }
/**
* A plain dial in flight — [hostName] labels the unified [ConnectOverlay]'s "Connecting…" phase, and
* [cancelled] lets its Cancel abort. The native connect is a blocking call with no abort, so Cancel
* returns the UI immediately and a late-arriving handle is torn down silently rather than navigating
* into a session the user already backed out of. Mirrors [RequestAccessState]'s late-result handling.
*/
private class ConnectAttempt(val hostName: String) {
val cancelled = AtomicBoolean(false)
}
@Composable @Composable
fun ConnectScreen( fun ConnectScreen(
settings: Settings, settings: Settings,
@@ -107,6 +117,9 @@ fun ConnectScreen(
var port by remember { mutableStateOf("9777") } var port by remember { mutableStateOf("9777") }
var connecting by remember { mutableStateOf(false) } var connecting by remember { mutableStateOf(false) }
var status by remember { mutableStateOf<String?>(null) } var status by remember { mutableStateOf<String?>(null) }
// A plain dial in flight (drives the "Connecting…" phase of the full-screen ConnectOverlay); null
// when idle or when the request-access / wake flows own the screen instead.
var attempt by remember { mutableStateOf<ConnectAttempt?>(null) }
// The host streams at exactly this mode; "Native" settings resolve from the device display. // The host streams at exactly this mode; "Native" settings resolve from the device display.
val (w, h, hz) = settings.effectiveMode(context) val (w, h, hz) = settings.effectiveMode(context)
@@ -267,11 +280,20 @@ fun ConnectScreen(
status = "Identity not ready yet — try again in a moment" status = "Identity not ready yet — try again in a moment"
return return
} }
val thisAttempt = ConnectAttempt(name)
attempt = thisAttempt // shows the ConnectOverlay's "Connecting…" phase immediately
connecting = true connecting = true
status = "Connecting to $targetHost:$targetPort" status = null
discovery.stop() // free the Wi-Fi radio before the stream session discovery.stop() // free the Wi-Fi radio before the stream session
scope.launch { scope.launch {
val handle = connectNative(id, targetHost, targetPort, pinHex ?: "", CONNECT_TIMEOUT_MS) val handle = connectNative(id, targetHost, targetPort, pinHex ?: "", CONNECT_TIMEOUT_MS)
// Cancelled mid-dial: the UI's already been returned (and discovery restarted) by
// cancelConnect — drop the just-opened session silently rather than navigating into it.
if (thisAttempt.cancelled.get()) {
if (handle != 0L) withContext(Dispatchers.IO) { NativeBridge.nativeClose(handle) }
return@launch
}
attempt = null
connecting = false connecting = false
if (handle != 0L) { if (handle != 0L) {
if (pinHex == null) { // TOFU: pin what we observed (unpaired) if (pinHex == null) { // TOFU: pin what we observed (unpaired)
@@ -284,7 +306,9 @@ fun ConnectScreen(
} else { } else {
discovery.start() discovery.start()
if (onFailure != null) { if (onFailure != null) {
status = "" // Hand off to the wake-and-wait flow — clearing `attempt` above and setting
// `waker.waking` here land in one recompose, so the overlay slides
// Connecting → Waking without a blank frame.
onFailure() onFailure()
} else { } else {
status = "Connection failed — check host/port, PIN, and logcat" status = "Connection failed — check host/port, PIN, and logcat"
@@ -293,6 +317,16 @@ fun ConnectScreen(
} }
} }
// Cancel a plain dial in flight (the overlay's "Connecting…" phase, B / Cancel). The native
// connect can't be aborted, so flag this attempt (a late handle is closed silently in
// doConnectDirect) and return the UI now, resuming the discovery we paused for the dial.
fun cancelConnect() {
attempt?.cancelled?.set(true)
attempt = null
connecting = false
discovery.start()
}
// Wake-aware connect. If auto-wake is on (Settings.autoWakeEnabled) and the target is a saved // Wake-aware connect. If auto-wake is on (Settings.autoWakeEnabled) and the target is a saved
// host with a learned MAC that ISN'T currently advertising, fire a wake packet and DIAL // host with a learned MAC that ISN'T currently advertising, fire a wake packet and DIAL
// IMMEDIATELY — mDNS absence does NOT mean unreachable (a host reached over a routed network — // IMMEDIATELY — mDNS absence does NOT mean unreachable (a host reached over a routed network —
@@ -506,27 +540,9 @@ fun ConnectScreen(
Spacer(Modifier.height(24.dp)) Spacer(Modifier.height(24.dp))
status?.let { status?.let {
// While connecting it's progress (spinner, neutral); otherwise it's a // In-flight progress (connecting / waking) is the full-screen ConnectOverlay's
// result/error (red). Previously every status showed in error-red, so a // job now, so `status` only ever carries a result/error here — a filled error
// normal "Connecting…" looked like a failure. // container reads as a real failure banner, not just red text lost in the layout.
if (connecting) {
Row(
verticalAlignment = Alignment.CenterVertically,
horizontalArrangement = Arrangement.spacedBy(8.dp),
) {
CircularProgressIndicator(
modifier = Modifier.size(16.dp),
strokeWidth = 2.dp,
)
Text(
it,
style = MaterialTheme.typography.bodyMedium,
color = MaterialTheme.colorScheme.onSurfaceVariant,
)
}
} else {
// Result/error: a filled error container reads as a real failure banner,
// not just red text lost in the layout.
Surface( Surface(
color = MaterialTheme.colorScheme.errorContainer, color = MaterialTheme.colorScheme.errorContainer,
shape = MaterialTheme.shapes.medium, shape = MaterialTheme.shapes.medium,
@@ -540,7 +556,6 @@ fun ConnectScreen(
modifier = Modifier.padding(horizontal = 16.dp, vertical = 12.dp), modifier = Modifier.padding(horizontal = 16.dp, vertical = 12.dp),
) )
} }
}
Spacer(Modifier.height(16.dp)) Spacer(Modifier.height(16.dp))
} }
} }
@@ -837,8 +852,15 @@ fun ConnectScreen(
} }
} }
// Topmost: the "Waking…" overlay rides over both the touch grid and the console home. // Topmost: the full-screen connect takeover — instant "Connecting…" feedback on any dial, flowing
WakeOverlay(waker, gamepadUi) // seamlessly into the "Waking…" wait if the host turns out to be asleep. Rides over both the touch
// grid and the console home.
ConnectOverlay(
connectingHostName = attempt?.hostName,
waker = waker,
gamepadUi = gamepadUi,
onCancelConnect = { cancelConnect() },
)
} }
/** /**
@@ -158,8 +158,11 @@ fun ControllersScreen(gamepadSetting: Int, onBack: () -> Unit) {
color = MaterialTheme.colorScheme.onSurfaceVariant, color = MaterialTheme.colorScheme.onSurfaceVariant,
) )
} }
pads.forEachIndexed { i, dev -> // Every real controller is forwarded now (Automatic forwards them all, each on its own
PadRow(dev, forwarded = i == 0, gamepadSetting = gamepadSetting) // wire pad index) — not just the first. A joystick-only device Android doesn't classify as
// a gamepad still can't be forwarded (the host wants a gamepad), so gate the badge on it.
pads.forEach { dev ->
PadRow(dev, forwarded = isForwarded(dev), gamepadSetting = gamepadSetting)
} }
} }
@@ -222,8 +225,12 @@ private fun PadRow(dev: InputDevice, forwarded: Boolean, gamepadSetting: Int) {
Row(modifier = Modifier.fillMaxWidth(), verticalAlignment = Alignment.CenterVertically) { Row(modifier = Modifier.fillMaxWidth(), verticalAlignment = Alignment.CenterVertically) {
Text(dev.name, style = MaterialTheme.typography.bodyLarge, modifier = Modifier.weight(1f)) Text(dev.name, style = MaterialTheme.typography.bodyLarge, modifier = Modifier.weight(1f))
if (forwarded) { if (forwarded) {
// Android's own controller number (1-based; 0 = unassigned), shown so a multi-pad
// user can tell which physical pad is which. The stream's wire pad index is
// assigned separately (lowest-free per device) once streaming starts.
val number = dev.controllerNumber
Text( Text(
"forwarded to host", if (number > 0) "forwarded · player $number" else "forwarded to host",
style = MaterialTheme.typography.labelSmall, style = MaterialTheme.typography.labelSmall,
color = MaterialTheme.colorScheme.primary, color = MaterialTheme.colorScheme.primary,
) )
@@ -319,6 +326,15 @@ private fun Group(title: String, content: @Composable ColumnScope.() -> Unit) {
} }
} }
/**
* Whether this device is actually forwarded to the host — the same rule the stream's [GamepadRouter]
* applies: a real, non-virtual controller whose source classes include GAMEPAD. A joystick-only node
* (e.g. a DualSense motion-sensor sibling, or an adapter that enumerates as bare joystick) shows in
* the list but isn't forwarded.
*/
private fun isForwarded(dev: InputDevice): Boolean =
!dev.isVirtual && dev.sources and InputDevice.SOURCE_GAMEPAD == InputDevice.SOURCE_GAMEPAD
/** Whether the controller reports a rumble motor — via VibratorManager (API 31+) or the legacy Vibrator. */ /** Whether the controller reports a rumble motor — via VibratorManager (API 31+) or the legacy Vibrator. */
private fun deviceHasVibrator(dev: InputDevice): Boolean = private fun deviceHasVibrator(dev: InputDevice): Boolean =
if (Build.VERSION.SDK_INT >= 31) { if (Build.VERSION.SDK_INT >= 31) {
@@ -16,6 +16,7 @@ import androidx.compose.runtime.mutableStateOf
import androidx.compose.runtime.setValue import androidx.compose.runtime.setValue
import androidx.compose.ui.Modifier import androidx.compose.ui.Modifier
import io.unom.punktfunk.kit.Gamepad import io.unom.punktfunk.kit.Gamepad
import io.unom.punktfunk.kit.GamepadRouter
import io.unom.punktfunk.kit.Keymap import io.unom.punktfunk.kit.Keymap
import io.unom.punktfunk.kit.NativeBridge import io.unom.punktfunk.kit.NativeBridge
@@ -27,8 +28,12 @@ class MainActivity : ComponentActivity() {
*/ */
var streamHandle: Long = 0L var streamHandle: Long = 0L
/** Joystick-axis state mapper for the active session (built/reset by StreamScreen). */ /**
var axisMapper: Gamepad.AxisMapper? = null * Multi-controller router for the active session (built/released by StreamScreen): assigns each
* connected pad a stable wire index, threads it onto every event, declares/removes pads on
* hot-plug, and routes rumble/HID feedback back by pad index. Null while not streaming.
*/
var gamepadRouter: GamepadRouter? = null
/** /**
* Input observers for the Controllers debug screen (set while it is shown, like [streamHandle]). * Input observers for the Controllers debug screen (set while it is shown, like [streamHandle]).
@@ -44,9 +49,6 @@ class MainActivity : ComponentActivity() {
*/ */
var requestStreamExit: (() -> Unit)? = null var requestStreamExit: (() -> Unit)? = null
/** Currently-held forwarded pad buttons (bitmask of `Gamepad.BTN_*`), for chord detection. */
private var heldPadButtons = 0
/** /**
* Whether the last console input came from a real gamepad (face buttons / stick) vs. a TV D-pad * Whether the last console input came from a real gamepad (face buttons / stick) vs. a TV D-pad
* remote (which has no A/B/X/Y). The console UI reads this to show glyphs the user recognises — pad * remote (which has no A/B/X/Y). The console UI reads this to show glyphs the user recognises — pad
@@ -125,23 +127,12 @@ class MainActivity : ComponentActivity() {
if (event.isFromSource(InputDevice.SOURCE_GAMEPAD)) { if (event.isFromSource(InputDevice.SOURCE_GAMEPAD)) {
val bit = Gamepad.buttonBit(event.keyCode) val bit = Gamepad.buttonBit(event.keyCode)
if (bit != 0) { if (bit != 0) {
when (event.action) { // The router forwards the bit on this device's own wire pad index, tracks held
// repeatCount guard: don't re-send a held button as auto-repeat. // state per pad, and reports when the emergency-exit chord (Select + Start + L1 +
KeyEvent.ACTION_DOWN -> { // R1) completed on any one pad (a couch user has no keyboard/Back).
if (event.repeatCount == 0) NativeBridge.nativeSendGamepadButton(handle, bit, true) if (gamepadRouter?.onButton(event, bit) == true) {
heldPadButtons = heldPadButtons or bit
// Emergency exit: Select + Start + L1 + R1 held together leaves the stream
// (a couch user has no keyboard/Back). Fired once per full chord.
if (heldPadButtons and STREAM_EXIT_CHORD == STREAM_EXIT_CHORD) {
heldPadButtons = 0
requestStreamExit?.let { exit -> window.decorView.post { exit() } } requestStreamExit?.let { exit -> window.decorView.post { exit() } }
} }
}
KeyEvent.ACTION_UP -> {
NativeBridge.nativeSendGamepadButton(handle, bit, false)
heldPadButtons = heldPadButtons and bit.inv()
}
}
return true // consumed return true // consumed
} }
} }
@@ -203,7 +194,7 @@ class MainActivity : ComponentActivity() {
override fun dispatchGenericMotionEvent(event: MotionEvent): Boolean { override fun dispatchGenericMotionEvent(event: MotionEvent): Boolean {
if (streamHandle != 0L) { if (streamHandle != 0L) {
if (axisMapper?.onMotion(event) == true) return true if (gamepadRouter?.onMotion(event) == true) return true
return super.dispatchGenericMotionEvent(event) return super.dispatchGenericMotionEvent(event)
} }
// The Controllers debug screen sees pad motion before the stick→D-pad synthesis below. // The Controllers debug screen sees pad motion before the stick→D-pad synthesis below.
@@ -248,9 +239,4 @@ class MainActivity : ComponentActivity() {
-> true -> true
else -> KeyEvent.isGamepadButton(kc) else -> KeyEvent.isGamepadButton(kc)
} }
private companion object {
/** Emergency stream-exit chord: Select + Start + L1 + R1 held together. */
val STREAM_EXIT_CHORD = Gamepad.BTN_BACK or Gamepad.BTN_START or Gamepad.BTN_LB or Gamepad.BTN_RB
}
} }
@@ -32,8 +32,8 @@ import androidx.core.content.ContextCompat
import androidx.core.view.WindowCompat import androidx.core.view.WindowCompat
import androidx.core.view.WindowInsetsCompat import androidx.core.view.WindowInsetsCompat
import androidx.core.view.WindowInsetsControllerCompat import androidx.core.view.WindowInsetsControllerCompat
import io.unom.punktfunk.kit.Gamepad
import io.unom.punktfunk.kit.GamepadFeedback import io.unom.punktfunk.kit.GamepadFeedback
import io.unom.punktfunk.kit.GamepadRouter
import io.unom.punktfunk.kit.NativeBridge import io.unom.punktfunk.kit.NativeBridge
import io.unom.punktfunk.kit.VideoDecoders import io.unom.punktfunk.kit.VideoDecoders
import java.util.concurrent.atomic.AtomicBoolean import java.util.concurrent.atomic.AtomicBoolean
@@ -174,18 +174,24 @@ fun StreamScreen(handle: Long, micEnabled: Boolean, onDisconnect: () -> Unit) {
val priorOrientation = activity?.requestedOrientation val priorOrientation = activity?.requestedOrientation
activity?.requestedOrientation = ActivityInfo.SCREEN_ORIENTATION_SENSOR_LANDSCAPE activity?.requestedOrientation = ActivityInfo.SCREEN_ORIENTATION_SENSOR_LANDSCAPE
activity?.streamHandle = handle // route hardware keys to this session activity?.streamHandle = handle // route hardware keys to this session
activity?.axisMapper = Gamepad.AxisMapper(handle) // route joystick axes // Multi-controller router: a stable wire pad index per connected controller, per-device axis
// state, Arrival/Remove on hot-plug, and feedback routed back by pad index. Forwards every
// controller (Automatic). Built here, released on dispose.
val router = GamepadRouter(context, handle, initialSettings.gamepad)
activity?.gamepadRouter = router
// Select+Start+L1+R1 chord leaves the stream — a deliberate quit (signal it so the host skips // Select+Start+L1+R1 chord leaves the stream — a deliberate quit (signal it so the host skips
// the keep-alive linger), unlike a host-ended / backgrounded drop. // the keep-alive linger), unlike a host-ended / backgrounded drop.
activity?.requestStreamExit = { NativeBridge.nativeDisconnectQuit(handle); onDisconnect() } activity?.requestStreamExit = { NativeBridge.nativeDisconnectQuit(handle); onDisconnect() }
activity?.setConsoleHighRefreshRate(false) // let the decoder's setFrameRate pick the panel rate activity?.setConsoleHighRefreshRate(false) // let the decoder's setFrameRate pick the panel rate
// Host→client feedback (rumble + DualSense lightbar/LEDs); poll threads stopped before close. // Host→client feedback (rumble + DualSense lightbar/LEDs), routed to each controller by pad
val feedback = GamepadFeedback(handle).also { it.start() } // index via the router; poll threads stopped + joined before the router is released and the
// session closed.
val feedback = GamepadFeedback(handle, router).also { it.start() }
onDispose { onDispose {
closed.set(true) // from here the handle gets freed; surfaceDestroyed must not touch it closed.set(true) // from here the handle gets freed; surfaceDestroyed must not touch it
feedback.stop() // stop + join the poll threads BEFORE nativeClose frees the handle feedback.stop() // stop + join the poll threads BEFORE the router is released / handle freed
activity?.axisMapper?.reset() // release-all so nothing sticks on the host router.release() // flush every slot (nothing sticks host-side) + drop the hot-plug listener
activity?.axisMapper = null activity?.gamepadRouter = null
activity?.streamHandle = 0L activity?.streamHandle = 0L
activity?.requestStreamExit = null activity?.requestStreamExit = null
activity?.setConsoleHighRefreshRate(true) // back to the console UI's max refresh activity?.setConsoleHighRefreshRate(true) // back to the console UI's max refresh
@@ -26,7 +26,7 @@ import kotlinx.coroutines.launch
* [isOnline]/[onOnline] callbacks all run on the main thread; only the blocking send is off-loaded. * [isOnline]/[onOnline] callbacks all run on the main thread; only the blocking send is off-loaded.
*/ */
class WakeController(private val scope: CoroutineScope) { class WakeController(private val scope: CoroutineScope) {
/** null = idle; non-null drives [WakeOverlay]. */ /** null = idle; non-null drives the "Waking…" phase of [ConnectOverlay]. */
data class Waking( data class Waking(
val hostName: String, val hostName: String,
/** Whether coming online chains into a connect (Wake & Connect) vs. just stopping. */ /** Whether coming online chains into a connect (Wake & Connect) vs. just stopping. */
@@ -1,124 +0,0 @@
package io.unom.punktfunk
import androidx.activity.compose.BackHandler
import androidx.compose.foundation.background
import androidx.compose.foundation.border
import androidx.compose.foundation.clickable
import androidx.compose.foundation.interaction.MutableInteractionSource
import androidx.compose.foundation.layout.Arrangement
import androidx.compose.foundation.layout.Box
import androidx.compose.foundation.layout.Column
import androidx.compose.foundation.layout.Row
import androidx.compose.foundation.layout.fillMaxSize
import androidx.compose.foundation.layout.padding
import androidx.compose.foundation.layout.size
import androidx.compose.foundation.layout.widthIn
import androidx.compose.foundation.shape.RoundedCornerShape
import androidx.compose.material.icons.Icons
import androidx.compose.material.icons.filled.Bedtime
import androidx.compose.material3.Button
import androidx.compose.material3.CircularProgressIndicator
import androidx.compose.material3.Icon
import androidx.compose.material3.OutlinedButton
import androidx.compose.material3.Text
import androidx.compose.runtime.Composable
import androidx.compose.runtime.remember
import androidx.compose.ui.Alignment
import androidx.compose.ui.Modifier
import androidx.compose.ui.draw.clip
import androidx.compose.ui.graphics.Color
import androidx.compose.ui.text.font.FontFamily
import androidx.compose.ui.text.font.FontWeight
import androidx.compose.ui.text.style.TextAlign
import androidx.compose.ui.unit.dp
import androidx.compose.ui.unit.sp
/**
* The "Waking <host>…" modal shown while [WakeController] brings a sleeping host back — a spinner + a
* live elapsed counter, escalating to a retry/cancel prompt on timeout. The Android mirror of the
* Apple client's `WakeOverlay`. Rendered over BOTH the touch grid and the console home; it swallows
* input to the screen behind it, and in console mode the pad drives it (B cancels, A retries once
* timed out) while the touch buttons work for a pointer.
*/
@Composable
fun WakeOverlay(waker: WakeController, gamepadUi: Boolean) {
val w = waker.waking ?: return
BackHandler { waker.cancel() } // system Back / pad B (remapped) cancels the wait
if (gamepadUi) {
// A retries once timed out; B falls through to the BackHandler above.
GamepadNavEffect2D(
active = true,
onDirection = {},
onActivate = { if (w.timedOut) waker.retry() },
)
}
Box(
Modifier
.fillMaxSize()
.background(Color.Black.copy(alpha = 0.6f))
// Swallow taps so the home behind can't be touched while waking.
.clickable(interactionSource = remember { MutableInteractionSource() }, indication = null) {},
contentAlignment = Alignment.Center,
) {
Column(
Modifier
.padding(40.dp)
.widthIn(max = 380.dp)
.clip(RoundedCornerShape(22.dp))
.background(Color(0xF01A1730))
.border(1.dp, Color.White.copy(alpha = 0.12f), RoundedCornerShape(22.dp))
.padding(28.dp),
horizontalAlignment = Alignment.CenterHorizontally,
verticalArrangement = Arrangement.spacedBy(14.dp),
) {
if (w.timedOut) {
Icon(
Icons.Filled.Bedtime,
contentDescription = null,
tint = Color.White.copy(alpha = 0.85f),
modifier = Modifier.size(34.dp),
)
Text(
"${w.hostName} didn't wake",
color = Color.White,
fontWeight = FontWeight.Bold,
fontSize = 19.sp,
textAlign = TextAlign.Center,
)
Text(
"It may still be booting, or it's powered off / off this network.",
color = Color.White.copy(alpha = 0.6f),
fontSize = 13.sp,
textAlign = TextAlign.Center,
)
Row(
horizontalArrangement = Arrangement.spacedBy(12.dp),
modifier = Modifier.padding(top = 6.dp),
) {
OutlinedButton(onClick = { waker.cancel() }) { Text("Cancel") }
Button(onClick = { waker.retry() }) { Text("Try Again") }
}
} else {
CircularProgressIndicator(color = Color.White)
Text(
"Waking ${w.hostName}",
color = Color.White,
fontWeight = FontWeight.Bold,
fontSize = 19.sp,
textAlign = TextAlign.Center,
)
Text(
"Waiting for it to come online · ${w.seconds}s",
color = Color.White.copy(alpha = 0.6f),
fontSize = 13.sp,
fontFamily = FontFamily.Monospace,
)
OutlinedButton(onClick = { waker.cancel() }, modifier = Modifier.padding(top = 6.dp)) {
Text(if (w.connectsAfter) "Cancel" else "Stop Waiting")
}
}
}
}
}
@@ -68,6 +68,29 @@ class ScreenshotTest {
@Config(sdk = [36], qualifiers = "w800dp-h360dp-xxhdpi") @Config(sdk = [36], qualifiers = "w800dp-h360dp-xxhdpi")
fun streamNormal() = shootRoot("stream-normal") { StreamScene(io.unom.punktfunk.StatsVerbosity.NORMAL) } fun streamNormal() = shootRoot("stream-normal") { StreamScene(io.unom.punktfunk.StatsVerbosity.NORMAL) }
// The touch flow is a Material dialog over the host grid (a separate window → shootScreen).
@Test
fun connecting() = shootScreen("connecting") {
HostsScene()
ConnectingScene()
}
@Test
fun waking() = shootScreen("waking") {
HostsScene()
WakingScene()
}
@Test
fun wakeTimedOut() = shootScreen("wake-timed-out") {
HostsScene()
WakeTimedOutScene()
}
// The console flow is the full-screen aurora takeover (a root capture).
@Test
fun connectingConsole() = shootRoot("connecting-console") { ConnectConsoleScene() }
@Test @Test
fun trust() = shootScreen("trust") { fun trust() = shootScreen("trust") {
HostsScene() HostsScene()
@@ -26,6 +26,9 @@ import androidx.compose.ui.graphics.Color
import androidx.compose.ui.text.style.TextAlign import androidx.compose.ui.text.style.TextAlign
import androidx.compose.ui.unit.dp import androidx.compose.ui.unit.dp
import io.unom.punktfunk.BrandDark import io.unom.punktfunk.BrandDark
import io.unom.punktfunk.ConnectModal
import io.unom.punktfunk.ConnectPhase
import io.unom.punktfunk.ConnectTakeover
import io.unom.punktfunk.Settings import io.unom.punktfunk.Settings
import io.unom.punktfunk.TouchMode import io.unom.punktfunk.TouchMode
import io.unom.punktfunk.SettingsScreen import io.unom.punktfunk.SettingsScreen
@@ -215,3 +218,31 @@ internal fun StreamScene(verbosity: StatsVerbosity = StatsVerbosity.DETAILED) {
) )
} }
} }
/**
* The default-UI connect flow (the real [ConnectModal]) in each phase — instant "Connecting…"
* feedback, the "Waking…" wait, and the wake-timed-out prompt. These render as a Material dialog over
* the host grid, so the test composes [HostsScene] behind them and captures the whole screen.
*/
@Composable
internal fun ConnectingScene() =
ConnectModal(ConnectPhase.Connecting("Living Room PC"), onCancel = {}, onRetry = {})
@Composable
internal fun WakingScene() =
ConnectModal(
ConnectPhase.Waking("Living Room PC", seconds = 12, connectsAfter = true),
onCancel = {}, onRetry = {},
)
@Composable
internal fun WakeTimedOutScene() =
ConnectModal(ConnectPhase.WakeTimedOut("Living Room PC"), onCancel = {}, onRetry = {})
/**
* The console / gamepad connect flow (the real full-screen [ConnectTakeover]) — the aurora backdrop
* with a bottom hint bar, the same signature look the console home uses.
*/
@Composable
internal fun ConnectConsoleScene() =
ConnectTakeover(ConnectPhase.Connecting("Living Room PC"), onCancel = {}, onRetry = {})
@@ -171,47 +171,26 @@ object Gamepad {
} }
/** /**
* Maps joystick MotionEvents to axis (+ HAT→dpad) sends for one session, **on change only**. * Maps one controller's joystick MotionEvents to axis (+ HAT→dpad) sends on wire pad index [pad],
* Holds the previous axis/hat state so an unchanged frame emits nothing. One instance per * **on change only**. Holds the previous axis/hat state so an unchanged frame emits nothing. One
* session; call [reset] on release-all (focus loss / disconnect / session stop) so nothing * instance per forwarded controller (owned by [GamepadRouter], which routes each device's events
* sticks on the host (which has no client-side held-state knowledge). * to its own mapper so a second pad can't clobber the first); call [reset] on that slot closing
* (disconnect / session stop) so nothing sticks on the host (which has no client-side held-state
* knowledge).
* *
* Single-source: only ONE qualifying controller feeds pad 0. Events must come from a device * The router only ever feeds this a qualifying event from the mapper's own device — a real
* whose source classes include GAMEPAD (see [onMotion]) and the mapper pins itself to the * gamepad (its source classes include GAMEPAD), never a controller's joystick-classified sibling
* first such device — a controller's joystick-classified sibling nodes (DualSense/DS4 motion * node (DualSense/DS4 motion sensors), which reports every pad axis as 0. [onMotion] therefore
* sensors) and any second pad report every axis as 0, and folding them into the same state * folds the event straight in without re-qualifying it.
* flapped a held trigger/stick between its value and 0 on every event interleave.
*/ */
class AxisMapper(private val handle: Long) { class AxisMapper(private val handle: Long, private val pad: Int) {
// Sentinel so the first real value (incl. 0) always sends once after attach (Linux parity). // Sentinel so the first real value (incl. 0) always sends once after attach (Linux parity).
private val last = IntArray(6) { Int.MIN_VALUE } private val last = IntArray(6) { Int.MIN_VALUE }
private var hatX = 0 // -1 / 0 / +1 private var hatX = 0 // -1 / 0 / +1
private var hatY = 0 private var hatY = 0
/** deviceId of the controller pad 0 is pinned to; 1 until the first qualifying event. */ /** Fold one joystick ACTION_MOVE from this mapper's controller onto its pad index. */
private var deviceId = -1 fun onMotion(event: MotionEvent) {
/** Returns true if this was a joystick ACTION_MOVE we consumed. */
fun onMotion(event: MotionEvent): Boolean {
if (!event.isFromSource(InputDevice.SOURCE_JOYSTICK)) return false
if (event.actionMasked != MotionEvent.ACTION_MOVE) return false
// Only a true gamepad drives pad 0. A joystick ACTION_MOVE's own source is plain
// JOYSTICK for every sender, so qualify by the DEVICE's source classes: a real pad
// carries the GAMEPAD (button) class too, its sensor/touchpad sibling nodes and
// joystick-class remotes don't — and those report every pad axis as 0 (see the
// class doc for the held-trigger flap this caused).
val dev = event.device ?: return false
if (dev.sources and InputDevice.SOURCE_GAMEPAD != InputDevice.SOURCE_GAMEPAD) return false
// Single-pad model: pin to the first qualifying controller so a second pad (or its
// stick drift) can't fight pad 0; re-adopt only once the pinned device is gone.
if (deviceId != event.deviceId) {
if (deviceId != -1) {
if (InputDevice.getDevice(deviceId) != null) return false
reset() // the pinned pad is gone — lift its held state before adopting
}
deviceId = event.deviceId
}
// Sticks: Android floats 1..1, +y = down → ±32767, negate Y for the wire's +y = up. // Sticks: Android floats 1..1, +y = down → ±32767, negate Y for the wire's +y = up.
sendAxis(AXIS_LS_X, stick(event.getAxisValue(MotionEvent.AXIS_X))) sendAxis(AXIS_LS_X, stick(event.getAxisValue(MotionEvent.AXIS_X)))
sendAxis(AXIS_LS_Y, stick(-event.getAxisValue(MotionEvent.AXIS_Y))) sendAxis(AXIS_LS_Y, stick(-event.getAxisValue(MotionEvent.AXIS_Y)))
@@ -253,10 +232,9 @@ object Gamepad {
if (hy < 0) btn(BTN_DPAD_UP, true) else if (hy > 0) btn(BTN_DPAD_DOWN, true) if (hy < 0) btn(BTN_DPAD_UP, true) else if (hy > 0) btn(BTN_DPAD_DOWN, true)
hatY = hy hatY = hy
} }
return true
} }
/** Release-all: zero every axis and clear the held dpad. */ /** Release-all: zero every axis and clear the held dpad (all on this mapper's pad index). */
fun reset() { fun reset() {
for (id in 0..5) sendAxis(id, 0) for (id in 0..5) sendAxis(id, 0)
if (hatX < 0) btn(BTN_DPAD_LEFT, false) else if (hatX > 0) btn(BTN_DPAD_RIGHT, false) if (hatX < 0) btn(BTN_DPAD_LEFT, false) else if (hatX > 0) btn(BTN_DPAD_RIGHT, false)
@@ -268,10 +246,10 @@ object Gamepad {
private fun sendAxis(id: Int, v: Int) { private fun sendAxis(id: Int, v: Int) {
if (last[id] == v) return if (last[id] == v) return
last[id] = v last[id] = v
NativeBridge.nativeSendGamepadAxis(handle, id, v) NativeBridge.nativeSendGamepadAxis(handle, id, v, pad)
} }
private fun btn(bit: Int, down: Boolean) = NativeBridge.nativeSendGamepadButton(handle, bit, down) private fun btn(bit: Int, down: Boolean) = NativeBridge.nativeSendGamepadButton(handle, bit, down, pad)
// 1..1 float → ±32767 i16 (matches the Apple client's 32767 scale). // 1..1 float → ±32767 i16 (matches the Apple client's 32767 scale).
private fun stick(v: Float): Int = (v.coerceIn(-1f, 1f) * 32767f).toInt() private fun stick(v: Float): Int = (v.coerceIn(-1f, 1f) * 32767f).toInt()
@@ -15,21 +15,26 @@ import android.view.InputDevice
import java.nio.ByteBuffer import java.nio.ByteBuffer
/** /**
* Host→client gamepad feedback for one session (single-pad model — pad 0 only). Two daemon poll * Host→client gamepad feedback for one session, routed per controller by wire pad index. Two daemon
* threads drain the blocking native pulls and render in Kotlin: rumble → the controller's * poll threads drain the blocking native pulls and render in Kotlin: rumble → the addressed
* `VibratorManager` (API 31+) or its single legacy `Vibrator` on API 2830; HID-output → lightbar / * controller's `VibratorManager` (API 31+) or its single legacy `Vibrator` on API 2830; HID-output
* player-LED via `LightsManager` (API 33+); adaptive * → that controller's lightbar / player-LED via `LightsManager` (API 33+); adaptive triggers are
* triggers are parse-validated and logged (Android has no public adaptive-trigger API). * parse-validated and logged (Android has no public adaptive-trigger API).
*
* Each pull carries the wire pad index it is addressed to; [GamepadRouter.deviceForPad] resolves it
* to the physical controller currently holding that index — so a rumble the host aimed at pad 1
* drives pad 1's motors, and an update for an index with no live controller (a pad that just
* unplugged) is dropped. Per-controller rumble/light bindings are built lazily and cached by device
* id (bounded — at most 16 pads).
* *
* Mirrors `nativeStartAudio`'s lifecycle: [start]/[stop] driven by the StreamScreen. [stop] flips a * Mirrors `nativeStartAudio`'s lifecycle: [start]/[stop] driven by the StreamScreen. [stop] flips a
* flag; the ~100 ms native pull timeout lets the threads exit, then they're joined (bounded) — and * flag; the ~100 ms native pull timeout lets the threads exit, then they're joined (bounded) — and
* this MUST run before `nativeClose` frees the session handle. * this MUST run before the router is released and `nativeClose` frees the session handle.
* *
* The active pad is resolved from the connected input devices (first gamepad/joystick). With none * With no controller connected (emulator) rumble/lights become logged no-ops — exactly the
* connected (emulator) rumble/lights become logged no-ops — exactly the verification path; the * verification path; the `Log.i` receipt lines fire regardless of rendering hardware.
* `Log.i` receipt lines fire regardless of rendering hardware.
*/ */
class GamepadFeedback(private val handle: Long) { class GamepadFeedback(private val handle: Long, private val router: GamepadRouter?) {
private companion object { private companion object {
const val TAG = "pf.feedback" const val TAG = "pf.feedback"
const val TAG_LED: Byte = 0x01 const val TAG_LED: Byte = 0x01
@@ -40,42 +45,48 @@ class GamepadFeedback(private val handle: Long) {
const val LEGACY_RUMBLE_MS = 60_000L const val LEGACY_RUMBLE_MS = 60_000L
} }
/** One controller's rumble binding — VibratorManager (API 31+) OR the legacy single Vibrator (API 2830). */
private class RumbleBind(
val vm: VibratorManager?,
val legacy: Vibrator?,
val ids: IntArray,
val amplitudeControlled: Boolean,
)
/** One controller's lights binding (API 33+): its open session + the RGB / player-id lights it exposes. */
private class LightBind(
val session: LightsManager.LightsSession,
val rgb: Light?,
val player: Light?,
)
@Volatile private var running = false @Volatile private var running = false
private var rumbleThread: Thread? = null private var rumbleThread: Thread? = null
private var hidoutThread: Thread? = null private var hidoutThread: Thread? = null
private var vm: VibratorManager? = null // Per-controller bindings, keyed by device id, built lazily. rumbleBinds is touched ONLY by the
// API 2830 fallback: the controller's single legacy Vibrator (no per-motor VibratorManager // rumble thread and lightBinds ONLY by the hidout thread while running; stop() reads both from the
// until API 31). Exactly one of [vm] / [legacy] is bound; rumble degrades to one blended motor. // main thread AFTER joining those threads (join establishes the happens-before), so plain maps are
private var legacy: Vibrator? = null // race-free. A null value caches "this controller has no vibrator / no controllable lights".
private var vibratorIds: IntArray = IntArray(0) private val rumbleBinds = HashMap<Int, RumbleBind?>()
private var amplitudeControlled = false private val lightBinds = HashMap<Int, LightBind?>()
private var lightsSession: LightsManager.LightsSession? = null
private var rgbLight: Light? = null
private var playerLight: Light? = null
fun start() { fun start() {
val dev = resolvePad()
bindRumble(dev)
if (Build.VERSION.SDK_INT >= 33) {
bindLights(dev)
} else {
Log.i(TAG, "lights need API 33 (have ${Build.VERSION.SDK_INT}) — lightbar/playerLed no-op")
}
running = true running = true
rumbleThread = Thread({ rumbleThread = Thread({
while (running) { while (running) {
val ev = NativeBridge.nativeNextRumble(handle) val ev = NativeBridge.nativeNextRumble(handle)
if (ev < 0L) continue // timeout / closed if (ev < 0L) continue // timeout / closed
// ev bit 48 = has a v2 lease; bits 32..47 = ttl_ms; 16..31 = low; 0..15 = high. The // ev bits 49..52 = wire pad index; bit 48 = has a v2 lease; bits 32..47 = ttl_ms;
// lease flag is out-of-band, so any ttl_ms (incl. 0xFFFF) is a real lease — no // 16..31 = low; 0..15 = high. The lease flag is out-of-band, so any ttl_ms (incl.
// in-band sentinel. No lease (legacy host) → the prior long one-shot. // 0xFFFF) is a real lease — no in-band sentinel. No lease (legacy host) → the prior
// long one-shot.
val pad = ((ev ushr 49) and 0xFL).toInt()
val hasLease = ((ev ushr 48) and 0x1L) == 0x1L val hasLease = ((ev ushr 48) and 0x1L) == 0x1L
val ttl = ((ev ushr 32) and 0xFFFF).toInt() val ttl = ((ev ushr 32) and 0xFFFF).toInt()
val durationMs = if (hasLease) ttl.toLong() else LEGACY_RUMBLE_MS val durationMs = if (hasLease) ttl.toLong() else LEGACY_RUMBLE_MS
renderRumble( renderRumble(
pad,
((ev ushr 16) and 0xFFFF).toInt(), ((ev ushr 16) and 0xFFFF).toInt(),
(ev and 0xFFFF).toInt(), (ev and 0xFFFF).toInt(),
durationMs, durationMs,
@@ -93,100 +104,99 @@ class GamepadFeedback(private val handle: Long) {
}, "pf-hidout").apply { isDaemon = true; start() } }, "pf-hidout").apply { isDaemon = true; start() }
} }
/** Idempotent. Stops + joins the poll threads (must complete before the session handle is freed). */ /** Idempotent. Stops + joins the poll threads (must complete before the router is released / handle freed). */
fun stop() { fun stop() {
running = false running = false
rumbleThread?.interrupt() rumbleThread?.interrupt()
hidoutThread?.interrupt() hidoutThread?.interrupt()
runCatching { vm?.cancel() } // drop any held rumble immediately
runCatching { legacy?.cancel() }
// Join WITHOUT a timeout. These poll threads dereference the native session handle on every // Join WITHOUT a timeout. These poll threads dereference the native session handle on every
// pull (nativeNextRumble/nativeNextHidout), so they MUST be dead before StreamScreen's // pull (nativeNextRumble/nativeNextHidout) and read the router, so they MUST be dead before
// onDispose reaches nativeClose, which frees that handle. A *bounded* join that times out // StreamScreen's onDispose reaches router.release() / nativeClose, which free that state. A
// would let a thread survive into the freed handle → use-after-free SIGSEGV (the // *bounded* join that times out would let a thread survive into the freed handle → use-after-
// back-while-streaming crash, on the one path the main-thread `closed` guard can't cover). // free SIGSEGV (the back-while-streaming crash, on the one path the main-thread `closed` guard
// Safe to block unbounded: the native pulls are internally time-bounded (PULL_TIMEOUT ~100 ms) // can't cover). Safe to block unbounded: the native pulls are internally time-bounded
// and rendering is a quick best-effort binder call, so each thread observes running=false and // (PULL_TIMEOUT ~100 ms) and rendering is a quick best-effort binder call, so each thread
// exits within ~one timeout — the join returns promptly (well under any ANR threshold). // observes running=false and exits within ~one timeout — the join returns promptly.
runCatching { rumbleThread?.join() } runCatching { rumbleThread?.join() }
runCatching { hidoutThread?.join() } runCatching { hidoutThread?.join() }
rumbleThread = null rumbleThread = null
hidoutThread = null hidoutThread = null
runCatching { lightsSession?.close() } // Threads are dead — drop any held rumble and close every lights session.
lightsSession = null for (b in rumbleBinds.values) b?.let {
rgbLight = null runCatching { it.vm?.cancel() }
playerLight = null runCatching { it.legacy?.cancel() }
vm = null }
legacy = null for (b in lightBinds.values) b?.let { runCatching { it.session.close() } }
vibratorIds = IntArray(0) rumbleBinds.clear()
lightBinds.clear()
} }
/** First connected gamepad/joystick InputDevice, or null (→ logged no-op on the emulator). */
private fun resolvePad(): InputDevice? = Gamepad.firstPad()
// ---- Rumble ---- // ---- Rumble ----
private fun bindRumble(dev: InputDevice?) { /** The rumble binding for the controller on wire pad [pad], or null (no live pad / no vibrator). Cached by device id. */
if (dev == null) { private fun rumbleBindFor(pad: Int): RumbleBind? {
Log.i(TAG, "rumble: no controller connected — rumble no-op (emulator path)") val dev = router?.deviceForPad(pad) ?: return null
return if (rumbleBinds.containsKey(dev.id)) return rumbleBinds[dev.id]
val bind = bindRumble(dev)
rumbleBinds[dev.id] = bind
return bind
} }
private fun bindRumble(dev: InputDevice): RumbleBind? {
if (Build.VERSION.SDK_INT >= 31) { if (Build.VERSION.SDK_INT >= 31) {
val m = dev.vibratorManager val m = dev.vibratorManager
val ids = m.vibratorIds val ids = m.vibratorIds
if (ids.isEmpty()) { if (ids.isEmpty()) {
Log.i(TAG, "rumble: controller '${dev.name}' has no vibrators — rumble no-op") Log.i(TAG, "rumble: controller '${dev.name}' has no vibrators — rumble no-op")
return return null
}
val amp = ids.all { m.getVibrator(it).hasAmplitudeControl() }
Log.i(TAG, "rumble: bound ${ids.size} vibrators for '${dev.name}' amplitudeControl=$amp")
return RumbleBind(m, null, ids, amp)
} }
vm = m
vibratorIds = ids
amplitudeControlled = ids.all { m.getVibrator(it).hasAmplitudeControl() }
Log.i(TAG, "rumble: bound ${ids.size} vibrators amplitudeControl=$amplitudeControlled")
} else {
// API 2830: no VibratorManager — fall back to the controller's single legacy Vibrator. // API 2830: no VibratorManager — fall back to the controller's single legacy Vibrator.
@Suppress("DEPRECATION") @Suppress("DEPRECATION")
val v = dev.vibrator val v = dev.vibrator
if (!v.hasVibrator()) { if (!v.hasVibrator()) {
Log.i(TAG, "rumble: controller '${dev.name}' has no vibrator — rumble no-op") Log.i(TAG, "rumble: controller '${dev.name}' has no vibrator — rumble no-op")
return return null
}
legacy = v
amplitudeControlled = v.hasAmplitudeControl()
Log.i(TAG, "rumble: bound legacy vibrator amplitudeControl=$amplitudeControlled")
} }
Log.i(TAG, "rumble: bound legacy vibrator for '${dev.name}' amplitudeControl=${v.hasAmplitudeControl()}")
return RumbleBind(null, v, IntArray(0), v.hasAmplitudeControl())
} }
/** /**
* low = heavy/left motor, high = light/right motor; both 0..0xFFFF (the host's u16 amplitudes). * low = heavy/left motor, high = light/right motor; both 0..0xFFFF (the host's u16 amplitudes),
* `durationMs` is the host's v2 envelope TTL — the one-shot self-terminates after it unless the * addressed to wire pad [pad]. `durationMs` is the host's v2 envelope TTL — the one-shot self-
* host renews, so a lost stop (or a dead host) silences at the lease instead of the old fixed * terminates after it unless the host renews, so a lost stop (or a dead host) silences at the
* 60 s. Against a legacy host it is [LEGACY_RUMBLE_MS] (the prior fixed duration). * lease instead of the old fixed 60 s. Against a legacy host it is [LEGACY_RUMBLE_MS].
*/ */
private fun renderRumble(low: Int, high: Int, durationMs: Long) { private fun renderRumble(pad: Int, low: Int, high: Int, durationMs: Long) {
Log.i(TAG, "rumble low=$low high=$high ttlMs=$durationMs") // verification line — BEFORE any no-op return Log.i(TAG, "rumble pad=$pad low=$low high=$high ttlMs=$durationMs") // verification line — BEFORE any no-op return
val bind = rumbleBindFor(pad) ?: return
val lo = toAmplitude(low) val lo = toAmplitude(low)
val hi = toAmplitude(high) val hi = toAmplitude(high)
val m = vm val m = bind.vm
if (m != null) { if (m != null) {
if (lo == 0 && hi == 0) { if (lo == 0 && hi == 0) {
m.cancel() // (0,0) = stop m.cancel() // (0,0) = stop
return return
} }
val combo = CombinedVibration.startParallel() val combo = CombinedVibration.startParallel()
if (amplitudeControlled && vibratorIds.size >= 2) { if (bind.amplitudeControlled && bind.ids.size >= 2) {
// ids[0] = light/right, ids[1] = heavy/left (XInput/Moonlight convention). // ids[0] = light/right, ids[1] = heavy/left (XInput/Moonlight convention).
if (hi != 0) combo.addVibrator(vibratorIds[0], oneShot(hi, durationMs)) if (hi != 0) combo.addVibrator(bind.ids[0], oneShot(hi, durationMs))
if (lo != 0) combo.addVibrator(vibratorIds[1], oneShot(lo, durationMs)) if (lo != 0) combo.addVibrator(bind.ids[1], oneShot(lo, durationMs))
} else { } else {
// Single motor or no amplitude control: blend both into one effect. // Single motor or no amplitude control: blend both into one effect.
val a = (lo * 0.8 + hi * 0.33).toInt().coerceIn(1, 255) val a = (lo * 0.8 + hi * 0.33).toInt().coerceIn(1, 255)
for (id in vibratorIds) combo.addVibrator(id, oneShot(a, durationMs)) for (id in bind.ids) combo.addVibrator(id, oneShot(a, durationMs))
} }
runCatching { m.vibrate(combo.combine()) } runCatching { m.vibrate(combo.combine()) }
return return
} }
// API 2830 legacy single-motor path: blend both motors into one effect. // API 2830 legacy single-motor path: blend both motors into one effect.
val lv = legacy ?: return val lv = bind.legacy ?: return
if (lo == 0 && hi == 0) { if (lo == 0 && hi == 0) {
lv.cancel() // (0,0) = stop lv.cancel() // (0,0) = stop
return return
@@ -194,7 +204,7 @@ class GamepadFeedback(private val handle: Long) {
val a = (lo * 0.8 + hi * 0.33).toInt().coerceIn(1, 255) val a = (lo * 0.8 + hi * 0.33).toInt().coerceIn(1, 255)
runCatching { runCatching {
lv.vibrate( lv.vibrate(
if (amplitudeControlled) oneShot(a, durationMs) if (bind.amplitudeControlled) oneShot(a, durationMs)
else oneShot(VibrationEffect.DEFAULT_AMPLITUDE, durationMs) else oneShot(VibrationEffect.DEFAULT_AMPLITUDE, durationMs)
) )
} }
@@ -215,28 +225,29 @@ class GamepadFeedback(private val handle: Long) {
private fun dispatchHidout(buf: ByteBuffer, n: Int) { private fun dispatchHidout(buf: ByteBuffer, n: Int) {
buf.rewind() buf.rewind()
val pad = buf.get().toInt() and 0xFF // wire pad index the event is addressed to
when (buf.get()) { // kind tag when (buf.get()) { // kind tag
TAG_LED -> { TAG_LED -> {
val r = buf.get().toInt() and 0xFF val r = buf.get().toInt() and 0xFF
val g = buf.get().toInt() and 0xFF val g = buf.get().toInt() and 0xFF
val b = buf.get().toInt() and 0xFF val b = buf.get().toInt() and 0xFF
Log.i(TAG, "hidout Led r=$r g=$g b=$b") // verification line Log.i(TAG, "hidout pad=$pad Led r=$r g=$g b=$b") // verification line
if (Build.VERSION.SDK_INT >= 33) setLightbar(Color.rgb(r, g, b)) if (Build.VERSION.SDK_INT >= 33) setLightbar(pad, Color.rgb(r, g, b))
} }
TAG_PLAYER_LEDS -> { TAG_PLAYER_LEDS -> {
val bits = buf.get().toInt() and 0x1F val bits = buf.get().toInt() and 0x1F
val player = playerIndexForBits(bits) val player = playerIndexForBits(bits)
Log.i(TAG, "hidout PlayerLeds bits=$bits player=$player") // verification line Log.i(TAG, "hidout pad=$pad PlayerLeds bits=$bits player=$player") // verification line
if (Build.VERSION.SDK_INT >= 33) setPlayerId(player) if (Build.VERSION.SDK_INT >= 33) setPlayerId(pad, player)
} }
TAG_TRIGGER -> { TAG_TRIGGER -> {
val which = buf.get().toInt() and 0xFF // 0 = L2, 1 = R2 val which = buf.get().toInt() and 0xFF // 0 = L2, 1 = R2
val effLen = n - 2 val effLen = n - 3 // [pad][kind][which] header, then the effect block
val mode = if (effLen > 0) buf.get().toInt() and 0xFF else 0 val mode = if (effLen > 0) buf.get().toInt() and 0xFF else 0
// No public adaptive-trigger API on Android — parse-validate the mode + log only. // No public adaptive-trigger API on Android — parse-validate the mode + log only.
Log.i( Log.i(
TAG, TAG,
"hidout Trigger which=$which effLen=$effLen mode=0x%02x (adaptive triggers unsupported on Android)".format(mode), "hidout pad=$pad Trigger which=$which effLen=$effLen mode=0x%02x (adaptive triggers unsupported on Android)".format(mode),
) )
} }
else -> Log.d(TAG, "hidout: unknown kind, dropped") else -> Log.d(TAG, "hidout: unknown kind, dropped")
@@ -253,37 +264,46 @@ class GamepadFeedback(private val handle: Long) {
else -> Integer.bitCount(bits and 0x1F).coerceIn(1, 4) else -> Integer.bitCount(bits and 0x1F).coerceIn(1, 4)
} }
private fun bindLights(dev: InputDevice?) { /** The lights binding for the controller on wire pad [pad], or null (no live pad / no lights / < API 33). Cached by device id. */
if (dev == null) { private fun lightBindFor(pad: Int): LightBind? {
Log.i(TAG, "lights: no controller connected — lightbar/playerLed no-op (emulator path)") if (Build.VERSION.SDK_INT < 33) return null
return val dev = router?.deviceForPad(pad) ?: return null
if (lightBinds.containsKey(dev.id)) return lightBinds[dev.id]
val bind = bindLights(dev)
lightBinds[dev.id] = bind
return bind
} }
private fun bindLights(dev: InputDevice): LightBind? {
val lm = dev.lightsManager val lm = dev.lightsManager
var rgb: Light? = null
var player: Light? = null
for (l in lm.lights) { for (l in lm.lights) {
if (rgbLight == null && l.hasRgbControl()) rgbLight = l if (rgb == null && l.hasRgbControl()) rgb = l
if (playerLight == null && l.type == Light.LIGHT_TYPE_PLAYER_ID) playerLight = l if (player == null && l.type == Light.LIGHT_TYPE_PLAYER_ID) player = l
} }
if (rgbLight == null && playerLight == null) { if (rgb == null && player == null) {
Log.i(TAG, "lights: controller '${dev.name}' exposes no controllable lights — no-op") Log.i(TAG, "lights: controller '${dev.name}' exposes no controllable lights — no-op")
return return null
} }
lightsSession = lm.openSession() val session = lm.openSession()
Log.i(TAG, "lights: bound rgb=${rgbLight != null} playerLed=${playerLight != null}") Log.i(TAG, "lights: bound rgb=${rgb != null} playerLed=${player != null} for '${dev.name}'")
return LightBind(session, rgb, player)
} }
private fun setLightbar(argb: Int) { private fun setLightbar(pad: Int, argb: Int) {
val s = lightsSession ?: return val bind = lightBindFor(pad) ?: return
val l = rgbLight ?: return val l = bind.rgb ?: return
runCatching { runCatching {
s.requestLights(LightsRequest.Builder().addLight(l, LightState.Builder().setColor(argb).build()).build()) bind.session.requestLights(LightsRequest.Builder().addLight(l, LightState.Builder().setColor(argb).build()).build())
} }
} }
private fun setPlayerId(player: Int) { private fun setPlayerId(pad: Int, player: Int) {
val s = lightsSession ?: return val bind = lightBindFor(pad) ?: return
val l = playerLight ?: return val l = bind.player ?: return
runCatching { runCatching {
s.requestLights(LightsRequest.Builder().addLight(l, LightState.Builder().setPlayerId(player).build()).build()) bind.session.requestLights(LightsRequest.Builder().addLight(l, LightState.Builder().setPlayerId(player).build()).build())
} }
} }
} }
@@ -0,0 +1,204 @@
package io.unom.punktfunk.kit
import android.content.Context
import android.hardware.input.InputManager
import android.os.Handler
import android.os.Looper
import android.view.InputDevice
import android.view.KeyEvent
import android.view.MotionEvent
import java.util.concurrent.ConcurrentHashMap
/**
* Multi-controller router for one stream session — the Android analogue of the Linux client's gamepad
* `Worker`/`Slot` model (`pf-client-core/src/gamepad.rs`) over the shared native-plane wire contract
* (`punktfunk-core/src/input.rs`). Each physical controller (Android `deviceId`) gets a STABLE
* lowest-free wire pad index (0..15) held for its lifetime and freed only on disconnect, so a pad
* dropping never renumbers the others (a game must not see its players shuffle). Every forwarded event
* carries that pad index; a [NativeBridge.nativeSendGamepadArrival] declaring the pad's type is sent
* once BEFORE its first input, a [NativeBridge.nativeSendGamepadRemove] on disconnect. Per-device axis
* state lives in each slot's [Gamepad.AxisMapper] so a second controller can't clobber the first.
* Feedback (rumble / HID) is routed BACK to the originating device by pad index via [deviceForPad].
*
* Selection: forward EVERY real controller (the Linux client's single-player pin has no Android UI
* surface yet — Automatic is the only mode). Lifetime matches the session: constructed on stream
* attach (opening a slot for every already-connected pad, so its Arrival lands before any input),
* released on detach.
*
* A single controller lands on wire index 0, so its per-transition button/axis wire is byte-identical
* to the old single-pad path (plus the Arrival/Remove declarations the contract requires — which an
* older host simply ignores).
*
* Threading: slot mutation + dispatch run on the main thread (Android input dispatch and the
* InputManager hot-plug callbacks both land there). [deviceForPad] is read from the feedback poll
* threads, so the slot table is a [ConcurrentHashMap].
*/
class GamepadRouter(context: Context, private val handle: Long, private val setting: Int) {
/** One forwarded controller: its stable wire pad index, per-device axis state, and held buttons. */
private class Slot(val index: Int, val mapper: Gamepad.AxisMapper) {
/** Forwarded button bits currently held (Gamepad.BTN_*) — for release-on-close + chord detection. */
var held = 0
}
/** deviceId → slot. Concurrent: the feedback poll threads read it via [deviceForPad]. */
private val slots = ConcurrentHashMap<Int, Slot>()
private val inputManager = context.getSystemService(InputManager::class.java)
private val listener = object : InputManager.InputDeviceListener {
override fun onInputDeviceAdded(deviceId: Int) {
InputDevice.getDevice(deviceId)?.let { if (isForwardable(it)) openSlot(it) }
}
override fun onInputDeviceRemoved(deviceId: Int) = closeSlot(deviceId)
override fun onInputDeviceChanged(deviceId: Int) {}
}
init {
inputManager?.registerInputDeviceListener(listener, Handler(Looper.getMainLooper()))
// Open a slot for every controller already connected when the session starts — the pads that
// will never fire onInputDeviceAdded during this session; their Arrival lands before any input.
for (id in InputDevice.getDeviceIds()) {
InputDevice.getDevice(id)?.let { if (isForwardable(it)) openSlot(it) }
}
}
/**
* One gamepad button transition for the device that produced [event] (already resolved to BTN_*
* bit [bit]). Opens the device's slot (declaring its type) if unseen, forwards the bit on the
* slot's pad index, tracks held state, and returns true when this press completed the emergency
* stream-exit chord (Select + Start + L1 + R1) on THIS pad — the caller then leaves the stream
* (mirrors the Linux client's escape chord: any one controller can leave).
*/
fun onButton(event: KeyEvent, bit: Int): Boolean {
val slot = slotFor(event.device) ?: return false
when (event.action) {
KeyEvent.ACTION_DOWN -> {
// repeatCount guard: don't re-send a held button as auto-repeat.
if (event.repeatCount == 0) NativeBridge.nativeSendGamepadButton(handle, bit, true, slot.index)
slot.held = slot.held or bit
if (slot.held and EXIT_CHORD == EXIT_CHORD) {
slot.held = 0
return true
}
}
KeyEvent.ACTION_UP -> {
NativeBridge.nativeSendGamepadButton(handle, bit, false, slot.index)
slot.held = slot.held and bit.inv()
}
}
return false
}
/**
* One joystick MotionEvent — routed to the producing device's own [Gamepad.AxisMapper] (per-device
* state). Returns true if consumed. Only a real gamepad drives a pad: a DualSense/DS4 motion-sensor
* sibling node classifies as bare joystick (no GAMEPAD source class) and reports every pad axis as
* 0, so [isForwardable] filters it out before it can open a slot or clobber axes.
*/
fun onMotion(event: MotionEvent): Boolean {
if (!event.isFromSource(InputDevice.SOURCE_JOYSTICK)) return false
if (event.actionMasked != MotionEvent.ACTION_MOVE) return false
val dev = event.device ?: return false
if (!isForwardable(dev)) return false
val slot = slotFor(dev) ?: return false
slot.mapper.onMotion(event)
return true
}
/**
* The controller currently mapped to wire pad [pad], for feedback routing; null if that index
* holds no live slot (a pad that just unplugged — the update is then dropped). Read from the
* feedback poll threads.
*/
fun deviceForPad(pad: Int): InputDevice? {
for ((deviceId, slot) in slots) {
if (slot.index == pad) return InputDevice.getDevice(deviceId)
}
return null
}
/**
* Flush + drop every slot and unregister the hot-plug listener. Call on session teardown, AFTER
* the feedback poll threads are joined (they read [deviceForPad]).
*/
fun release() {
inputManager?.unregisterInputDeviceListener(listener)
// Snapshot the ids first — closeSlot mutates the map.
for (id in slots.keys.toList()) closeSlot(id)
}
// ---- slots ----
/** A real, non-virtual controller we forward — its source classes include GAMEPAD (excludes a pad's bare-joystick sensor node). */
private fun isForwardable(dev: InputDevice): Boolean =
!dev.isVirtual && dev.sources and InputDevice.SOURCE_GAMEPAD == InputDevice.SOURCE_GAMEPAD
/**
* The slot for [dev], opening one (and declaring the pad) if this device is unseen; null when [dev]
* isn't a forwardable controller or every wire index is taken. The [isForwardable] gate lives here —
* the single lazy-open chokepoint both [onButton] and [onMotion] funnel through — so no entry point
* can open a phantom slot for a virtual/non-gamepad source (the hot-plug listener and init loop
* pre-filter and call [openSlot] directly).
*/
private fun slotFor(dev: InputDevice?): Slot? {
if (dev == null) return null
slots[dev.id]?.let { return it }
if (!isForwardable(dev)) return null
return openSlot(dev)
}
/**
* Open a slot for [dev] on the lowest free wire index, declaring its kind ([NativeBridge.nativeSendGamepadArrival])
* before any input so the host builds a matching virtual device (mixed types across pads).
* Idempotent; null when all 16 wire indices are already forwarded.
*/
private fun openSlot(dev: InputDevice): Slot? {
slots[dev.id]?.let { return it }
val index = lowestFreeIndex() ?: return null // 16 pads already forwarded — drop this one
// Automatic resolves the pad's type from its VID/PID; an explicit setting forces every pad
// to that type (a single global choice — matches the handshake's session-default pref).
val pref = if (setting == Gamepad.PREF_AUTO) Gamepad.prefFor(dev) else setting
NativeBridge.nativeSendGamepadArrival(handle, pref, index)
val slot = Slot(index, Gamepad.AxisMapper(handle, index))
slots[dev.id] = slot
return slot
}
/**
* Flush a slot's held wire state (so nothing sticks host-side), signal the removal, and free its
* index. Safe against an already-gone device — the flush emits wire events only, no device access.
*/
private fun closeSlot(deviceId: Int) {
val slot = slots.remove(deviceId) ?: return
releaseHeld(slot)
NativeBridge.nativeSendGamepadRemove(handle, slot.index)
}
/** Lift every held button + zero the axes/HAT dpad for [slot] (wire events only, all on its index). */
private fun releaseHeld(slot: Slot) {
var bits = slot.held
while (bits != 0) {
val bit = bits and -bits // lowest set bit
NativeBridge.nativeSendGamepadButton(handle, bit, false, slot.index)
bits = bits and bit.inv()
}
slot.held = 0
slot.mapper.reset() // zero sticks/triggers + release the HAT dpad
}
/** Lowest wire index 0..[MAX_PADS) not held by a slot, or null when full — stable lowest-free keeps indices from shuffling on hot-plug. */
private fun lowestFreeIndex(): Int? {
val taken = slots.values.mapTo(HashSet()) { it.index }
for (i in 0 until MAX_PADS) if (i !in taken) return i
return null
}
private companion object {
/** Mirror of `punktfunk-core::input::MAX_PADS` — wire pad indices 0..15. */
const val MAX_PADS = 16
/** Emergency stream-exit chord: Select + Start + L1 + R1 held together (matches the legacy single-pad chord). */
const val EXIT_CHORD = Gamepad.BTN_BACK or Gamepad.BTN_START or Gamepad.BTN_LB or Gamepad.BTN_RB
}
}
@@ -269,26 +269,43 @@ object NativeBridge {
/** One key transition. vk: Windows VK (0 = dropped by Rust). mods: VK modifier mask (0 for now). */ /** One key transition. vk: Windows VK (0 = dropped by Rust). mods: VK modifier mask (0 for now). */
external fun nativeSendKey(handle: Long, vk: Int, down: Boolean, mods: Int) external fun nativeSendKey(handle: Long, vk: Int, down: Boolean, mods: Int)
// ---- Gamepad: one pad forwarded as pad 0 (Rust hardcodes flags=0) ---- // ---- Gamepad: each controller forwarded on its own wire pad index (0..15, low byte of flags) ----
// The pad index is assigned per Android device by GamepadRouter; a single controller lands on 0,
// so its wire is byte-identical to the old single-pad path. The core folds the per-transition
// events into seq'd GamepadState snapshots keyed on this index and owns the per-pad seq.
/** One gamepad button transition. bit: a [Gamepad].BTN_* bit. down: press/release. */ /** One gamepad button transition on wire pad [pad] (0..15). bit: a [Gamepad].BTN_* bit. down: press/release. */
external fun nativeSendGamepadButton(handle: Long, bit: Int, down: Boolean) external fun nativeSendGamepadButton(handle: Long, bit: Int, down: Boolean, pad: Int)
/** One gamepad axis update. axisId: [Gamepad].AXIS_* (0..5). value: stick i16 (+y=up) / trigger 0..255. */ /** One gamepad axis update on wire pad [pad] (0..15). axisId: [Gamepad].AXIS_* (0..5). value: stick i16 (+y=up) / trigger 0..255. */
external fun nativeSendGamepadAxis(handle: Long, axisId: Int, value: Int) external fun nativeSendGamepadAxis(handle: Long, axisId: Int, value: Int, pad: Int)
/**
* Declare the controller KIND presented on wire pad [pad] (0..15) so the host builds a matching
* virtual device (mixed types across pads). pref: a [Gamepad].PREF_* wire byte. Send ONCE when a
* pad opens, BEFORE any of its input; an older host ignores it (that pad then uses the handshake's
* session-default kind — the pre-existing single-pad behaviour on pad 0).
*/
external fun nativeSendGamepadArrival(handle: Long, pref: Int, pad: Int)
/** Signal wire pad [pad] (0..15) was unplugged so the host tears its virtual device down. The core stamps the seq + re-sends. */
external fun nativeSendGamepadRemove(handle: Long, pad: Int)
// ---- Host→client gamepad feedback: Rust pulls block ~100ms, Kotlin renders (see GamepadFeedback) ---- // ---- Host→client gamepad feedback: Rust pulls block ~100ms, Kotlin renders (see GamepadFeedback) ----
/** /**
* Block up to ~100 ms for the next rumble update. Returns `(low shl 16) or high` (each * Block up to ~100 ms for the next rumble update. Returns a packed positive long: bits 49..52 =
* 0..0xFFFF; 0 = stop), or -1 on timeout / session closed. Call from a dedicated poll thread. * wire pad index (0..15), bit 48 = has a v2 lease, bits 32..47 = ttl_ms, bits 16..31 = low, bits
* 0..15 = high (each amplitude 0..0xFFFF; 0/0 = stop), or -1 on timeout / session closed. Kotlin
* routes the update to the controller holding that pad index. Call from a dedicated poll thread.
*/ */
external fun nativeNextRumble(handle: Long): Long external fun nativeNextRumble(handle: Long): Long
/** /**
* Block up to ~100 ms for the next DualSense HID-output event, written into [buf] (a direct * Block up to ~100 ms for the next DualSense HID-output event, written into [buf] (a direct
* ByteBuffer, capacity >= 64) as `[kind][fields…]`: Led=01 r g b, PlayerLeds=02 bits, * ByteBuffer, capacity >= 64) as `[pad][kind][fields…]` (leading pad = the wire pad index to
* Trigger=03 which effect…. Returns the byte count, or -1 on timeout / session closed. * route to): Led=pad 01 r g b, PlayerLeds=pad 02 bits, Trigger=pad 03 which effect…. Returns the
* byte count, or -1 on timeout / session closed.
*/ */
external fun nativeNextHidout(handle: Long, buf: java.nio.ByteBuffer): Int external fun nativeNextHidout(handle: Long, buf: java.nio.ByteBuffer): Int
} }
+134 -36
View File
@@ -15,6 +15,7 @@ use ndk::media::media_format::MediaFormat;
use ndk::native_window::NativeWindow; use ndk::native_window::NativeWindow;
use punktfunk_core::client::NativeClient; use punktfunk_core::client::NativeClient;
use punktfunk_core::error::PunktfunkError; use punktfunk_core::error::PunktfunkError;
use punktfunk_core::reanchor::{GateVerdict, ReanchorGate};
use punktfunk_core::session::Frame; use punktfunk_core::session::Frame;
use std::collections::VecDeque; use std::collections::VecDeque;
use std::ffi::c_void; use std::ffi::c_void;
@@ -208,9 +209,15 @@ fn run_sync(
// pressure the AU stays parked here instead of being dropped (a drop forces a keyframe // pressure the AU stays parked here instead of being dropped (a drop forces a keyframe
// round-trip) and we only pop the next one once it's queued. // round-trip) and we only pop the next one once it's queued.
let mut pending: Option<Frame> = None; let mut pending: Option<Frame> = None;
// Loss recovery: watch the host→client unrecoverable-drop count and ask for an IDR when it // Freeze-until-reanchor: the shared post-loss gate ([`punktfunk_core::reanchor::ReanchorGate`]).
// climbs. // Armed on a frame-index gap or a dropped-count climb, it withholds the decoder's concealed output
let mut last_dropped = client.frames_dropped(); // (released WITHOUT rendering — the SurfaceView keeps the last rendered frame on glass) until a
// proven clean re-anchor lifts it: an IDR (wire FLAG_SOF), an RFI anchor, or the 2nd recovery mark.
// `last_kf_req` throttles the keyframe intents it emits; `recovery_flags` carries each AU's
// user_flags from feed to present (keyed by the codec-echoed pts) so `on_decoded` reads the
// re-anchor signalling the platform decoder doesn't expose.
let mut gate = ReanchorGate::new(client.frames_dropped());
let mut recovery_flags: VecDeque<(u64, u32)> = VecDeque::new();
let mut last_kf_req: Option<Instant> = None; let mut last_kf_req: Option<Instant> = None;
// Skew-corrected latency stats (spec: design/stats-unification.md) use the negotiated // Skew-corrected latency stats (spec: design/stats-unification.md) use the negotiated
// host-minus-client clock offset (0 if the host didn't answer the skew handshake — then the // host-minus-client clock offset (0 if the host didn't answer the skew handshake — then the
@@ -243,6 +250,20 @@ fn run_sync(
if pending.is_none() { if pending.is_none() {
match client.next_frame(Duration::from_millis(5)) { match client.next_frame(Duration::from_millis(5)) {
Ok(frame) => { Ok(frame) => {
// Loss recovery (RFI): feed the frame index so a forward gap fires a throttled
// reference-frame-invalidation request — an RFI-capable host (AMD LTR / NVENC)
// recovers with a cheap clean P-frame instead of a full IDR. The same forward gap
// arms the freeze gate so the decoder's concealment is held off the screen until the
// recovery re-anchors. The frames_dropped keyframe path below stays the backstop.
if client.note_frame_index(frame.frame_index) {
gate.arm(Instant::now());
}
// Park this AU's re-anchor flags for the present side (keyed by the pts the codec
// echoes on the output buffer) — unconditional, unlike the HUD's `in_flight` map.
recovery_flags.push_back((frame.pts_ns / 1000, frame.flags));
if recovery_flags.len() > IN_FLIGHT_CAP {
recovery_flags.pop_front();
}
if fed == 0 { if fed == 0 {
let p = &frame.data; let p = &frame.data;
log::info!( log::info!(
@@ -331,6 +352,8 @@ fn run_sync(
&mut in_flight, &mut in_flight,
clock_offset.load(Ordering::Relaxed), clock_offset.load(Ordering::Relaxed),
&tracker, &tracker,
&mut gate,
&mut recovery_flags,
); );
rendered += r; rendered += r;
discarded += d; discarded += d;
@@ -370,21 +393,19 @@ fn run_sync(
work_accum_ns = 0; work_accum_ns = 0;
} }
// Loss recovery: under infinite GOP the only recovery keyframe is one we request. The // Loss recovery + overdue backstop, folded through the gate. Under infinite GOP the only
// reassembler drops unrecoverable AUs (frames_dropped); the decoder then conceals the // recovery keyframe is one we request; the reassembler drops unrecoverable AUs (frames_dropped)
// reference-missing delta frames that follow and renders them without error, so keying off // and the decoder then conceals the reference-missing deltas and renders them without error, so
// a decode error rarely fires. Request an IDR when the drop count climbs, throttled — the // a decode-error trigger rarely fires — the gate arms the freeze on the drop-count climb
// decode stays wedged for several frames until the IDR lands, so requesting every frame // instead. An overdue freeze (held REANCHOR_FREEZE_MAX with no clean re-anchor) re-asks while it
// would flood the control stream. // keeps holding: never resume to gray — a dead stream is the QUIC idle-timeout watchdog's job.
let dropped = client.frames_dropped();
if dropped > last_dropped {
last_dropped = dropped;
let now = Instant::now(); let now = Instant::now();
if last_kf_req.is_none_or(|t| now.duration_since(t) >= Duration::from_millis(100)) { if gate.poll(client.frames_dropped(), now)
&& last_kf_req.is_none_or(|t| now.duration_since(t) >= Duration::from_millis(100))
{
last_kf_req = Some(now); last_kf_req = Some(now);
let _ = client.request_keyframe(); let _ = client.request_keyframe();
log::debug!("decode: requested keyframe (loss recovery, dropped={dropped})"); log::debug!("decode: requested keyframe (loss recovery / overdue re-anchor)");
}
} }
} }
@@ -702,8 +723,10 @@ struct OutputReady {
/// internal looper thread) push the codec ones; the feeder thread pushes `Au`. Each carries only /// internal looper thread) push the codec ones; the feeder thread pushes `Au`. Each carries only
/// owned/`Copy` data so the callback closures satisfy the `Send` bound and never touch the codec. /// owned/`Copy` data so the callback closures satisfy the `Send` bound and never touch the codec.
enum DecodeEvent { enum DecodeEvent {
/// A received access unit from the feeder, ready to queue into the decoder. /// A received access unit from the feeder, ready to queue into the decoder. The `bool` is the
Au(Frame), /// feeder's [`NativeClient::note_frame_index`] verdict — `true` when this AU revealed a forward
/// frame-index gap, so the loop arms the freeze gate (the feeder already fired the RFI request).
Au(Frame, bool),
/// An input buffer slot freed (index) — we can queue an AU into it. /// An input buffer slot freed (index) — we can queue an AU into it.
InputAvailable(usize), InputAvailable(usize),
/// A decoded frame is ready (buffer index + echoed pts + the callback-time `decoded` stamp). /// A decoded frame is ready (buffer index + echoed pts + the callback-time `decoded` stamp).
@@ -889,7 +912,12 @@ fn run_async(
let mut discarded: u64 = 0; let mut discarded: u64 = 0;
// AUs larger than the codec input buffer, dropped whole (see `feed`/`feed_ready`). // AUs larger than the codec input buffer, dropped whole (see `feed`/`feed_ready`).
let mut oversized_dropped: u64 = 0; let mut oversized_dropped: u64 = 0;
let mut last_dropped = client.frames_dropped(); // Freeze-until-reanchor gate (see the sync loop for the rationale). Armed on a frame-index gap
// (the feeder's Au verdict), a parked-AU overflow drop, a dropped-count climb, or a recoverable
// codec error; `recovery_flags` carries each AU's user_flags from `dispatch_event` (feed) to
// `present_ready` (present), keyed by the codec-echoed pts.
let mut gate = ReanchorGate::new(client.frames_dropped());
let mut recovery_flags: VecDeque<(u64, u32)> = VecDeque::new();
let mut last_kf_req: Option<Instant> = None; let mut last_kf_req: Option<Instant> = None;
// Productive (dispatch+feed+present) time between displayed frames; reported to ADPF once one is // Productive (dispatch+feed+present) time between displayed frames; reported to ADPF once one is
// presented. The blocking event wait is excluded (idle, not work) — same accounting as the sync loop. // presented. The blocking event wait is excluded (idle, not work) — same accounting as the sync loop.
@@ -915,6 +943,8 @@ fn run_async(
&mut ready, &mut ready,
&mut fmt_dirty, &mut fmt_dirty,
&mut fatal, &mut fatal,
&mut gate,
&mut recovery_flags,
)); ));
} }
// Coalesce every other event already queued into this one work pass — correct newest-only // Coalesce every other event already queued into this one work pass — correct newest-only
@@ -927,6 +957,8 @@ fn run_async(
&mut ready, &mut ready,
&mut fmt_dirty, &mut fmt_dirty,
&mut fatal, &mut fatal,
&mut gate,
&mut recovery_flags,
)); ));
} }
stats.note_skipped(aus_dropped); // parked-AU overflow drops are client-side skips too stats.note_skipped(aus_dropped); // parked-AU overflow drops are client-side skips too
@@ -951,6 +983,8 @@ fn run_async(
&tracker, &tracker,
&mut rendered, &mut rendered,
&mut discarded, &mut discarded,
&mut gate,
&mut recovery_flags,
); );
work_accum_ns += work_t0.elapsed().as_nanos() as i64; work_accum_ns += work_t0.elapsed().as_nanos() as i64;
@@ -982,19 +1016,21 @@ fn run_async(
log::info!("decode: fed={fed} rendered={rendered} discarded={discarded}"); log::info!("decode: fed={fed} rendered={rendered} discarded={discarded}");
} }
} }
// Loss recovery: request an IDR when the reassembler's unrecoverable-drop count climbs (or we // Loss recovery + overdue backstop, folded through the gate. A parked-AU overflow drop is itself
// dropped a parked AU on overflow), throttled so a multi-frame recovery gap doesn't flood the // a loss, so it arms the freeze directly; the gate's `poll` then arms on a dropped-count climb
// control stream. // and re-asks on an overdue freeze. All keyframe intents route through the shared 100 ms
let dropped = client.frames_dropped(); // throttle so a multi-frame recovery gap can't flood the control stream.
if dropped > last_dropped || aus_dropped > 0 {
last_dropped = dropped;
let now = Instant::now(); let now = Instant::now();
if last_kf_req.is_none_or(|t| now.duration_since(t) >= Duration::from_millis(100)) { if aus_dropped > 0 {
gate.arm(now);
}
if (gate.poll(client.frames_dropped(), now) || aus_dropped > 0)
&& last_kf_req.is_none_or(|t| now.duration_since(t) >= Duration::from_millis(100))
{
last_kf_req = Some(now); last_kf_req = Some(now);
let _ = client.request_keyframe(); let _ = client.request_keyframe();
} }
} }
}
let _ = codec.stop(); let _ = codec.stop();
shutdown.store(true, Ordering::SeqCst); // ensure the feeder wakes and exits, then join it shutdown.store(true, Ordering::SeqCst); // ensure the feeder wakes and exits, then join it
@@ -1026,6 +1062,11 @@ fn feeder_loop(
while !shutdown.load(Ordering::Relaxed) { while !shutdown.load(Ordering::Relaxed) {
match client.next_frame(Duration::from_millis(5)) { match client.next_frame(Duration::from_millis(5)) {
Ok(frame) => { Ok(frame) => {
// Loss recovery (RFI): a forward frame-index gap fires a throttled reference-frame-
// invalidation request so an RFI-capable host recovers with a cheap clean P-frame
// instead of a full IDR (the frames_dropped keyframe path is the backstop). The gap
// verdict rides the Au event so the decode loop arms its freeze gate on the same signal.
let gap = client.note_frame_index(frame.frame_index);
if stats.enabled() { if stats.enabled() {
let received_ns = now_realtime_ns(); let received_ns = now_realtime_ns();
let clock_offset = clock_offset.load(Ordering::Relaxed) as i128; let clock_offset = clock_offset.load(Ordering::Relaxed) as i128;
@@ -1058,7 +1099,7 @@ fn feeder_loop(
} }
} }
} }
if ev_tx.send(DecodeEvent::Au(frame)).is_err() { if ev_tx.send(DecodeEvent::Au(frame, gap)).is_err() {
break; // the decode loop is gone break; // the decode loop is gone
} }
} }
@@ -1070,6 +1111,7 @@ fn feeder_loop(
/// Route one [`DecodeEvent`] into the loop's working sets. Returns `true` only when a parked AU was /// Route one [`DecodeEvent`] into the loop's working sets. Returns `true` only when a parked AU was
/// dropped on overflow (the caller then requests a keyframe). /// dropped on overflow (the caller then requests a keyframe).
#[allow(clippy::too_many_arguments)] // two call sites; the freeze gate + flag map are threaded in
fn dispatch_event( fn dispatch_event(
ev: DecodeEvent, ev: DecodeEvent,
pending_aus: &mut VecDeque<Frame>, pending_aus: &mut VecDeque<Frame>,
@@ -1077,9 +1119,20 @@ fn dispatch_event(
ready: &mut Vec<OutputReady>, ready: &mut Vec<OutputReady>,
fmt_dirty: &mut bool, fmt_dirty: &mut bool,
fatal: &mut bool, fatal: &mut bool,
gate: &mut ReanchorGate,
recovery_flags: &mut VecDeque<(u64, u32)>,
) -> bool { ) -> bool {
match ev { match ev {
DecodeEvent::Au(f) => { DecodeEvent::Au(f, gap) => {
// A forward frame-index gap arms the freeze; park this AU's flags for the present side to
// fold `on_decoded` (keyed by the pts the codec will echo).
if gap {
gate.arm(Instant::now());
}
recovery_flags.push_back((f.pts_ns / 1000, f.flags));
if recovery_flags.len() > IN_FLIGHT_CAP {
recovery_flags.pop_front();
}
pending_aus.push_back(f); pending_aus.push_back(f);
if pending_aus.len() > FRAME_PARK_CAP { if pending_aus.len() > FRAME_PARK_CAP {
pending_aus.pop_front(); // sustained overflow — drop oldest, signal a keyframe request pending_aus.pop_front(); // sustained overflow — drop oldest, signal a keyframe request
@@ -1100,6 +1153,10 @@ fn dispatch_event(
DecodeEvent::Error { fatal: f } => { DecodeEvent::Error { fatal: f } => {
if f { if f {
*fatal = true; *fatal = true;
} else {
// A recoverable/transient codec error is a decode hiccup on a broken reference chain —
// arm the freeze so the concealed output it recovers into is held off the screen.
gate.arm(Instant::now());
} }
} }
} }
@@ -1171,6 +1228,8 @@ fn present_ready(
tracker: &DisplayTracker, tracker: &DisplayTracker,
rendered: &mut u64, rendered: &mut u64,
discarded: &mut u64, discarded: &mut u64,
gate: &mut ReanchorGate,
recovery_flags: &mut VecDeque<(u64, u32)>,
) { ) {
if ready.is_empty() { if ready.is_empty() {
return; return;
@@ -1183,10 +1242,16 @@ fn present_ready(
note_decoded_pts(stats, &mut g, clock_offset, o.pts_us, o.decoded_ns); note_decoded_pts(stats, &mut g, clock_offset, o.pts_us, o.decoded_ns);
} }
} }
// Fold EVERY output through the gate in pts (== decode) order — even the ones newest-wins discards —
// so the two-mark re-anchor count stays correct; the newest's verdict decides whether it reaches
// glass (`false` = withheld concealment; the SurfaceView keeps the last rendered frame frozen on).
let now = Instant::now();
let last = ready.len() - 1; let last = ready.len() - 1;
let mut skipped: u64 = 0; let mut skipped: u64 = 0;
for (i, o) in ready.drain(..).enumerate() { for (i, o) in ready.drain(..).enumerate() {
let render = i == last; let flags = take_flags(recovery_flags, o.pts_us);
let present = gate.on_decoded(flags, false, now) == GateVerdict::Present;
let render = i == last && present;
match codec.release_output_buffer_by_index(o.index, render) { match codec.release_output_buffer_by_index(o.index, render) {
Ok(()) if render => { Ok(()) if render => {
*rendered += 1; *rendered += 1;
@@ -1206,7 +1271,7 @@ fn present_ready(
} }
} }
} }
stats.note_skipped(skipped); // HUD `skipped` counter (newest-wins drops); no-op while hidden stats.note_skipped(skipped); // HUD `skipped` counter (newest-wins + held-off drops); no-op hidden
} }
/// React to an output-format change by signalling the stream's HDR dataspace on the Surface (SDR /// React to an output-format change by signalling the stream's HDR dataspace on the Surface (SDR
@@ -1402,19 +1467,30 @@ fn drain(
in_flight: &mut VecDeque<(u64, i128)>, in_flight: &mut VecDeque<(u64, i128)>,
clock_offset: i64, clock_offset: i64,
tracker: &DisplayTracker, tracker: &DisplayTracker,
gate: &mut ReanchorGate,
recovery_flags: &mut VecDeque<(u64, u32)>,
) -> (u64, u64) { ) -> (u64, u64) {
// Newest ready buffer so far (presented after the loop) with its HUD metadata — // Newest ready buffer so far (presented after the loop) with its HUD metadata —
// `Some((pts_us, decoded_ns))` only while the HUD is visible (the stamp read is gated). // `Some((pts_us, decoded_ns))` only while the HUD is visible. `held_present` is the freeze gate's
// verdict for that newest buffer (`false` = a post-loss concealment to withhold).
let mut held: Option<(OutputBuffer<'_>, Option<(u64, i128)>)> = None; let mut held: Option<(OutputBuffer<'_>, Option<(u64, i128)>)> = None;
let mut held_present = true;
let mut discarded: u64 = 0; let mut discarded: u64 = 0;
let mut wait = first_wait; let mut wait = first_wait;
loop { loop {
match codec.dequeue_output_buffer(wait) { match codec.dequeue_output_buffer(wait) {
Ok(DequeuedOutputBufferInfoResult::Buffer(buf)) => { Ok(DequeuedOutputBufferInfoResult::Buffer(buf)) => {
wait = Duration::ZERO; // only the first dequeue may block // Only the first dequeue may block; later ones poll (wait == ZERO).
wait = Duration::ZERO;
// Fold every dequeued frame through the gate in pts (== decode) order — even the ones
// the newest-wins policy discards — so the two-mark re-anchor count stays correct; the
// verdict of the newest (last folded) buffer decides whether it reaches glass.
let pts_us = buf.info().presentation_time_us().max(0) as u64;
let flags = take_flags(recovery_flags, pts_us);
held_present =
gate.on_decoded(flags, false, Instant::now()) == GateVerdict::Present;
let meta = if stats.enabled() { let meta = if stats.enabled() {
// The dequeue IS the sync loop's decoded-availability instant. // The dequeue IS the sync loop's decoded-availability instant.
let pts_us = buf.info().presentation_time_us().max(0) as u64;
let decoded_ns = now_realtime_ns(); let decoded_ns = now_realtime_ns();
note_decoded_pts(stats, in_flight, clock_offset, pts_us, decoded_ns); note_decoded_pts(stats, in_flight, clock_offset, pts_us, decoded_ns);
Some((pts_us, decoded_ns)) Some((pts_us, decoded_ns))
@@ -1460,16 +1536,19 @@ fn drain(
} }
} }
} }
// Present the newest ready frame, if any, and park its metadata for the render callback. // Present the newest ready frame — UNLESS the gate is withholding it as a post-loss concealment,
// in which case release it without rendering (the SurfaceView keeps the last rendered frame frozen
// on glass) and count it as a discard rather than a display.
let mut rendered = 0; let mut rendered = 0;
if let Some((buf, meta)) = held { if let Some((buf, meta)) = held {
match codec.release_output_buffer(buf, true) { match codec.release_output_buffer(buf, held_present) {
Ok(()) => { Ok(()) if held_present => {
rendered = 1; rendered = 1;
if let Some((pts_us, decoded_ns)) = meta { if let Some((pts_us, decoded_ns)) = meta {
tracker.note_rendered(pts_us, decoded_ns); tracker.note_rendered(pts_us, decoded_ns);
} }
} }
Ok(()) => discarded += 1, // held off the screen — awaiting a clean re-anchor
Err(e) => log::warn!("decode: release_output_buffer: {e}"), Err(e) => log::warn!("decode: release_output_buffer: {e}"),
} }
} }
@@ -1511,6 +1590,25 @@ fn note_decoded_pts(
stats.note_decoded(e2e_us, decode_us); stats.note_decoded(e2e_us, decode_us);
} }
/// The AU `user_flags` for a decoded output, keyed by the echoed `presentationTimeUs`. Recovery
/// signalling (FLAG_SOF IDR marker / RECOVERY_ANCHOR / RECOVERY_POINT) rides the AU's flags, which are
/// only in scope at feed time — so the feed side parks `(pts_us, flags)` here and the present side
/// looks them up to fold [`ReanchorGate::on_decoded`]. Decode order == input order (low-latency, no
/// B-frames), so this evicts entries older than `pts_us` as it goes; a miss (probe filler, or an entry
/// aged past the cap) reads `0` — no recovery flags, decoded normally.
fn take_flags(map: &mut VecDeque<(u64, u32)>, pts_us: u64) -> u32 {
while let Some(&(p, f)) = map.front() {
if p > pts_us {
break; // future frame — leave it for its own output buffer
}
map.pop_front();
if p == pts_us {
return f;
}
}
0
}
/// Map the decoder's reported output colour to a BT.2020 HDR dataspace, or `None` for SDR. The /// Map the decoder's reported output colour to a BT.2020 HDR dataspace, or `None` for SDR. The
/// integer values are the Android MediaFormat colour constants the NDK shares: COLOR_TRANSFER /// integer values are the Android MediaFormat colour constants the NDK shares: COLOR_TRANSFER
/// ST2084 = 6 (PQ/HDR10), HLG = 7; COLOR_RANGE FULL = 1, LIMITED = 2 (the host encodes limited). /// ST2084 = 6 (PQ/HDR10), HLG = 7; COLOR_RANGE FULL = 1, LIMITED = 2 (the host encodes limited).
+42 -30
View File
@@ -24,12 +24,13 @@ const TAG_PLAYER_LEDS: u8 = 0x02;
const TAG_TRIGGER: u8 = 0x03; const TAG_TRIGGER: u8 = 0x03;
/// `NativeBridge.nativeNextRumble(handle): Long` — block up to ~100 ms for the next rumble update. /// `NativeBridge.nativeNextRumble(handle): Long` — block up to ~100 ms for the next rumble update.
/// Returns a packed positive long: bit 48 = "has a v2 lease", bits 32..47 = `ttl_ms`, bits 16..31 = /// Returns a packed positive long: bits 49..52 = wire `pad` index (0..15), bit 48 = "has a v2 lease",
/// `low`, bits 0..15 = `high` (`low`/`high` 0..=0xFFFF, `0/0` = stop). The lease flag is /// bits 32..47 = `ttl_ms`, bits 16..31 = `low`, bits 0..15 = `high` (`low`/`high` 0..=0xFFFF, `0/0` =
/// out-of-band so ANY 16-bit `ttl_ms` — including 0xFFFF — is unambiguous (no in-band sentinel to /// stop). The lease flag is out-of-band so ANY 16-bit `ttl_ms` — including 0xFFFF — is unambiguous (no
/// collide with a real 65535 ms lease). No lease (legacy host) → bit 48 clear, and Kotlin falls /// in-band sentinel to collide with a real 65535 ms lease). No lease (legacy host) → bit 48 clear, and
/// back to its long one-shot. `-1` on timeout / session closed (all packed values are positive, so /// Kotlin falls back to its long one-shot. `-1` on timeout / session closed (all packed values are
/// `-1` stays unambiguous). Pad index is dropped (single-pad model). Run from a Kotlin poll thread. /// positive, so `-1` stays unambiguous). Kotlin routes the update back to the controller holding that
/// wire `pad` index (multi-pad rumble). Run from a Kotlin poll thread.
#[no_mangle] #[no_mangle]
pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeNextRumble( pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeNextRumble(
_env: JNIEnv, _env: JNIEnv,
@@ -46,14 +47,19 @@ pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeNextRumble(
// threads (and joins them — unbounded) before nativeClose frees the handle. // threads (and joins them — unbounded) before nativeClose frees the handle.
let h = unsafe { &*(handle as *const SessionHandle) }; let h = unsafe { &*(handle as *const SessionHandle) };
match h.client.next_rumble_ttl(PULL_TIMEOUT) { match h.client.next_rumble_ttl(PULL_TIMEOUT) {
Ok((_pad, low, high, ttl)) => { Ok((pad, low, high, ttl)) => {
// The reorder gate already ran in the core, so this update is fresh. Encode the // The reorder gate already ran in the core, so this update is fresh. Encode the
// Option out-of-band: a real lease sets bit 48 and carries ttl_ms verbatim. // Option out-of-band: a real lease sets bit 48 and carries ttl_ms verbatim. The pad
// index rides above the lease flag (bits 49..52), keeping the whole word positive.
let (lease_flag, ttl_bits) = match ttl { let (lease_flag, ttl_bits) = match ttl {
Some(ms) => (1i64 << 48, jlong::from(ms) << 32), Some(ms) => (1i64 << 48, jlong::from(ms) << 32),
None => (0, 0), None => (0, 0),
}; };
lease_flag | ttl_bits | (jlong::from(low) << 16) | jlong::from(high) (jlong::from(pad & 0xF) << 49)
| lease_flag
| ttl_bits
| (jlong::from(low) << 16)
| jlong::from(high)
} }
Err(_) => -1, // NoFrame (timeout) or Closed — Kotlin loops on its running flag Err(_) => -1, // NoFrame (timeout) or Closed — Kotlin loops on its running flag
} }
@@ -61,10 +67,12 @@ pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeNextRumble(
} }
/// `NativeBridge.nativeNextHidout(handle, buf): Int` — block up to ~100 ms for the next DualSense /// `NativeBridge.nativeNextHidout(handle, buf): Int` — block up to ~100 ms for the next DualSense
/// HID-output event, written into the caller's direct ByteBuffer as `[kind][fields…]`: /// HID-output event, written into the caller's direct ByteBuffer as `[pad][kind][fields…]` (the
/// Led → `[0x01][r][g][b]` (len 4) /// leading `pad` is the wire pad index the event is addressed to, so Kotlin routes it to that
/// PlayerLeds → `[0x02][bits]` (len 2) /// controller — multi-pad HID feedback):
/// Trigger → `[0x03][which][effect…]` (len 2 + effect.len()) /// Led → `[pad][0x01][r][g][b]` (len 5)
/// PlayerLeds → `[pad][0x02][bits]` (len 3)
/// Trigger → `[pad][0x03][which][effect…]` (len 3 + effect.len())
/// Returns the byte count written, or `-1` on timeout / session closed / buffer too small. /// Returns the byte count written, or `-1` on timeout / session closed / buffer too small.
#[no_mangle] #[no_mangle]
pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeNextHidout( pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeNextHidout(
@@ -97,33 +105,37 @@ pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeNextHidout(
// SAFETY: `ptr`/`cap` describe the direct ByteBuffer's backing store, valid for this call. // SAFETY: `ptr`/`cap` describe the direct ByteBuffer's backing store, valid for this call.
let out = unsafe { std::slice::from_raw_parts_mut(ptr, cap) }; let out = unsafe { std::slice::from_raw_parts_mut(ptr, cap) };
// out[0] = wire pad index; out[1] = kind tag; the rest is the per-kind payload.
let n = match ev { let n = match ev {
HidOutput::Led { r, g, b, .. } => { HidOutput::Led { pad, r, g, b } => {
if cap < 4 { if cap < 5 {
return -1; return -1;
} }
out[0] = TAG_LED; out[0] = pad;
out[1] = r; out[1] = TAG_LED;
out[2] = g; out[2] = r;
out[3] = b; out[3] = g;
4 out[4] = b;
5
} }
HidOutput::PlayerLeds { bits, .. } => { HidOutput::PlayerLeds { pad, bits } => {
if cap < 2 { if cap < 3 {
return -1; return -1;
} }
out[0] = TAG_PLAYER_LEDS; out[0] = pad;
out[1] = bits; out[1] = TAG_PLAYER_LEDS;
2 out[2] = bits;
3
} }
HidOutput::Trigger { which, effect, .. } => { HidOutput::Trigger { pad, which, effect } => {
let n = 2 + effect.len(); let n = 3 + effect.len();
if cap < n { if cap < n {
return -1; // the raw DS5 trigger block is ~11 bytes; Kotlin allocates 64 return -1; // the raw DS5 trigger block is ~11 bytes; Kotlin allocates 64
} }
out[0] = TAG_TRIGGER; out[0] = pad;
out[1] = which; out[1] = TAG_TRIGGER;
out[2..n].copy_from_slice(&effect); out[2] = which;
out[3..n].copy_from_slice(&effect);
n n
} }
HidOutput::TrackpadHaptic { .. } => { HidOutput::TrackpadHaptic { .. } => {
+64 -13
View File
@@ -145,13 +145,19 @@ pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeSendKey(
} }
// ---- Gamepad: Kotlin captures (KeyEvent/MotionEvent) → NativeClient::send_input --------------- // ---- Gamepad: Kotlin captures (KeyEvent/MotionEvent) → NativeClient::send_input ---------------
// Single-pad model: exactly one controller, forwarded as pad 0 (flags = 0). Buttons carry the // Multi-pad model: each physical controller is forwarded on its own wire pad index (0..15), carried
// gamepad::BTN_* bit in `code` and pressed/released in `x` (1/0); axes carry the gamepad::AXIS_* id // in the low byte of `flags` on every per-pad event — the Kotlin side (`GamepadRouter`) assigns a
// in `code` and the value in `x` (sticks i16 32768..32767, +y = up; triggers 0..255). The host // stable lowest-free index per Android device and threads it here. Buttons carry the gamepad::BTN_*
// accumulates the incremental events into its virtual xpad. Wire contract: input.rs::gamepad. // bit in `code` and pressed/released in `x` (1/0); axes carry the gamepad::AXIS_* id in `code` and
// the value in `x` (sticks i16 32768..32767, +y = up; triggers 0..255). The host accumulates the
// incremental events per pad into a matching virtual device. The core input task folds these into
// the seq'd GamepadState snapshots (keyed on this same `flags` index) and owns the per-pad seq — so
// the only thing this layer must get right is the index. Wire contract: input.rs::gamepad. A single
// controller lands on index 0, so its wire is byte-identical to the old single-pad path.
/// `NativeBridge.nativeSendGamepadButton(handle, bit, down)` — one gamepad button transition. /// `NativeBridge.nativeSendGamepadButton(handle, bit, down, pad)` — one gamepad button transition on
/// `bit`: a `gamepad::BTN_*` bit (e.g. BTN_A = 0x1000). `down`: 1=press, 0=release. /// wire pad index `pad`. `bit`: a `gamepad::BTN_*` bit (e.g. BTN_A = 0x1000). `down`: 1=press,
/// 0=release. `pad`: wire pad index 0..15 (rides `flags`).
#[no_mangle] #[no_mangle]
pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeSendGamepadButton( pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeSendGamepadButton(
_env: JNIEnv, _env: JNIEnv,
@@ -159,21 +165,21 @@ pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeSendGamepad
handle: jlong, handle: jlong,
bit: jint, bit: jint,
down: jboolean, down: jboolean,
pad: jint,
) { ) {
// flags = 0: pad index 0 — single-pad model.
send_event( send_event(
handle, handle,
InputKind::GamepadButton, InputKind::GamepadButton,
bit as u32, bit as u32,
i32::from(down != 0), i32::from(down != 0),
0, 0,
0, pad as u32,
); );
} }
/// `NativeBridge.nativeSendGamepadAxis(handle, axisId, value)` — one gamepad axis update. /// `NativeBridge.nativeSendGamepadAxis(handle, axisId, value, pad)` — one gamepad axis update on wire
/// `axisId`: a `gamepad::AXIS_*` id (LS_X=0..RT=5). `value`: stick i16 (32768..32767, +y=up) or /// pad index `pad`. `axisId`: a `gamepad::AXIS_*` id (LS_X=0..RT=5). `value`: stick i16
/// trigger 0..255. /// (32768..32767, +y=up) or trigger 0..255. `pad`: wire pad index 0..15 (rides `flags`).
#[no_mangle] #[no_mangle]
pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeSendGamepadAxis( pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeSendGamepadAxis(
_env: JNIEnv, _env: JNIEnv,
@@ -181,7 +187,52 @@ pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeSendGamepad
handle: jlong, handle: jlong,
axis_id: jint, axis_id: jint,
value: jint, value: jint,
pad: jint,
) { ) {
// flags = 0: pad index 0 — single-pad model. send_event(
send_event(handle, InputKind::GamepadAxis, axis_id as u32, value, 0, 0); handle,
InputKind::GamepadAxis,
axis_id as u32,
value,
0,
pad as u32,
);
}
/// `NativeBridge.nativeSendGamepadArrival(handle, pref, pad)` — declare the controller KIND presented
/// on wire pad index `pad` so the host builds a matching virtual device (mixed types — pad 0 a
/// DualSense, pad 1 an Xbox pad). `pref`: the `GamepadPref` wire byte (rides `code`). `pad`: wire pad
/// index 0..15 (rides `flags`). Sent ONCE when a pad opens, BEFORE any of its input; the core re-sends
/// it a few times against datagram loss, and an older host ignores the unknown tag (that pad then uses
/// the session-default kind from the handshake — the pre-existing single-pad behaviour on pad 0).
#[no_mangle]
pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeSendGamepadArrival(
_env: JNIEnv,
_this: JObject,
handle: jlong,
pref: jint,
pad: jint,
) {
send_event(
handle,
InputKind::GamepadArrival,
pref as u32,
0,
0,
pad as u32,
);
}
/// `NativeBridge.nativeSendGamepadRemove(handle, pad)` — signal that wire pad index `pad` was
/// unplugged so the host tears its virtual device down. `pad` (rides `flags`) is the only field; the
/// core stamps the per-pad seq (in the snapshot seq space, so a reordered snapshot can't resurrect the
/// pad) and arms a re-send burst against datagram loss. An older host ignores the unknown tag.
#[no_mangle]
pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeSendGamepadRemove(
_env: JNIEnv,
_this: JObject,
handle: jlong,
pad: jint,
) {
send_event(handle, InputKind::GamepadRemove, 0, 0, 0, pad as u32);
} }
+13 -15
View File
@@ -2,24 +2,22 @@
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"> <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0"> <plist version="1.0">
<dict> <dict>
<!-- Custom keys merged into the auto-generated Info.plist (GENERATE_INFOPLIST_FILE=YES <key>CADisableMinimumFrameDurationOnPhone</key>
supplies the rest). NSBonjourServices is required for NWBrowser to browse this <true/>
service type on iOS/tvOS — without it the system blocks the browse and discovery <key>GCSupportedGameControllers</key>
returns nothing. Kept OUT of the synchronized App/ + Sources/ groups so it isn't <array>
auto-added as a bundle resource (which collides with Info.plist processing). --> <dict>
<key>ProfileName</key>
<string>ExtendedGamepad</string>
</dict>
<dict>
<key>ProfileName</key>
<string>MicroGamepad</string>
</dict>
</array>
<key>NSBonjourServices</key> <key>NSBonjourServices</key>
<array> <array>
<string>_punktfunk._udp</string> <string>_punktfunk._udp</string>
</array> </array>
<!-- Standard-algorithm crypto only (AES-GCM via the Rust core) — exempt from export
compliance, but the key must be declared or every TestFlight build stalls on the
compliance question. -->
<key>ITSAppUsesNonExemptEncryption</key>
<false/>
<!-- Allow CADisplayLink above 60 Hz on ProMotion iPhones: without this key the system
silently caps the link at 60 even when SessionPresenter asks for the stream's rate
via preferredFrameRateRange, so a 120 fps stream would present at half rate. -->
<key>CADisableMinimumFrameDurationOnPhone</key>
<true/>
</dict> </dict>
</plist> </plist>
+4 -12
View File
@@ -14,19 +14,11 @@
<!-- Wake-on-LAN needs to send a UDP broadcast magic packet (a sleeping host has no ARP <!-- Wake-on-LAN needs to send a UDP broadcast magic packet (a sleeping host has no ARP
entry, so unicast can't wake it). Since iOS 14 / tvOS 14 the OS blocks sending to entry, so unicast can't wake it). Since iOS 14 / tvOS 14 the OS blocks sending to
broadcast/multicast addresses unless the app carries this managed entitlement — it must broadcast/multicast addresses unless the app carries this managed entitlement — it must
be requested from and approved by Apple for the App ID, then enabled in the provisioning be approved by Apple for the App ID and enabled in the provisioning profile. macOS is not
profile. macOS is not gated by this (its App Sandbox network.client/server cover it). gated by this (its App Sandbox network.client/server cover it), hence its separate file.
Approved and provisioned, so it's enabled here and PunktfunkConnection.wakeOnLANAvailable
GATED pending Apple's approval of the request (form filed) — an unauthorized managed is true on iOS/tvOS too. -->
entitlement breaks iOS/tvOS signing, so it's commented out to keep those apps releasable.
ON APPROVAL: (1) uncomment the two lines below, and (2) flip
PunktfunkConnection.wakeOnLANAvailable (PunktfunkConnection.swift) to enable the iOS/tvOS
wake path + UI. Until then iOS/tvOS Wake-on-LAN is a clean no-op — MACs are still learned
from mDNS so it works immediately once ungated. macOS is unaffected (separate entitlements
file, no multicast entitlement needed). -->
<!--
<key>com.apple.developer.networking.multicast</key> <key>com.apple.developer.networking.multicast</key>
<true/> <true/>
-->
</dict> </dict>
</plist> </plist>
+3 -1
View File
@@ -40,6 +40,8 @@ let package = Package(
// its manifest breaks SwiftPM whole-graph validation on macOS, and only the // its manifest breaks SwiftPM whole-graph validation on macOS, and only the
// Punktfunk-tvOS target links it; the #if os(tvOS) import never compiles here.) // Punktfunk-tvOS target links it; the #if os(tvOS) import never compiles here.)
.executableTarget(name: "PunktfunkClient", dependencies: ["PunktfunkKit"]), .executableTarget(name: "PunktfunkClient", dependencies: ["PunktfunkKit"]),
.testTarget(name: "PunktfunkKitTests", dependencies: ["PunktfunkKit"]), // PunktfunkCore is a direct dep too so the wire tests can name the C ABI's
// `PunktfunkInputEvent` / `PUNKTFUNK_INPUT_KIND_*` when asserting the gamepad byte layout.
.testTarget(name: "PunktfunkKitTests", dependencies: ["PunktfunkKit", "PunktfunkCore"]),
] ]
) )
@@ -436,6 +436,7 @@
INFOPLIST_KEY_CFBundleDisplayName = Punktfunk; INFOPLIST_KEY_CFBundleDisplayName = Punktfunk;
INFOPLIST_KEY_GCSupportsControllerUserInteraction = YES; INFOPLIST_KEY_GCSupportsControllerUserInteraction = YES;
INFOPLIST_KEY_GCSupportsGameMode = YES; INFOPLIST_KEY_GCSupportsGameMode = YES;
INFOPLIST_KEY_ITSAppUsesNonExemptEncryption = NO;
INFOPLIST_KEY_LSApplicationCategoryType = "public.app-category.games"; INFOPLIST_KEY_LSApplicationCategoryType = "public.app-category.games";
INFOPLIST_KEY_NSLocalNetworkUsageDescription = "Punktfunk connects directly to your punktfunk host on the local network to stream video, audio, and input."; INFOPLIST_KEY_NSLocalNetworkUsageDescription = "Punktfunk connects directly to your punktfunk host on the local network to stream video, audio, and input.";
INFOPLIST_KEY_NSMicrophoneUsageDescription = "Your microphone is streamed to the connected punktfunk host, where it appears as a virtual microphone."; INFOPLIST_KEY_NSMicrophoneUsageDescription = "Your microphone is streamed to the connected punktfunk host, where it appears as a virtual microphone.";
@@ -477,6 +478,7 @@
INFOPLIST_KEY_CFBundleDisplayName = Punktfunk; INFOPLIST_KEY_CFBundleDisplayName = Punktfunk;
INFOPLIST_KEY_GCSupportsControllerUserInteraction = YES; INFOPLIST_KEY_GCSupportsControllerUserInteraction = YES;
INFOPLIST_KEY_GCSupportsGameMode = YES; INFOPLIST_KEY_GCSupportsGameMode = YES;
INFOPLIST_KEY_ITSAppUsesNonExemptEncryption = NO;
INFOPLIST_KEY_LSApplicationCategoryType = "public.app-category.games"; INFOPLIST_KEY_LSApplicationCategoryType = "public.app-category.games";
INFOPLIST_KEY_NSLocalNetworkUsageDescription = "Punktfunk connects directly to your punktfunk host on the local network to stream video, audio, and input."; INFOPLIST_KEY_NSLocalNetworkUsageDescription = "Punktfunk connects directly to your punktfunk host on the local network to stream video, audio, and input.";
INFOPLIST_KEY_NSMicrophoneUsageDescription = "Your microphone is streamed to the connected punktfunk host, where it appears as a virtual microphone."; INFOPLIST_KEY_NSMicrophoneUsageDescription = "Your microphone is streamed to the connected punktfunk host, where it appears as a virtual microphone.";
@@ -49,6 +49,13 @@
ReferencedContainer = "container:Punktfunk.xcodeproj"> ReferencedContainer = "container:Punktfunk.xcodeproj">
</BuildableReference> </BuildableReference>
</BuildableProductRunnable> </BuildableProductRunnable>
<EnvironmentVariables>
<EnvironmentVariable
key = "PUNKTFUNK_BILINEAR_LUMA"
value = "1"
isEnabled = "YES">
</EnvironmentVariable>
</EnvironmentVariables>
</LaunchAction> </LaunchAction>
<ProfileAction <ProfileAction
buildConfiguration = "Release" buildConfiguration = "Release"
@@ -60,7 +60,8 @@ struct ContentView: View {
@State private var speedTestTarget: StoredHost? @State private var speedTestTarget: StoredHost?
@State private var libraryTarget: StoredHost? @State private var libraryTarget: StoredHost?
/// Wakes a sleeping host and waits for it to come back online before connecting (drives the /// Wakes a sleeping host and waits for it to come back online before connecting (drives the
/// "Waking" overlay). macOS-only in practice WoL is gated off on iOS/tvOS. /// "Waking" phase of the connect overlay). Available on every platform now that the iOS/tvOS
/// multicast entitlement is granted (see PunktfunkConnection.wakeOnLANAvailable).
@StateObject private var waker = HostWaker() @StateObject private var waker = HostWaker()
#if os(macOS) #if os(macOS)
/// Whether the hosting window is native-fullscreen right now (reported by /// Whether the hosting window is native-fullscreen right now (reported by
@@ -86,6 +87,10 @@ struct ContentView: View {
// with no (extended) controller attached tvOS falls back to HomeView as before. // with no (extended) controller attached tvOS falls back to HomeView as before.
@ObservedObject private var gamepadManager = GamepadManager.shared @ObservedObject private var gamepadManager = GamepadManager.shared
@AppStorage(DefaultsKey.gamepadUIEnabled) private var gamepadUIEnabled = true @AppStorage(DefaultsKey.gamepadUIEnabled) private var gamepadUIEnabled = true
/// Auto-wake on connect (Settings General). On (default): a dial to an offline saved host
/// fires Wake-on-LAN up front and falls into the "Waking" wait if the dial fails. Off: connects
/// go straight through with no wake. The explicit "Wake Host" action is unaffected either way.
@AppStorage(DefaultsKey.autoWake) private var autoWakeEnabled = true
private var gamepadUIActive: Bool { private var gamepadUIActive: Bool {
GamepadUIEnvironment.isActive( GamepadUIEnvironment.isActive(
gamepadConnected: gamepadManager.active != nil, enabledSetting: gamepadUIEnabled) gamepadConnected: gamepadManager.active != nil, enabledSetting: gamepadUIEnabled)
@@ -259,9 +264,26 @@ struct ContentView: View {
} }
private var home: some View { private var home: some View {
// The "Waking" overlay rides over BOTH home UIs (and the pre-connect window is still // The full-screen connect takeover rides over BOTH home UIs (and the pre-connect window is
// `home`, so it covers the whole wakeonlineconnect sequence). // still `home`, so it covers the whole dial wake online connect sequence): instant
homeBase.overlay { WakeOverlay(waker: waker) } // "Connecting" feedback on any dial, flowing seamlessly into the "Waking" wait if the host
// turns out to be asleep.
homeBase.overlay {
ConnectOverlay(
connectingHostName: connectingOverlayName,
waker: waker,
gamepadUI: gamepadUIActive,
onCancelConnect: { model.disconnect() })
}
}
/// The host label for the connect takeover's "Connecting" phase a plain dial in flight. Nil
/// during the delegated-approval wait (that has its own "Waiting for approval" prompt, so the
/// takeover must not stack over it) and, of course, when idle or streaming.
private var connectingOverlayName: String? {
guard awaitingApproval == nil, model.phase == .connecting, let host = model.activeHost
else { return nil }
return host.displayName
} }
@ViewBuilder private var homeBase: some View { @ViewBuilder private var homeBase: some View {
@@ -327,12 +349,25 @@ struct ContentView: View {
}() }()
return ZStack { return ZStack {
stream(captureEnabled: pendingFingerprint == nil) stream(captureEnabled: pendingFingerprint == nil)
.blur(radius: pendingFingerprint != nil ? 32 : 0) // Blur the live stream during the trust prompt (heavy) and during a resize (lighter
// the deliberate "hold on" while the host rebuilds its pipeline and the decoder
// re-inits on the new-mode IDR). Only the resize blur animates; the trust blur snaps
// as before (its own overlay handles the transition).
.blur(radius: pendingFingerprint != nil ? 32 : (model.resizing ? 16 : 0))
.animation(.easeInOut(duration: 0.22), value: model.resizing)
.overlay { .overlay {
if pendingFingerprint != nil { if pendingFingerprint != nil {
Color.black.opacity(0.45) Color.black.opacity(0.45)
} }
} }
// The resize spinner rides over the (blurred) stream; suppressed under the trust
// prompt, which owns the screen. It never hit-tests, so window-drag resizes keep
// steering and the next click still reaches the stream.
.overlay {
if pendingFingerprint == nil {
ResizeIndicatorView(active: model.resizing)
}
}
if let fp = pendingFingerprint { if let fp = pendingFingerprint {
TrustCardView( TrustCardView(
fingerprint: fp, fingerprint: fp,
@@ -410,6 +445,16 @@ struct ContentView: View {
onSessionEnd: { [weak model] in onSessionEnd: { [weak model] in
Task { @MainActor in model?.sessionEnded() } Task { @MainActor in model?.sessionEnded() }
}, },
// Resize overlay START the follower is main-actor, so this drives the blur
// + spinner synchronously the instant the window differs from the live mode.
onResizeTarget: { [weak model] w, h in
model?.resizeTargeted(width: w, height: h)
},
// Resize overlay END the coded dims of each new-mode IDR, reported from the
// decode pump thread; hop to the main actor to clear the overlay.
onDecodedSize: { [weak model] w, h in
Task { @MainActor in model?.resizeDecoded(width: w, height: h) }
},
endToEndMeter: model.endToEnd, endToEndMeter: model.endToEnd,
decodeMeter: model.decodeStage, decodeMeter: model.decodeStage,
displayMeter: model.displayStage displayMeter: model.displayStage
@@ -544,7 +589,8 @@ struct ContentView: View {
// packet up front, so a genuinely-asleep host is waking while the connect times out; only // packet up front, so a genuinely-asleep host is waking while the connect times out; only
// when that dial FAILS do we fall into the visible "Waking" wait a cold box takes far // when that dial FAILS do we fall into the visible "Waking" wait a cold box takes far
// longer to boot than a connect will sit and redial once it's back on mDNS. // longer to boot than a connect will sit and redial once it's back on mDNS.
if PunktfunkConnection.wakeOnLANAvailable, !host.wakeMacs.isEmpty, !discovery.advertises(host) { if autoWakeEnabled, PunktfunkConnection.wakeOnLANAvailable,
!host.wakeMacs.isEmpty, !discovery.advertises(host) {
discovery.start() // so the wake-wait can observe it reappear discovery.start() // so the wake-wait can observe it reappear
startSessionDirect( startSessionDirect(
host, launchID: launchID, allowTofu: allowTofu, host, launchID: launchID, allowTofu: allowTofu,
@@ -601,7 +647,9 @@ struct ContentView: View {
private func prepareWake(for host: StoredHost) { private func prepareWake(for host: StoredHost) {
if let live = discovery.hosts.first(where: { host.matches($0) }) { if let live = discovery.hosts.first(where: { host.matches($0) }) {
store.updateMacs(host.id, macs: live.macAddresses) // learn on every platform store.updateMacs(host.id, macs: live.macAddresses) // learn on every platform
} else if PunktfunkConnection.wakeOnLANAvailable, !host.wakeMacs.isEmpty { } else if autoWakeEnabled, PunktfunkConnection.wakeOnLANAvailable, !host.wakeMacs.isEmpty {
// Auto-wake only: fire the up-front packet so a genuinely-asleep host is booting while the
// dial times out. With auto-wake off, connects go straight through (no packet).
let macs = host.wakeMacs let macs = host.wakeMacs
let ip = host.address let ip = host.address
DispatchQueue.global(qos: .userInitiated).async { DispatchQueue.global(qos: .userInitiated).async {
@@ -0,0 +1,145 @@
// The unified "getting you connected" overlay one look for BOTH phases of reaching a host, so the
// user gets feedback the instant they pick one and it flows seamlessly into a wake if the host turns
// out to be asleep. The Apple mirror of the Android client's `ConnectOverlay` and the shared console
// UI's connect/wake takeover; it replaces the old centered-card `WakeOverlay`.
//
// - Connecting (`connectingHostName` non-nil): the dial is in flight. Shown immediately on activate
// so a host that takes a beat to answer no longer looks like nothing happened.
// - Waking (`waker.waking` non-nil): the dial failed on a sleeping host, so we're firing
// Wake-on-LAN and waiting for it to advertise again, escalating to a retry/cancel prompt on
// timeout.
//
// Presentation is mode-aware: the gamepad ("console") UI gets a full-screen aurora takeover the
// same living backdrop the console home wears, so it reads as a deliberate 10-foot moment; the
// default touch/desktop UI gets a Liquid Glass modal over a dim scrim, which sits right at home among
// the app's other floating surfaces (the trust card, the HUD) instead of a full-screen aurora that
// looked out of place there.
//
// The two phases hand off within a single view update (HostWaker clears `waking` and starts the
// connect in the same MainActor step), so the overlay never blinks between them. It swallows input to
// the screen behind it, and on iOS/macOS the pad drives it (B cancels, A retries once timed out).
import PunktfunkKit
import SwiftUI
struct ConnectOverlay: View {
/// Non-nil while a plain dial is in flight (the delegated-approval wait has its own prompt, so it
/// passes nil here). Drives the "Connecting" phase.
let connectingHostName: String?
@ObservedObject var waker: HostWaker
/// The console launcher is up full-screen aurora takeover; otherwise the default UI's Liquid
/// Glass modal.
var gamepadUI: Bool
/// Cancel a dial in flight tears down the (uncancelable) connect and returns the UI; the late
/// result is discarded by SessionModel's connect guard.
var onCancelConnect: () -> Void
private enum Phase {
case connecting(name: String)
case waking(HostWaker.Waking)
}
/// Waking takes precedence it only ever exists after a dial has already failed, so a stray
/// overlap can't strand the "Connecting" phase over a wake in progress.
private var phase: Phase? {
if let w = waker.waking { return .waking(w) }
if let name = connectingHostName { return .connecting(name: name) }
return nil
}
var body: some View {
if let phase {
ZStack {
if gamepadUI {
// Console: an opaque, living aurora over everything.
Color.black.ignoresSafeArea()
GamepadScreenBackground().ignoresSafeArea()
Color.clear.contentShape(Rectangle()).onTapGesture {}
content(phase).padding(40).frame(maxWidth: 460)
} else {
// Default UI: a Liquid Glass modal over a dim scrim.
Rectangle().fill(.black.opacity(0.5)).ignoresSafeArea()
.contentShape(Rectangle()).onTapGesture {}
content(phase)
.padding(28)
.frame(maxWidth: 380)
.glassBackground(RoundedRectangle(cornerRadius: 26, style: .continuous))
.overlay(
RoundedRectangle(cornerRadius: 26, style: .continuous)
.strokeBorder(.white.opacity(0.12), lineWidth: 1))
.padding(40)
}
}
.environment(\.colorScheme, .dark)
.transition(.opacity)
#if os(iOS) || os(macOS)
.background { ConnectControllerInput(waker: waker, onCancelConnect: onCancelConnect) }
#endif
}
}
@ViewBuilder private func content(_ phase: Phase) -> some View {
// The takeover carries larger type than the compact modal.
let titleSize: CGFloat = gamepadUI ? 24 : 19
let bodySize: CGFloat = gamepadUI ? 14 : 13
VStack(spacing: gamepadUI ? 16 : 14) {
switch phase {
case .connecting(let name):
ProgressView().controlSize(.large).tint(.white)
Text("Connecting to \(name)")
.font(.geist(titleSize, .bold, relativeTo: .title3)).foregroundStyle(.white)
.multilineTextAlignment(.center)
Text("Establishing a secure connection…")
.font(.geist(bodySize, relativeTo: .caption)).foregroundStyle(.white.opacity(0.6))
Button("Cancel") { onCancelConnect() }.buttonStyle(.bordered).padding(.top, 6)
case .waking(let w) where w.timedOut:
Image(systemName: "moon.zzz.fill")
.font(.system(size: gamepadUI ? 40 : 34)).foregroundStyle(.white.opacity(0.9))
Text("\(w.hostName) didn't wake")
.font(.geist(titleSize, .bold, relativeTo: .title3)).foregroundStyle(.white)
.multilineTextAlignment(.center)
Text("It may still be booting, or it's powered off / off this network.")
.font(.geist(bodySize, relativeTo: .caption)).foregroundStyle(.white.opacity(0.6))
.multilineTextAlignment(.center)
HStack(spacing: 12) {
Button("Cancel") { waker.cancel() }.buttonStyle(.bordered)
Button("Try Again") { waker.retry() }.glassProminentButtonStyle()
}
.padding(.top, 6)
case .waking(let w):
ProgressView().controlSize(.large).tint(.white)
Text("Waking \(w.hostName)")
.font(.geist(titleSize, .bold, relativeTo: .title3)).foregroundStyle(.white)
.multilineTextAlignment(.center)
Text("Waiting for it to come online · \(w.seconds)s")
.font(.geistFixed(bodySize)).foregroundStyle(.white.opacity(0.6)).monospacedDigit()
// A wake-only wait (no dial after) offers "Stop Waiting"; a wake-&-connect is "Cancel".
Button(w.connectsAfter ? "Cancel" : "Stop Waiting") { waker.cancel() }
.buttonStyle(.bordered).padding(.top, 6)
}
}
}
}
#if os(iOS) || os(macOS)
/// Controller binding for the overlay: B cancels whatever's in flight (a dial or the wake wait); A
/// retries once a wake has timed out. The closures read the live state on each press, so they stay
/// correct across the Connecting Waking handoff without the view re-mounting. A zero-size backing
/// view owning a `GamepadMenuInput` for the overlay's lifetime (the home is gated inactive while the
/// overlay is up, so nothing else is consuming the pad).
private struct ConnectControllerInput: View {
@ObservedObject var waker: HostWaker
var onCancelConnect: () -> Void
@State private var input = GamepadMenuInput(manager: .shared)
var body: some View {
Color.clear
.onAppear {
input.onBack = { if waker.waking != nil { waker.cancel() } else { onCancelConnect() } }
input.onConfirm = { if waker.waking?.timedOut == true { waker.retry() } }
input.start()
}
.onDisappear { input.stop() }
}
}
#endif
@@ -65,6 +65,9 @@ struct GamepadHomeView: View {
/// Same gate the touch grid's "Browse Library" context-menu item uses (default ON; the /// Same gate the touch grid's "Browse Library" context-menu item uses (default ON; the
/// Settings "Game library" toggle opts out). /// Settings "Game library" toggle opts out).
@AppStorage(DefaultsKey.libraryEnabled) private var libraryEnabled = true @AppStorage(DefaultsKey.libraryEnabled) private var libraryEnabled = true
/// Auto-wake on connect (default ON) when off, activating an offline host just dials (no wake),
/// so the tile drops its "Wake & Connect" affordance for a plain "Connect".
@AppStorage(DefaultsKey.autoWake) private var autoWakeEnabled = true
#if os(iOS) #if os(iOS)
/// `.compact` in a landscape phone window drives tighter chrome so everything still fits. /// `.compact` in a landscape phone window drives tighter chrome so everything still fits.
@Environment(\.verticalSizeClass) private var vSizeClass @Environment(\.verticalSizeClass) private var vSizeClass
@@ -192,9 +195,12 @@ struct GamepadHomeView: View {
onActivate: { $0.activate() }, onActivate: { $0.activate() },
onSecondary: { openLibraryForSelected() }, onSecondary: { openLibraryForSelected() },
onTertiary: { showSettings = true }, onTertiary: { showSettings = true },
// Stop consuming the controller while another screen (or the wake overlay) is on top // Stop consuming the controller while another screen (or the connect/wake takeover) is on
// otherwise the launcher navigates behind it (invisibly on iPhone, visibly on iPad). // top otherwise the launcher navigates behind it (invisibly on iPhone, visibly on iPad),
isActive: libraryTarget == nil && !showSettings && !showAddHost && waker.waking == nil // and a second A during a dial would launch a concurrent connect. `.connecting` covers the
// takeover's Connecting phase; `waker.waking` covers its Waking phase.
isActive: libraryTarget == nil && !showSettings && !showAddHost
&& waker.waking == nil && model.phase != .connecting
) { tile in ) { tile in
hostCard(tile, size: CGSize(width: cardWidth, height: cardHeight)) hostCard(tile, size: CGSize(width: cardWidth, height: cardHeight))
} }
@@ -256,7 +262,7 @@ struct GamepadHomeView: View {
isConnecting: model.phase == .connecting && model.activeHost?.id == host.id, isConnecting: model.phase == .connecting && model.activeHost?.id == host.id,
filled: true, filled: true,
hasLibrary: true, hasLibrary: true,
canWake: PunktfunkConnection.wakeOnLANAvailable canWake: autoWakeEnabled && PunktfunkConnection.wakeOnLANAvailable
&& !discovery.advertises(host) && !store.probedOnline.contains(host.id) && !discovery.advertises(host) && !store.probedOnline.contains(host.id)
&& !host.wakeMacs.isEmpty, && !host.wakeMacs.isEmpty,
activate: { connect(host) }) activate: { connect(host) })
@@ -1,84 +0,0 @@
// The "Waking <host>" modal shown while HostWaker brings a sleeping host back a spinner + a
// live elapsed counter, escalating to a retry/cancel prompt on timeout. Presented over BOTH the
// touch and gamepad home (a wake only ever starts on macOS today, where WoL is ungated), and it
// drives from either a pointer (the buttons) or a controller (B cancels, A retries once timed out).
import PunktfunkKit
import SwiftUI
struct WakeOverlay: View {
@ObservedObject var waker: HostWaker
var body: some View {
if let w = waker.waking {
ZStack {
// Dim + swallow input to the home behind it.
Rectangle().fill(.black.opacity(0.6)).ignoresSafeArea()
.contentShape(Rectangle())
.onTapGesture {}
card(w)
.frame(maxWidth: 380)
.padding(28)
.consoleGlass(RoundedRectangle(cornerRadius: 22, style: .continuous))
.overlay(
RoundedRectangle(cornerRadius: 22, style: .continuous)
.strokeBorder(.white.opacity(0.12), lineWidth: 1))
.padding(40)
}
.environment(\.colorScheme, .dark)
.transition(.opacity)
#if os(iOS) || os(macOS)
.background { WakeControllerInput(waker: waker) }
#endif
}
}
@ViewBuilder private func card(_ w: HostWaker.Waking) -> some View {
VStack(spacing: 14) {
if w.timedOut {
Image(systemName: "moon.zzz.fill")
.font(.system(size: 34)).foregroundStyle(.white.opacity(0.85))
Text("\(w.hostName) didn't wake")
.font(.geist(19, .bold, relativeTo: .title3)).foregroundStyle(.white)
Text("It may still be booting, or it's powered off / off this network.")
.font(.geist(13, relativeTo: .caption)).foregroundStyle(.white.opacity(0.6))
.multilineTextAlignment(.center)
HStack(spacing: 12) {
Button("Cancel") { waker.cancel() }.buttonStyle(.bordered)
Button("Try Again") { waker.retry() }.glassProminentButtonStyle()
}
.padding(.top, 6)
} else {
ProgressView().controlSize(.large).tint(.white)
Text("Waking \(w.hostName)")
.font(.geist(19, .bold, relativeTo: .title3)).foregroundStyle(.white)
Text("Waiting for it to come online · \(w.seconds)s")
.font(.geistFixed(13)).foregroundStyle(.white.opacity(0.6))
.monospacedDigit()
Button(w.connectsAfter ? "Cancel" : "Stop Waiting") { waker.cancel() }
.buttonStyle(.bordered)
.padding(.top, 6)
}
}
}
}
#if os(iOS) || os(macOS)
/// Controller binding for the overlay: B cancels; A retries once it has timed out. A zero-size
/// backing view owning a `GamepadMenuInput` for the overlay's lifetime (the home carousel/list is
/// gated inactive while a wake is up, so nothing else is consuming the pad).
private struct WakeControllerInput: View {
@ObservedObject var waker: HostWaker
@State private var input = GamepadMenuInput(manager: .shared)
var body: some View {
Color.clear
.onAppear {
input.onBack = { waker.cancel() }
input.onConfirm = { if waker.waking?.timedOut == true { waker.retry() } }
input.start()
}
.onDisappear { input.stop() }
}
}
#endif
@@ -49,8 +49,24 @@ enum ShotScenes {
ShotScene(name: "08-gamepad-addhost", orientation: .natural, colorScheme: .dark) { ShotScene(name: "08-gamepad-addhost", orientation: .natural, colorScheme: .dark) {
AnyView(ShotGamepadAddHost()) AnyView(ShotGamepadAddHost())
}, },
ShotScene(name: "09-waking", orientation: .natural, colorScheme: .dark) { ShotScene(name: "09-connecting", orientation: .natural, colorScheme: .dark) {
AnyView(ShotWaking()) AnyView(ShotConnect(kind: .connecting))
},
ShotScene(name: "09b-waking", orientation: .natural, colorScheme: .dark) {
AnyView(ShotConnect(kind: .waking))
},
ShotScene(name: "09c-wake-timed-out", orientation: .natural, colorScheme: .dark) {
AnyView(ShotConnect(kind: .timedOut))
},
// The default-UI presentation (Liquid Glass modal over the touch grid) of the same phases.
ShotScene(name: "09d-connecting-modal", orientation: .natural, colorScheme: .dark) {
AnyView(ShotConnect(kind: .connecting, gamepadUI: false))
},
ShotScene(name: "09e-waking-modal", orientation: .natural, colorScheme: .dark) {
AnyView(ShotConnect(kind: .waking, gamepadUI: false))
},
ShotScene(name: "09f-wake-timed-out-modal", orientation: .natural, colorScheme: .dark) {
AnyView(ShotConnect(kind: .timedOut, gamepadUI: false))
}, },
] ]
#endif #endif
@@ -137,23 +153,53 @@ private struct ShotGamepadAddHost: View {
var body: some View { GamepadAddHostView(onAdd: { _ in }) } var body: some View { GamepadAddHostView(onAdd: { _ in }) }
} }
private struct ShotWaking: View { /// The unified connect overlay (the real `ConnectOverlay`) in each phase instant "Connecting"
/// feedback, the "Waking" wait, and the wake-timed-out prompt. `gamepadUI` picks the presentation:
/// the console's full-screen aurora takeover over the gamepad home, or the default UI's Liquid Glass
/// modal over the touch host grid.
private struct ShotConnect: View {
enum Kind { case connecting, waking, timedOut }
let kind: Kind
var gamepadUI = true
@StateObject private var store = ShotMock.hostStore() @StateObject private var store = ShotMock.hostStore()
@StateObject private var model = SessionModel() @StateObject private var model = SessionModel()
@StateObject private var discovery = HostDiscovery() @StateObject private var discovery = HostDiscovery()
@StateObject private var waker = HostWaker() @StateObject private var waker = HostWaker()
var body: some View { var body: some View {
GamepadHomeView( backdrop
store: store, model: model, discovery: discovery, .overlay {
libraryTarget: .constant(nil), waker: waker, ConnectOverlay(
connect: { _ in }, connectDiscovered: { _ in } connectingHostName: kind == .connecting ? "Battlestation" : nil,
) waker: waker,
.overlay { WakeOverlay(waker: waker) } gamepadUI: gamepadUI,
onCancelConnect: {})
}
.onAppear { .onAppear {
switch kind {
case .connecting:
break
case .waking:
waker.debugSet(.init( waker.debugSet(.init(
hostID: store.hosts.first?.id ?? UUID(), hostID: store.hosts.first?.id ?? UUID(),
hostName: "Battlestation", connectsAfter: true, seconds: 14)) hostName: "Battlestation", connectsAfter: true, seconds: 14))
case .timedOut:
waker.debugSet(.init(
hostID: store.hosts.first?.id ?? UUID(),
hostName: "Battlestation", connectsAfter: true, seconds: 90, timedOut: true))
}
}
}
@ViewBuilder private var backdrop: some View {
if gamepadUI {
GamepadHomeView(
store: store, model: model, discovery: discovery,
libraryTarget: .constant(nil), waker: waker,
connect: { _ in }, connectDiscovered: { _ in })
} else {
ShotHome()
} }
} }
} }
@@ -22,7 +22,7 @@ final class HostWaker: ObservableObject {
var timedOut = false var timedOut = false
} }
/// nil = idle; non-nil drives `WakeOverlay`. /// nil = idle; non-nil drives the "Waking" phase of `ConnectOverlay`.
@Published private(set) var waking: Waking? @Published private(set) var waking: Waking?
/// How long to wait for the host to reappear before giving up. Generous a cold boot + service /// How long to wait for the host to reappear before giving up. Generous a cold boot + service
@@ -0,0 +1,41 @@
// The resize overlay (design/midstream-resolution-resize.md client resize UX). A Match-window
// resize renegotiates the host's virtual display + encoder and re-inits the local VideoToolbox
// decoder on the first new-mode IDR an unavoidable sub-second gap where the last frame lingers,
// briefly freezes, or the picture pops to the new geometry. Rather than let that read as a stutter,
// we make it DELIBERATE: the caller blurs the live stream and this centered spinner + caption
// acknowledges the transition. It clears the instant a frame at the requested size decodes (the
// `onDecodedSize` END signal) or on the follower's safety timeout see `SessionModel.resizing`.
//
// Floating overlay, never a hit-test target: input keeps flowing to the stream underneath so a
// resize the user triggers by dragging the window never swallows their next click.
import PunktfunkKit
import SwiftUI
struct ResizeIndicatorView: View {
/// Mirrors `SessionModel.resizing`; the fade in/out is driven off this.
let active: Bool
var body: some View {
ZStack {
if active {
VStack(spacing: 12) {
ProgressView().controlSize(.large).tint(.white)
Text("Resizing…")
.font(.geist(15, .medium, relativeTo: .callout))
.foregroundStyle(.white.opacity(0.85))
}
.padding(.horizontal, 30)
.padding(.vertical, 24)
.glassBackground(RoundedRectangle(cornerRadius: 20, style: .continuous))
.overlay(
RoundedRectangle(cornerRadius: 20, style: .continuous)
.strokeBorder(.white.opacity(0.12), lineWidth: 1))
.transition(.opacity.combined(with: .scale(scale: 0.92)))
}
}
.environment(\.colorScheme, .dark) // the spinner + glass read over any frame
.animation(.easeInOut(duration: 0.22), value: active)
.allowsHitTesting(false) // the stream keeps receiving input the whole time
}
}
@@ -109,6 +109,16 @@ final class SessionModel: ObservableObject {
/// Mirrors StreamView's capture state (it owns the input capture; this drives the /// Mirrors StreamView's capture state (it owns the input capture; this drives the
/// HUD's "click to capture" / " releases" hint). /// HUD's "click to capture" / " releases" hint).
@Published var mouseCaptured = false @Published var mouseCaptured = false
/// Resize overlay (design/midstream-resolution-resize.md client resize UX): true from the
/// instant a Match-window resize starts steering toward a new size until a frame at that size
/// decodes (or a safety timeout). Drives the blur+spinner so the unavoidable host-rebuild delay
/// reads as a deliberate, acknowledged transition instead of a stutter. Pure state lives in
/// `ResizeIndicator`; this mirrors its `active` for SwiftUI.
@Published private(set) var resizing = false
/// START = follower steering (main actor), END = a new-mode IDR's coded dims (decode pump,
/// hopped to main), TIMEOUT = safety net for a rejected/capped switch that never yields a
/// differently-sized frame. Ticked from the 1 Hz stats timer.
private var resizeIndicator = ResizeIndicator()
let meter = FrameMeter() let meter = FrameMeter()
/// Capturereceived (the host+network stage), fed per AU at receipt by the stream view's /// Capturereceived (the host+network stage), fed per AU at receipt by the stream view's
@@ -364,6 +374,8 @@ final class SessionModel: ObservableObject {
lostFrames = 0 lostFrames = 0
lostPct = 0 lostPct = 0
mouseCaptured = false mouseCaptured = false
resizing = false
resizeIndicator = ResizeIndicator() // no stale target/timer into the next session
} }
/// Called (via the main actor) when the pump hits end-of-session. /// Called (via the main actor) when the pump hits end-of-session.
@@ -374,6 +386,23 @@ final class SessionModel: ObservableObject {
errorMessage = "Session ended by \(name)." errorMessage = "Session ended by \(name)."
} }
/// Resize overlay START (main actor from the Match-window follower's `onResizeTarget`): the
/// window began differing from the live mode, so a `Reconfigure` toward `(width, height)` is
/// imminent. Show the blur+spinner immediately, before the debounced request even leaves.
func resizeTargeted(width: UInt32, height: UInt32) {
resizeIndicator.steering(
width: width, height: height, now: Date().timeIntervalSinceReferenceDate)
resizing = resizeIndicator.active
}
/// Resize overlay END (main actor hopped from the decode pump's `onDecodedSize`): a new-mode
/// IDR decoded at `(width, height)`. Clears the overlay only when that matches the size we're
/// steering to (a same-size loss-recovery IDR, or the initial connect IDR, is a no-op).
func resizeDecoded(width: Int, height: Int) {
resizeIndicator.decoded(width: UInt32(max(width, 0)), height: UInt32(max(height, 0)))
resizing = resizeIndicator.active
}
private func beginStreaming() { private func beginStreaming() {
guard let conn = connection else { return } guard let conn = connection else { return }
// Input capture itself is owned by StreamView (engaged by the captureEnabled // Input capture itself is owned by StreamView (engaged by the captureEnabled
@@ -390,9 +419,10 @@ final class SessionModel: ObservableObject {
micChannel: defaults.integer(forKey: DefaultsKey.micChannel), micChannel: defaults.integer(forKey: DefaultsKey.micChannel),
micEnabled: defaults.object(forKey: DefaultsKey.micEnabled) as? Bool ?? true) micEnabled: defaults.object(forKey: DefaultsKey.micEnabled) as? Bool ?? true)
self.audio = audio self.audio = audio
// Gamepads: forward GamepadManager's active controller as pad 0 and render the // Gamepads: forward every controller GamepadManager selected each on its own wire pad
// host's feedback (rumble always; lightbar/player-LEDs/adaptive-triggers when the // index (a pin forwards only one, Automatic forwards all) and render the host's feedback
// session's virtual pad is a DualSense). Same trust gate as audio nothing is // back to the pad it's addressed to (rumble always; lightbar/player-LEDs/adaptive-triggers
// when a pad's virtual device is a DualSense). Same trust gate as audio nothing is
// forwarded during the trust prompt. // forwarded during the trust prompt.
let capture = GamepadCapture(connection: conn, manager: .shared) let capture = GamepadCapture(connection: conn, manager: .shared)
// The cross-client escape chord (hold L1+R1+Start+Select 1.5 s) on tvOS the only // The cross-client escape chord (hold L1+R1+Start+Select 1.5 s) on tvOS the only
@@ -417,6 +447,11 @@ final class SessionModel: ObservableObject {
let timer = Timer(timeInterval: 1.0, repeats: true) { [weak self] _ in let timer = Timer(timeInterval: 1.0, repeats: true) { [weak self] _ in
guard let self else { return } guard let self else { return }
Task { @MainActor in Task { @MainActor in
// Resize-overlay safety net: clear a stuck overlay when a targeted size never
// decodes (a rejected/capped switch). The decoded-frame END clears it promptly on
// success; this only fires after the timeout.
self.resizeIndicator.tick(now: Date().timeIntervalSinceReferenceDate)
self.resizing = self.resizeIndicator.active
let (frames, bytes, total) = self.meter.drain() let (frames, bytes, total) = self.meter.drain()
self.fps = frames self.fps = frames
self.mbps = Double(bytes) * 8 / 1_000_000 self.mbps = Double(bytes) * 8 / 1_000_000
@@ -36,6 +36,7 @@ struct GamepadSettingsView: View {
@AppStorage(DefaultsKey.hudPlacement) private var hudPlacement = HUDPlacement.topTrailing.rawValue @AppStorage(DefaultsKey.hudPlacement) private var hudPlacement = HUDPlacement.topTrailing.rawValue
@AppStorage(DefaultsKey.libraryEnabled) private var libraryEnabled = true @AppStorage(DefaultsKey.libraryEnabled) private var libraryEnabled = true
@AppStorage(DefaultsKey.gamepadUIEnabled) private var gamepadUIEnabled = true @AppStorage(DefaultsKey.gamepadUIEnabled) private var gamepadUIEnabled = true
@AppStorage(DefaultsKey.autoWake) private var autoWakeEnabled = true
@AppStorage(DefaultsKey.presenter) private var presenter = SettingsOptions.presenterDefault @AppStorage(DefaultsKey.presenter) private var presenter = SettingsOptions.presenterDefault
@ObservedObject private var gamepads = GamepadManager.shared @ObservedObject private var gamepads = GamepadManager.shared
@@ -258,6 +259,11 @@ struct GamepadSettingsView: View {
+ "available on the host.", + "available on the host.",
options: SettingsOptions.compositors, current: compositor options: SettingsOptions.compositors, current: compositor
) { compositor = $0 }, ) { compositor = $0 },
toggleRow(
id: "autoWake", icon: "power", label: "Auto-wake on connect",
detail: "Send Wake-on-LAN to a sleeping saved host and wait for it before "
+ "streaming. Off connects straight through.",
value: $autoWakeEnabled),
choiceRow( choiceRow(
id: "codec", header: "Video", icon: "film", label: "Video codec", id: "codec", header: "Video", icon: "film", label: "Video codec",
@@ -13,6 +13,11 @@ extension SettingsView {
// failed exactly one slice: the iOS archive (macOS/tvOS never compile that branch). // failed exactly one slice: the iOS archive (macOS/tvOS never compile that branch).
@ViewBuilder var streamModeSection: some View { @ViewBuilder var streamModeSection: some View {
Section { Section {
#if os(iOS) || os(macOS)
// Match-window (design/midstream-resolution-resize.md D1): follow the session
// window/scene, renegotiating the host mode on a resize. Off the explicit mode below.
Toggle("Match window", isOn: $matchWindow)
#endif
#if os(iOS) #if os(iOS)
iosResolutionWheel iosResolutionWheel
iosRefreshRows iosRefreshRows
@@ -35,8 +40,12 @@ extension SettingsView {
} header: { } header: {
Text("Stream mode") Text("Stream mode")
} footer: { } footer: {
Text("The host creates a virtual output at exactly this mode — " Text(matchWindow
+ "native resolution, no scaling. \(Self.bitrateFooter)") ? "The stream follows this window — the host resizes its virtual output to match "
+ "as you resize, so the picture stays pixel-exact (1:1) with no scaling. "
+ "\(Self.bitrateFooter)"
: "The host creates a virtual output at exactly this mode — native resolution, but "
+ "a window that isn't this size is scaled to fit. \(Self.bitrateFooter)")
.font(.geist(12, relativeTo: .caption)) .font(.geist(12, relativeTo: .caption))
.foregroundStyle(.secondary) .foregroundStyle(.secondary)
} }
@@ -286,6 +295,24 @@ extension SettingsView {
} }
} }
/// Auto-wake on connect fire Wake-on-LAN + wait for a sleeping saved host to come back before
/// giving up. Now available on every platform (the iOS/tvOS multicast entitlement is granted).
@ViewBuilder var wakeSection: some View {
Section {
Toggle("Auto-wake on connect", isOn: $autoWakeEnabled)
} header: {
Text("Wake-on-LAN")
} footer: {
Text("Connecting to a saved host that isn't on the network yet sends a Wake-on-LAN "
+ "packet and waits for it to come back before streaming. Turn off if a host that's "
+ "already on just isn't visible here (e.g. over a VPN), so connects go straight "
+ "through instead of waiting out the wake. A host's “Wake” action still works either "
+ "way.")
.font(.geist(12, relativeTo: .caption))
.foregroundStyle(.secondary)
}
}
@ViewBuilder var windowSection: some View { @ViewBuilder var windowSection: some View {
#if os(macOS) #if os(macOS)
Section { Section {
@@ -133,8 +133,10 @@ extension SettingsView {
.foregroundStyle(.secondary) .foregroundStyle(.secondary)
} }
Spacer() Spacer()
if gamepads.active?.id == controller.id { // Every forwarded controller is surfaced (not just the primary `active`) with its
Text("In use") // wire pad index as a player number a pin forwards only one, Automatic forwards all.
if let pad = gamepads.padIndex(for: controller) {
Text("Player \(pad + 1)")
.font(.geist(11, .semibold, relativeTo: .caption2)) .font(.geist(11, .semibold, relativeTo: .caption2))
.padding(.horizontal, 8) .padding(.horizontal, 8)
.padding(.vertical, 3) .padding(.vertical, 3)
@@ -21,6 +21,10 @@ struct SettingsView: View {
@AppStorage(DefaultsKey.streamWidth) var width = 1920 @AppStorage(DefaultsKey.streamWidth) var width = 1920
@AppStorage(DefaultsKey.streamHeight) var height = 1080 @AppStorage(DefaultsKey.streamHeight) var height = 1080
@AppStorage(DefaultsKey.streamHz) var hz = 60 @AppStorage(DefaultsKey.streamHz) var hz = 60
// Opt-in (default OFF): the explicit mode below is used and never auto-resized. When ON, a
// windowed session instead streams at the window's native pixels (1:1, no scaling) so it stays
// pixel-exact rather than the presenter resampling a fixed-mode frame into the window.
@AppStorage(DefaultsKey.matchWindow) var matchWindow = false
@AppStorage(DefaultsKey.compositor) var compositor = 0 @AppStorage(DefaultsKey.compositor) var compositor = 0
@AppStorage(DefaultsKey.gamepadType) var gamepadType = 0 @AppStorage(DefaultsKey.gamepadType) var gamepadType = 0
@AppStorage(DefaultsKey.bitrateKbps) var bitrateKbps = 0 @AppStorage(DefaultsKey.bitrateKbps) var bitrateKbps = 0
@@ -44,6 +48,7 @@ struct SettingsView: View {
@AppStorage(DefaultsKey.hudPlacement) var hudPlacement = HUDPlacement.topTrailing.rawValue @AppStorage(DefaultsKey.hudPlacement) var hudPlacement = HUDPlacement.topTrailing.rawValue
@ObservedObject var gamepads = GamepadManager.shared @ObservedObject var gamepads = GamepadManager.shared
@AppStorage(DefaultsKey.gamepadUIEnabled) var gamepadUIEnabled = true @AppStorage(DefaultsKey.gamepadUIEnabled) var gamepadUIEnabled = true
@AppStorage(DefaultsKey.autoWake) var autoWakeEnabled = true
#if DEBUG && !os(tvOS) #if DEBUG && !os(tvOS)
@State var showControllerTest = false @State var showControllerTest = false
#endif #endif
@@ -105,6 +110,7 @@ struct SettingsView: View {
Form { Form {
streamModeSection streamModeSection
compositorSection compositorSection
wakeSection
} }
.formStyle(.grouped) .formStyle(.grouped)
.tabItem { Label("General", systemImage: "gearshape") } .tabItem { Label("General", systemImage: "gearshape") }
@@ -234,6 +240,7 @@ struct SettingsView: View {
streamModeSection streamModeSection
pointerSection pointerSection
compositorSection compositorSection
wakeSection
} }
.formStyle(.grouped) .formStyle(.grouped)
.navigationTitle("General") .navigationTitle("General")
@@ -304,6 +311,10 @@ struct SettingsView: View {
Binding(get: { gamepadUIEnabled ? "on" : "off" }, set: { gamepadUIEnabled = $0 == "on" }) Binding(get: { gamepadUIEnabled ? "on" : "off" }, set: { gamepadUIEnabled = $0 == "on" })
} }
private var autoWakeEnabledTag: Binding<String> {
Binding(get: { autoWakeEnabled ? "on" : "off" }, set: { autoWakeEnabled = $0 == "on" })
}
private var tvBody: some View { private var tvBody: some View {
let currentTag = "\(width)x\(height)x\(hz)" let currentTag = "\(width)x\(height)x\(hz)"
let bounds = UIScreen.main.nativeBounds let bounds = UIScreen.main.nativeBounds
@@ -343,9 +354,13 @@ struct SettingsView: View {
TVSelectionRow( TVSelectionRow(
title: "10-bit HDR", title: "10-bit HDR",
options: [("On", "on"), ("Off", "off")], selection: hdrEnabledTag) options: [("On", "on"), ("Off", "off")], selection: hdrEnabledTag)
TVSelectionRow(
title: "Auto-wake on connect",
options: [("On", "on"), ("Off", "off")], selection: autoWakeEnabledTag)
Text("The host creates a virtual output at exactly this mode — native " Text("The host creates a virtual output at exactly this mode — native "
+ "resolution, no scaling. \(Self.bitrateFooter) A specific compositor " + "resolution, no scaling. \(Self.bitrateFooter) A specific compositor "
+ "is honored only if available on the host.") + "is honored only if available on the host. Auto-wake sends Wake-on-LAN to a "
+ "sleeping saved host and waits for it before streaming.")
.font(.geist(20, relativeTo: .caption)) .font(.geist(20, relativeTo: .caption))
.foregroundStyle(.secondary) .foregroundStyle(.secondary)
.multilineTextAlignment(.center) .multilineTextAlignment(.center)
@@ -59,6 +59,26 @@ public extension PunktfunkInputEvent {
make(PUNKTFUNK_INPUT_KIND_GAMEPAD_AXIS.rawValue, code: axis, x: value, y: 0, flags: pad) make(PUNKTFUNK_INPUT_KIND_GAMEPAD_AXIS.rawValue, code: axis, x: value, y: 0, flags: pad)
} }
/// Declare a pad's controller KIND (`InputKind::GamepadArrival`): `pref` is the
/// `GamepadType` wire byte (Auto=0, Xbox360=1, DualSense=2, XboxOne=3, DualShock4=4,
/// SteamController=5, SteamDeck=6), `pad` the wire index. Sent once when a controller slot
/// opens BEFORE that pad's first input so the host builds a matching virtual device and a
/// session can mix types (pad 0 a DualSense, pad 1 an Xbox pad). The core re-sends it a few
/// times against datagram loss and folds per-pad state behind it; a host that predates the tag
/// ignores it and uses the session-default kind from the handshake. Idempotent on the host.
static func gamepadArrival(pref: UInt32, pad: UInt32) -> PunktfunkInputEvent {
make(PUNKTFUNK_INPUT_KIND_GAMEPAD_ARRIVAL.rawValue, code: pref, x: 0, y: 0, flags: pad)
}
/// A pad disconnected (`InputKind::GamepadRemove`): `flags` = pad index. The client sends the
/// bare index; the core stamps the per-pad removal seq (`encode_gamepad_remove`) in the shared
/// snapshot seq space and arms a loss-resistant re-send burst, so the host tears the pad's
/// virtual device down and no reordered snapshot can resurrect it. A host that predates the tag
/// ignores it (the pad then lingers until session end the pre-existing behaviour).
static func gamepadRemove(pad: UInt32) -> PunktfunkInputEvent {
make(PUNKTFUNK_INPUT_KIND_GAMEPAD_REMOVE.rawValue, code: 0, x: 0, y: 0, flags: pad)
}
// Touch (host-side: libei ei_touchscreen on the virtual output). `id` distinguishes // Touch (host-side: libei ei_touchscreen on the virtual output). `id` distinguishes
// fingers and is reusable after touchUp; coordinates are absolute pixels on the // fingers and is reusable after touchUp; coordinates are absolute pixels on the
// client's touch surface, whose size rides in `flags` so the host can rescale // client's touch surface, whose size rides in `flags` so the host can rescale
@@ -70,19 +70,10 @@ func withOptionalCString<R>(_ s: String?, _ body: (UnsafePointer<CChar>?) -> R)
public extension PunktfunkConnection { public extension PunktfunkConnection {
/// Whether the Wake-on-LAN broadcast path is usable on this platform/build. macOS can always /// Whether the Wake-on-LAN broadcast path is usable on this platform/build. macOS can always
/// broadcast (its App Sandbox network entitlements cover it). iOS/tvOS need the managed /// broadcast (its App Sandbox network entitlements cover it). iOS/tvOS need the managed
/// `com.apple.developer.networking.multicast` entitlement, which is GATED pending Apple's /// `com.apple.developer.networking.multicast` entitlement now approved and enabled (see
/// approval (see `Config/Punktfunk.entitlements`) until it's granted, sending a broadcast is /// `Config/Punktfunk.entitlements`), so wake is available on every platform. Kept as the single
/// blocked by the OS, so the wake path + its UI are gated off there to avoid a dead action. /// switch every call site gates on, should a future build ever need to disable it.
/// The MAC-learning path stays active on every platform, so flipping this on once the static var wakeOnLANAvailable: Bool { true }
/// entitlement lands makes wake work immediately. ON APPROVAL: change `#if os(macOS)` below to
/// `true` for iOS/tvOS too (and uncomment the entitlement).
static var wakeOnLANAvailable: Bool {
#if os(macOS)
return true
#else
return false
#endif
}
/// Send a Wake-on-LAN magic packet to wake a sleeping host. `macs` are the host's NIC MAC(s) /// Send a Wake-on-LAN magic packet to wake a sleeping host. `macs` are the host's NIC MAC(s)
/// (`aa:bb:cc:dd:ee:ff`, learned from its mDNS `mac` TXT while awake); malformed entries are /// (`aa:bb:cc:dd:ee:ff`, learned from its mDNS `mac` TXT while awake); malformed entries are
@@ -445,6 +436,35 @@ public final class PunktfunkConnection {
_ = punktfunk_connection_request_keyframe(h) _ = punktfunk_connection_request_keyframe(h)
} }
/// Feed each received AU's `frameIndex` (in receive order) so the client recovers from loss with a
/// cheap reference-frame invalidation instead of always paying for a full IDR. On a forward gap
/// a `frameIndex` jump means the intervening frames were lost and the following AUs reference a
/// picture that never arrived the core fires a THROTTLED RFI request for the lost range, and an
/// RFI-capable host (AMD LTR / NVENC) recovers with a clean P-frame rather than a 20-40× IDR
/// spike. Call it for every received AU; the `framesDropped`-driven `requestKeyframe()` path stays
/// the backstop for when the recovery frame itself is lost. Cheap; silently dropped after close.
public func noteFrameIndex(_ frameIndex: UInt32) {
abiLock.lock()
defer { abiLock.unlock() }
guard let h = handle, !closeRequested else { return }
_ = punktfunk_connection_note_frame_index(h, frameIndex, nil)
}
/// Like `noteFrameIndex`, but also reports whether the core saw a FORWARD frame-index gap the
/// signal that intervening frames were lost and the following AUs reference a picture that never
/// arrived. The post-loss re-anchor gate arms its display freeze on a gap (the earliest, most
/// precise loss trigger ahead of the `framesDropped` climb). Same core side effect as
/// `noteFrameIndex` (the throttled RFI request); call it for every received AU. Returns false
/// after close.
public func noteFrameIndexGap(_ frameIndex: UInt32) -> Bool {
abiLock.lock()
defer { abiLock.unlock() }
guard let h = handle, !closeRequested else { return false }
var gap = false
_ = punktfunk_connection_note_frame_index(h, frameIndex, &gap)
return gap
}
/// Cumulative access units the hostclient reassembler dropped as unrecoverable (FEC couldn't /// Cumulative access units the hostclient reassembler dropped as unrecoverable (FEC couldn't
/// rebuild them). The video pump polls this and calls `requestKeyframe()` when it climbs the /// rebuild them). The video pump polls this and calls `requestKeyframe()` when it climbs the
/// correct loss trigger under the host's infinite GOP, where unrecoverable loss yields /// correct loss trigger under the host's infinite GOP, where unrecoverable loss yields
@@ -1,24 +1,33 @@
// Gamepad capture punktfunk/1 datagrams. Forwards exactly ONE controller whatever // Gamepad capture punktfunk/1 datagrams. Forwards EVERY controller GamepadManager selected
// GamepadManager selected as pad 0, for the lifetime of a streaming session. // each on its own stable wire pad index (pf-client-core's slot model) for the lifetime of a
// streaming session. One physical controller with no pin is player 0 (byte-identical to the old
// single-pad path); a pin forwards only that one, also as pad 0.
// //
// The wire is incremental (one button/axis transition per 18-byte event, accumulated // Each forwarded controller gets a `Slot`: its open GC handlers plus the wire state (buttons,
// host-side into the virtual pad see punktfunk_core::input::gamepad), so we snapshot the // axes, touchpad fingers, motion throttle) for its pad index isolated per device so two
// full GCExtendedGamepad state on every valueChanged and diff against the previous // controllers never clobber each other. On connect a slot opens (GamepadArrival declares its
// snapshot. Sticks are ±32767 with +y = up (GC already matches, no flip), triggers 0...255. // kind, then input flows); on disconnect / pin change / stop it closes (held state flushed to
// rest on the wire, then GamepadRemove tells the host to tear the pad's virtual device down).
//
// The wire is incremental (one button/axis transition per 18-byte event, accumulated host-side
// into the virtual pad see punktfunk_core::input::gamepad), so we snapshot the full
// GCExtendedGamepad state on every valueChanged and diff against the previous snapshot. Sticks
// are ±32767 with +y = up (GC already matches, no flip), triggers 0...255. The core folds these
// per-pad transitions into idempotent, sequence-numbered snapshots keyed on the same pad index,
// so all this layer must get right is the index one controller per slot, one slot per index.
// //
// PlayStation-pad extras ride the rich-input plane (0xCC): touchpad contacts normalized // PlayStation-pad extras ride the rich-input plane (0xCC): touchpad contacts normalized
// 0...65535 (origin top-left, +y down GC's ±1/+y-up is converted here) and motion // 0...65535 (origin top-left, +y down GC's ±1/+y-up is converted here) and motion samples in
// samples in raw DualSense sensor units (gyro 20 LSB per deg/s, accel 10000 LSB per g // raw DualSense sensor units (gyro 20 LSB per deg/s, accel 10000 LSB per g derived from the
// derived from the host's fixed calibration blob; the conversion lives in ONE place, // host's fixed calibration blob; the conversion lives in ONE place, `Wire`, so a live sign/scale
// `Wire`, so a live sign/scale correction is a one-line change). The host ignores both // correction is a one-line change). The host ignores both unless a pad's virtual device is a
// unless the session's virtual pad is a DualSense or DualShock 4 both carry a touchpad // DualSense or DualShock 4 both carry a touchpad and motion, so the capture below covers either
// and motion, so the capture below covers either (`GCDualShockGamepad` exposes the same // (`GCDualShockGamepad` exposes the same `touchpad*` surface as `GCDualSenseGamepad`).
// `touchpad*` surface as `GCDualSenseGamepad`).
// //
// Unlike mouse/keyboard capture, gamepad forwarding is NOT gated on the mouse-capture // Unlike mouse/keyboard capture, gamepad forwarding is NOT gated on the mouse-capture toggle a
// toggle a controller can't click local UI, so it always drives the host while the app // controller can't click local UI, so it always drives the host while the app is active. On
// is active. On deactivation, controller switch, or stop, every held control is released // deactivation, controller switch, or stop, every held control is released on the wire (the host
// on the wire (the host pad would otherwise stay stuck on the last state). // pad would otherwise stay stuck on the last state).
#if os(macOS) #if os(macOS)
import AppKit import AppKit
@@ -33,17 +42,35 @@ import GameController
public final class GamepadCapture { public final class GamepadCapture {
private let connection: PunktfunkConnection private let connection: PunktfunkConnection
private let manager: GamepadManager private let manager: GamepadManager
private var activeSub: AnyCancellable? private var forwardedSub: AnyCancellable?
private var observers: [NSObjectProtocol] = [] private var observers: [NSObjectProtocol] = []
private var bound: GCController?
/// App inactive GC stops delivering; everything is released and stays silent. /// App inactive GC stops delivering; everything is released and stays silent.
private var suspended = false private var suspended = false
// Last wire state (the diff base also what releaseAll() unwinds). /// One forwarded controller: the open device plus the last wire state for its pad index (the
private var buttons: UInt32 = 0 /// diff base also what `flush` unwinds). Held per Slot so two controllers never clobber each
private var axes: [Int32] = [0, 0, 0, 0, 0, 0] /// other's held buttons/axes/fingers. Mirrors pf-client-core's `Slot`.
private var fingerActive: [Bool] = [false, false] private final class Slot {
private var lastMotionNs: UInt64 = 0 let controller: GCController
/// Wire pad index (GamepadManager's stable lowest-free assignment), threaded onto every
/// event this controller sends the low byte of `flags`.
let pad: UInt32
/// The controller KIND declared to the host (GamepadArrival) when the slot opened.
let pref: PunktfunkConnection.GamepadType
var buttons: UInt32 = 0
var axes: [Int32] = [0, 0, 0, 0, 0, 0]
var fingerActive: [Bool] = [false, false]
var lastMotionNs: UInt64 = 0
init(controller: GCController, pad: UInt32, pref: PunktfunkConnection.GamepadType) {
self.controller = controller
self.pad = pad
self.pref = pref
}
}
/// Open forwarded controllers, one Slot per physical pad on its own wire index. Reconciled
/// against `manager.forwarded` (empty until a session's `start`, cleared by `stop`).
private var slots: [Slot] = []
/// Motion forwarding floor: 4 ms between samples ( 250 Hz, the DualSense's own rate). /// Motion forwarding floor: 4 ms between samples ( 250 Hz, the DualSense's own rate).
private static let motionIntervalNs: UInt64 = 4_000_000 private static let motionIntervalNs: UInt64 = 4_000_000
@@ -71,10 +98,14 @@ public final class GamepadCapture {
} }
public func start() { public func start() {
// Fires immediately with the current selection, then on every change a switch // Session-scoped index assignment: a controller pinned before the session forwards as
// releases the old controller's wire state before the new one takes over. // pad 0 (pf-client-core assigns indices at slot-open time, not app-launch time).
activeSub = manager.$active.sink { [weak self] dc in manager.resetForwardingAssignment()
MainActor.assumeIsolated { self?.rebind(to: dc?.controller) } // Fires immediately with the current forwarded set, then on every change a connect,
// disconnect, or pin change reconciles the open slots against it (opening/closing devices
// and flushing wire state so nothing sticks down).
forwardedSub = manager.$forwarded.sink { [weak self] list in
MainActor.assumeIsolated { self?.reconcile(list) }
} }
#if os(macOS) #if os(macOS)
let resign = NSApplication.willResignActiveNotification let resign = NSApplication.willResignActiveNotification
@@ -97,53 +128,56 @@ public final class GamepadCapture {
MainActor.assumeIsolated { MainActor.assumeIsolated {
guard let self else { return } guard let self else { return }
self.suspended = false self.suspended = false
if let ext = self.bound?.extendedGamepad { self.sync(ext) } // Re-send every open pad's current state (GC delivered nothing while inactive).
for slot in self.slots {
if let ext = slot.controller.extendedGamepad { self.sync(slot, ext) }
}
} }
}) })
} }
public func stop() { public func stop() {
releaseAll() closeAllSlots()
rebind(to: nil) forwardedSub = nil
activeSub = nil
observers.forEach { NotificationCenter.default.removeObserver($0) } observers.forEach { NotificationCenter.default.removeObserver($0) }
observers.removeAll() observers.removeAll()
} }
private func rebind(to controller: GCController?) { /// Bring `slots` in line with the forwarded set: close any slot no longer wanted (flushing its
guard controller !== bound else { return } /// held wire state and sending GamepadRemove first) and open any newly-forwarded controller into
releaseAll() /// its assigned wire index. A controller that stays forwarded keeps its slot untouched, so a
if let ext = bound?.extendedGamepad { /// second pad connecting never disturbs the first. Mirrors pf-client-core's `reconcile_slots`.
ext.valueChangedHandler = nil private func reconcile(_ forwarded: [GamepadManager.DiscoveredController]) {
let tp = Self.touchpad(ext) let wantIDs = Set(forwarded.map { ObjectIdentifier($0.controller) })
tp?.primary.valueChangedHandler = nil for slot in slots where !wantIDs.contains(ObjectIdentifier(slot.controller)) {
tp?.secondary.valueChangedHandler = nil closeSlot(slot)
} }
// Hand the system gestures back to the OS before letting the old pad go outside a for dc in forwarded where !slots.contains(where: { $0.controller === dc.controller }) {
// stream the share button's screenshot and the Home overlay are the user's, not ours. openSlot(dc)
if let old = bound {
for element in old.physicalInputProfile.elements.values {
element.preferredSystemGestureState = .enabled
} }
// A chord-holding pad may have just unplugged re-evaluate so a stale hold disarms.
updateEscapeChord()
} }
if let motion = bound?.motion {
motion.valueChangedHandler = nil
// Power the sensors back down left active they keep the pad streaming
// gyro/accel over Bluetooth (battery drain) long after the session.
if motion.sensorsRequireManualActivation { motion.sensorsActive = false }
}
bound = controller
guard let c = controller, let ext = c.extendedGamepad else { return }
ext.valueChangedHandler = { [weak self] g, _ in /// Open one forwarded controller on its assigned wire index: attach GC handlers, claim its
MainActor.assumeIsolated { self?.sync(g) } /// system gestures, declare its kind (GamepadArrival before any input), then wake the host
/// pad and send its initial state. Skipped when the pad has no wire index (every slot taken)
/// or exposes no extended profile.
private func openSlot(_ dc: GamepadManager.DiscoveredController) {
guard let pad = manager.padIndex(for: dc), let ext = dc.controller.extendedGamepad else { return }
let c = dc.controller
let slot = Slot(controller: c, pad: UInt32(pad), pref: dc.kind)
slots.append(slot)
ext.valueChangedHandler = { [weak self, weak slot] g, _ in
MainActor.assumeIsolated { if let self, let slot { self.sync(slot, g) } }
} }
// Claim EVERY element's system gesture while this pad drives a stream. The OS attaches // Claim EVERY element's system gesture while this pad drives a stream. The OS attaches
// gestures to several controller buttons share/create local screenshot/recording, // gestures to several controller buttons share/create local screenshot/recording,
// Home Game Center overlay (iOS) / Launchpad's Games folder (macOS) and with a // Home Game Center overlay (iOS) / Launchpad's Games folder (macOS) and with a
// gesture attached the press is the system's, not the game's. During capture the remote // gesture attached the press is the system's, not the game's. During capture the remote
// session IS the game: the share button must reach the host (e.g. Steam screenshots), // session IS the game: the share button must reach the host (e.g. Steam screenshots),
// the PS button must open the host's Steam overlay. Restored to .enabled on unbind. // the PS button must open the host's Steam overlay. Restored to .enabled on close.
for element in c.physicalInputProfile.elements.values { for element in c.physicalInputProfile.elements.values {
element.preferredSystemGestureState = .disabled element.preferredSystemGestureState = .disabled
} }
@@ -153,42 +187,83 @@ public final class GamepadCapture {
// `extendedGamepad.buttonHome` is unreliable/often nil even when the physical element // `extendedGamepad.buttonHome` is unreliable/often nil even when the physical element
// exists. On tvOS the element is absent (reserved) nil, the whole block no-ops. // exists. On tvOS the element is absent (reserved) nil, the whole block no-ops.
if let home = c.physicalInputProfile.buttons[GCInputButtonHome] { if let home = c.physicalInputProfile.buttons[GCInputButtonHome] {
home.pressedChangedHandler = { [weak self] _, _, pressed in home.pressedChangedHandler = { [weak self, weak slot] _, _, pressed in
MainActor.assumeIsolated { self?.sendGuide(down: pressed) } MainActor.assumeIsolated { if let self, let slot { self.sendGuide(slot, down: pressed) } }
} }
} }
// Wake the host pad immediately (pads are created lazily from the first event; // Declare this pad's controller KIND before any of its input, so the host builds a
// a DualSense's UHID handshake + initial lightbar write only start then). // matching virtual device (mixed types pad 0 a DualSense, pad 1 an Xbox pad). The core
connection.send(.gamepadAxis(GamepadWire.axisLSX, value: 0, pad: 0)) // re-sends it a few times against datagram loss; an older host ignores it and uses the
sync(ext) // session-default kind. Then wake the host pad (pads are created lazily from the first
// event; a DualSense's UHID handshake + initial lightbar write only start then).
connection.send(.gamepadArrival(pref: slot.pref.rawValue, pad: slot.pad))
connection.send(.gamepadAxis(GamepadWire.axisLSX, value: 0, pad: slot.pad))
sync(slot, ext)
if let tp = Self.touchpad(ext) { if let tp = Self.touchpad(ext) {
tp.primary.valueChangedHandler = { [weak self] _, x, y in tp.primary.valueChangedHandler = { [weak self, weak slot] _, x, y in
MainActor.assumeIsolated { self?.touch(finger: 0, x: x, y: y) } MainActor.assumeIsolated { if let self, let slot { self.touch(slot, finger: 0, x: x, y: y) } }
} }
tp.secondary.valueChangedHandler = { [weak self] _, x, y in tp.secondary.valueChangedHandler = { [weak self, weak slot] _, x, y in
MainActor.assumeIsolated { self?.touch(finger: 1, x: x, y: y) } MainActor.assumeIsolated { if let self, let slot { self.touch(slot, finger: 1, x: x, y: y) } }
} }
} }
if let motion = c.motion { if let motion = c.motion {
if motion.sensorsRequireManualActivation { motion.sensorsActive = true } if motion.sensorsRequireManualActivation { motion.sensorsActive = true }
motion.valueChangedHandler = { [weak self] m in motion.valueChangedHandler = { [weak self, weak slot] m in
MainActor.assumeIsolated { self?.forwardMotion(m) } MainActor.assumeIsolated { if let self, let slot { self.forwardMotion(slot, m) } }
} }
} }
} }
/// Snapshot the profile into wire state and send every transition since the last one. /// Flush a slot's held wire state (so nothing sticks down host-side) and signal the host to tear
private func sync(_ g: GCExtendedGamepad) { /// its virtual device down (GamepadRemove), then detach GC handlers, hand the system gestures
/// back, and power the sensors down. Wire-only until the GC cleanup, so it is safe even when the
/// device already physically unplugged. Mirrors pf-client-core's `close_slot_at`.
private func closeSlot(_ slot: Slot) {
flush(slot)
// Sent after the flush so the core stamps it with a seq past the zeroing snapshots; the host
// seq-gates it, so a reordered snapshot can't resurrect the removed pad.
connection.send(.gamepadRemove(pad: slot.pad))
let c = slot.controller
if let ext = c.extendedGamepad {
ext.valueChangedHandler = nil
let tp = Self.touchpad(ext)
tp?.primary.valueChangedHandler = nil
tp?.secondary.valueChangedHandler = nil
}
c.physicalInputProfile.buttons[GCInputButtonHome]?.pressedChangedHandler = nil
// Hand the system gestures back to the OS before letting the pad go outside a stream the
// share button's screenshot and the Home overlay are the user's, not ours.
for element in c.physicalInputProfile.elements.values {
element.preferredSystemGestureState = .enabled
}
if let motion = c.motion {
motion.valueChangedHandler = nil
// Power the sensors back down left active they keep the pad streaming gyro/accel
// over Bluetooth (battery drain) long after the session.
if motion.sensorsRequireManualActivation { motion.sensorsActive = false }
}
slots.removeAll { $0 === slot }
}
private func closeAllSlots() {
while let slot = slots.first { closeSlot(slot) }
chordTimer?.invalidate()
chordTimer = nil
}
/// Snapshot the profile into a slot's wire state and send every transition since the last one,
/// tagged with the slot's wire pad index.
private func sync(_ slot: Slot, _ g: GCExtendedGamepad) {
guard !suspended else { return } guard !suspended else { return }
let newButtons = Self.buttonMask(g) let newButtons = Self.buttonMask(g)
updateEscapeChord(newButtons) let changed = newButtons ^ slot.buttons
let changed = newButtons ^ buttons
if changed != 0 { if changed != 0 {
for bit in GamepadWire.allButtons where changed & bit != 0 { for bit in GamepadWire.allButtons where changed & bit != 0 {
connection.send(.gamepadButton(bit, down: newButtons & bit != 0, pad: 0)) connection.send(.gamepadButton(bit, down: newButtons & bit != 0, pad: slot.pad))
} }
buttons = newButtons slot.buttons = newButtons
} }
let newAxes: [Int32] = [ let newAxes: [Int32] = [
Int32((g.leftThumbstick.xAxis.value * 32767).rounded()), Int32((g.leftThumbstick.xAxis.value * 32767).rounded()),
@@ -198,22 +273,23 @@ public final class GamepadCapture {
Int32((g.leftTrigger.value * 255).rounded()), Int32((g.leftTrigger.value * 255).rounded()),
Int32((g.rightTrigger.value * 255).rounded()), Int32((g.rightTrigger.value * 255).rounded()),
] ]
for (i, v) in newAxes.enumerated() where v != axes[i] { for (i, v) in newAxes.enumerated() where v != slot.axes[i] {
connection.send(.gamepadAxis(UInt32(i), value: v, pad: 0)) connection.send(.gamepadAxis(UInt32(i), value: v, pad: slot.pad))
axes[i] = v slot.axes[i] = v
} }
updateEscapeChord()
} }
/// Forward the guide (Home/PS) transition directly it's kept out of `buttonMask` (the legacy /// Forward the guide (Home/PS) transition directly it's kept out of `buttonMask` (the legacy
/// `buttonHome` element is unreliable). Folds into `buttons` so a held PS button is released by /// `buttonHome` element is unreliable). Folds into the slot's `buttons` so a held PS button is
/// `releaseAll` on focus loss just like the others. /// released by `flush` on focus loss / close just like the others.
private func sendGuide(down: Bool) { private func sendGuide(_ slot: Slot, down: Bool) {
guard !suspended else { return } guard !suspended else { return }
let bit = GamepadWire.guide let bit = GamepadWire.guide
let now = down ? (buttons | bit) : (buttons & ~bit) let now = down ? (slot.buttons | bit) : (slot.buttons & ~bit)
guard now != buttons else { return } guard now != slot.buttons else { return }
connection.send(.gamepadButton(bit, down: down, pad: 0)) connection.send(.gamepadButton(bit, down: down, pad: slot.pad))
buttons = now slot.buttons = now
} }
private static func buttonMask(_ g: GCExtendedGamepad) -> UInt32 { private static func buttonMask(_ g: GCExtendedGamepad) -> UInt32 {
@@ -234,7 +310,7 @@ public final class GamepadCapture {
if g.leftShoulder.isPressed { b |= GamepadWire.leftShoulder } if g.leftShoulder.isPressed { b |= GamepadWire.leftShoulder }
if g.rightShoulder.isPressed { b |= GamepadWire.rightShoulder } if g.rightShoulder.isPressed { b |= GamepadWire.rightShoulder }
// guide (Home/PS) is NOT read here it's forwarded directly by the Home button's // guide (Home/PS) is NOT read here it's forwarded directly by the Home button's
// pressedChangedHandler (the legacy `buttonHome` element is unreliable). See `rebind`. // pressedChangedHandler (the legacy `buttonHome` element is unreliable). See `openSlot`.
if g.buttonA.isPressed { b |= GamepadWire.a } if g.buttonA.isPressed { b |= GamepadWire.a }
if g.buttonB.isPressed { b |= GamepadWire.b } if g.buttonB.isPressed { b |= GamepadWire.b }
if g.buttonX.isPressed { b |= GamepadWire.x } if g.buttonX.isPressed { b |= GamepadWire.x }
@@ -262,29 +338,29 @@ public final class GamepadCapture {
return nil return nil
} }
/// One touchpad finger moved. GC reports ±1 positions and snaps to exactly (0, 0) on /// One touchpad finger moved on a slot's pad. GC reports ±1 positions and snaps to exactly
/// lift treated as the lift signal (a real finger landing on the precise center /// (0, 0) on lift treated as the lift signal (a real finger landing on the precise center
/// momentarily reads as a lift; harmless for a 1-in-65k coincidence). /// momentarily reads as a lift; harmless for a 1-in-65k coincidence).
private func touch(finger: Int, x: Float, y: Float) { private func touch(_ slot: Slot, finger: Int, x: Float, y: Float) {
guard !suspended else { return } guard !suspended else { return }
let lifted = x == 0 && y == 0 let lifted = x == 0 && y == 0
if lifted { if lifted {
if fingerActive[finger] { if slot.fingerActive[finger] {
fingerActive[finger] = false slot.fingerActive[finger] = false
connection.sendTouchpad(finger: UInt8(finger), active: false, x: 0, y: 0) connection.sendTouchpad(pad: UInt8(slot.pad), finger: UInt8(finger), active: false, x: 0, y: 0)
} }
return return
} }
fingerActive[finger] = true slot.fingerActive[finger] = true
let w = GamepadWire.touchpad(x: x, y: y) let w = GamepadWire.touchpad(x: x, y: y)
connection.sendTouchpad(finger: UInt8(finger), active: true, x: w.x, y: w.y) connection.sendTouchpad(pad: UInt8(slot.pad), finger: UInt8(finger), active: true, x: w.x, y: w.y)
} }
private func forwardMotion(_ m: GCMotion) { private func forwardMotion(_ slot: Slot, _ m: GCMotion) {
guard !suspended else { return } guard !suspended else { return }
let now = DispatchTime.now().uptimeNanoseconds let now = DispatchTime.now().uptimeNanoseconds
guard now &- lastMotionNs >= Self.motionIntervalNs else { return } guard now &- slot.lastMotionNs >= Self.motionIntervalNs else { return }
lastMotionNs = now slot.lastMotionNs = now
// Total acceleration in g: gravity + user when split, else the raw vector. // Total acceleration in g: gravity + user when split, else the raw vector.
let ax: Float let ax: Float
let ay: Float let ay: Float
@@ -301,6 +377,7 @@ public final class GamepadCapture {
let gs = GamepadWire.gyroLSBPerRadS let gs = GamepadWire.gyroLSBPerRadS
let as_ = GamepadWire.accelLSBPerG let as_ = GamepadWire.accelLSBPerG
connection.sendMotion( connection.sendMotion(
pad: UInt8(slot.pad),
gyro: ( gyro: (
GamepadWire.motionRaw(Float(m.rotationRate.x), scale: gs), GamepadWire.motionRaw(Float(m.rotationRate.x), scale: gs),
GamepadWire.motionRaw(Float(m.rotationRate.y), scale: gs), GamepadWire.motionRaw(Float(m.rotationRate.y), scale: gs),
@@ -313,13 +390,12 @@ public final class GamepadCapture {
)) ))
} }
/// Unwind everything held on the wire: button-ups, neutral axes, lifted fingers. The /// Arm the disconnect timer when ANY forwarded pad holds the full escape chord, disarm the
/// host's virtual pad returns to rest instead of running with the last state. /// moment none do a release, or the holding pad unplugged (pf-client-core's `chord_held` is
/// Arm the disconnect timer when the full chord lands, disarm the moment any of the four /// likewise any-slot). GC events only arrive on state CHANGES, so a held chord needs the timer:
/// releases. Events only arrive on state CHANGES, so a held chord needs the timer the /// the handler won't fire again until something moves.
/// handler won't fire again until something moves. private func updateEscapeChord() {
private func updateEscapeChord(_ newButtons: UInt32) { let held = slots.contains { $0.buttons & Self.escapeChord == Self.escapeChord }
let held = newButtons & Self.escapeChord == Self.escapeChord
if held, chordTimer == nil { if held, chordTimer == nil {
let timer = Timer(timeInterval: Self.disconnectHold, repeats: false) { [weak self] _ in let timer = Timer(timeInterval: Self.disconnectHold, repeats: false) { [weak self] _ in
Task { @MainActor in self?.onDisconnectRequest?() } Task { @MainActor in self?.onDisconnectRequest?() }
@@ -332,20 +408,31 @@ public final class GamepadCapture {
} }
} }
/// Unwind everything a slot holds on the wire: button-ups, neutral axes, lifted fingers. The
/// host's virtual pad returns to rest instead of running with the last state. Wire events only
/// (no GC calls) safe against an already-removed device. Does NOT close the slot or send
/// GamepadRemove (that's `closeSlot`).
private func flush(_ slot: Slot) {
for bit in GamepadWire.allButtons where slot.buttons & bit != 0 {
connection.send(.gamepadButton(bit, down: false, pad: slot.pad))
}
slot.buttons = 0
for (i, v) in slot.axes.enumerated() where v != 0 {
connection.send(.gamepadAxis(UInt32(i), value: 0, pad: slot.pad))
slot.axes[i] = 0
}
for (f, active) in slot.fingerActive.enumerated() where active {
connection.sendTouchpad(pad: UInt8(slot.pad), finger: UInt8(f), active: false, x: 0, y: 0)
slot.fingerActive[f] = false
}
}
/// Flush every open slot's held state (app deactivation) keeps the slots open (GC just stops
/// delivering; resume re-syncs), disarms the escape chord. Distinct from `closeAllSlots`, which
/// also sends GamepadRemove and detaches handlers.
private func releaseAll() { private func releaseAll() {
chordTimer?.invalidate() chordTimer?.invalidate()
chordTimer = nil chordTimer = nil
for bit in GamepadWire.allButtons where buttons & bit != 0 { for slot in slots { flush(slot) }
connection.send(.gamepadButton(bit, down: false, pad: 0))
}
buttons = 0
for (i, v) in axes.enumerated() where v != 0 {
connection.send(.gamepadAxis(UInt32(i), value: 0, pad: 0))
axes[i] = 0
}
for (f, active) in fingerActive.enumerated() where active {
connection.sendTouchpad(finger: UInt8(f), active: false, x: 0, y: 0)
fingerActive[f] = false
}
} }
} }
@@ -1,20 +1,23 @@
// Hostclient gamepad feedback rendering: one drain thread polls the rumble (0xCA) and // Hostclient gamepad feedback rendering: one drain thread polls the rumble (0xCA) and
// HID-output (0xCD) planes and replays them on the active physical controller // HID-output (0xCD) planes and replays each update on the forwarded physical controller it is
// ADDRESSED TO by wire pad index
// //
// rumble CHHapticEngine players (per-handle localities when the pad has them, // rumble CHHapticEngine players (per-handle localities when the pad has them,
// one combined engine otherwise), // one combined engine otherwise), a RumbleRenderer per pad,
// lightbar GCDeviceLight, // lightbar GCDeviceLight,
// player LEDs GCController.playerIndex (the DS bit patterns map to player 14), // player LEDs GCController.playerIndex (the DS bit patterns map to player 14),
// trigger FX DualSenseTriggerEffect.parse GCDualSenseAdaptiveTrigger. // trigger FX DualSenseTriggerEffect.parse GCDualSenseAdaptiveTrigger.
// //
// Only pad 0 is rendered (exactly one controller is forwarded). HID-output traffic exists // Every forwarded controller gets a per-pad feedback slot (its RumbleRenderer + last light /
// only on PlayStation-pad sessions (a DualSense, or a DualShock 4 = lightbar only) the // player-LED / trigger state) keyed on the same wire index GamepadCapture streams it on, so a
// drain always polls both planes with short timeouts and never spins, so an Xbox session // rumble the host aimed at pad 1 drives pad 1's actuator and nothing else. An update for a pad
// just renders rumble. GameController profile mutation // with no live slot (one that just closed) is dropped. HID-output traffic exists only on
// happens on main; CHHapticEngine work on its own serial queue; the drain thread itself // PlayStation-pad sessions (a DualSense, or a DualShock 4 = lightbar only); the drain always
// touches neither. When GamepadManager switches the active controller mid-session, the // polls both planes with short timeouts and never spins, so an Xbox pad just renders rumble.
// old pad is reset (triggers off, player index unset) and the last known feedback state // GameController profile mutation happens on main; CHHapticEngine work on the renderer's serial
// is replayed onto the new one. // queue; the drain thread itself touches neither (it routes rumble to the pad's renderer under a
// lock and hops HID to main). When a controller leaves the forwarded set the old pad is reset
// (triggers off, player index unset) and its renderer silenced.
import Combine import Combine
import Foundation import Foundation
@@ -22,26 +25,40 @@ import GameController
public final class GamepadFeedback { public final class GamepadFeedback {
private let connection: PunktfunkConnection private let connection: PunktfunkConnection
private let manager: GamepadManager
private let flag = StopFlag() private let flag = StopFlag()
private let drainDone = DispatchSemaphore(value: 0) private let drainDone = DispatchSemaphore(value: 0)
private var drainStarted = false private var drainStarted = false
private let rumble = RumbleRenderer(policy: .session) private var forwardedSub: AnyCancellable?
private var activeSub: AnyCancellable?
// Last applied feedback (main-actor) replayed when the active controller changes. /// One forwarded controller's non-rumble feedback state (main-actor) the GC target plus the
@MainActor private var target: GCController? /// last applied lightbar / player-LED / trigger, replayed if the controller on this pad swaps.
@MainActor private var lastLight: (r: UInt8, g: UInt8, b: UInt8)? @MainActor private final class Slot {
@MainActor private var lastPlayerBits: UInt8? var controller: GCController?
@MainActor private var lastTrigger: [DualSenseTriggerEffect?] = [nil, nil] var lastLight: (r: UInt8, g: UInt8, b: UInt8)?
var lastPlayerBits: UInt8?
var lastTrigger: [DualSenseTriggerEffect?] = [nil, nil]
init(controller: GCController?) { self.controller = controller }
}
/// HID / lightbar / player-LED slots, keyed by wire pad index. Main-actor only.
@MainActor private var slots: [UInt8: Slot] = [:]
/// Rumble renderers keyed by wire pad index, guarded by `routingLock` so the background drain
/// thread can route an incoming envelope to the right pad's renderer while the main actor
/// reconciles the set. RumbleRenderer serializes on its own queue, so calling `apply` from the
/// drain thread is safe only the map lookup needs the lock.
private let routingLock = NSLock()
private var rumbleByPad: [UInt8: RumbleRenderer] = [:]
public init(connection: PunktfunkConnection, manager: GamepadManager) { public init(connection: PunktfunkConnection, manager: GamepadManager) {
self.connection = connection self.connection = connection
self.manager = manager
// Capture self weakly in the hop too, so the inner sink's weak capture isn't shadowing // Capture self weakly in the hop too, so the inner sink's weak capture isn't shadowing
// an implicit strong one and the subscription (stored on self) never retain-cycles. // an implicit strong one and the subscription (stored on self) never retain-cycles.
Task { @MainActor [weak self] in Task { @MainActor [weak self] in
guard let self else { return } guard let self else { return }
self.activeSub = manager.$active.sink { [weak self] dc in self.forwardedSub = manager.$forwarded.sink { [weak self] list in
MainActor.assumeIsolated { self?.retarget(dc?.controller) } MainActor.assumeIsolated { self?.reconcile(list) }
} }
} }
} }
@@ -67,6 +84,38 @@ public final class GamepadFeedback {
} }
} }
/// Bring the per-pad feedback slots in line with the forwarded set: drop pads no longer
/// forwarded (silence + release their renderer, reset their controller), add a slot +
/// renderer for each new pad, and retarget a pad whose controller changed (a re-plug into the
/// same freed index) replaying its cached feedback onto the new device.
@MainActor
private func reconcile(_ forwarded: [GamepadManager.DiscoveredController]) {
var want: [UInt8: GCController] = [:]
for dc in forwarded {
if let pad = manager.padIndex(for: dc) { want[pad] = dc.controller }
}
for (pad, slot) in slots where want[pad] == nil {
reset(slot.controller)
slots[pad] = nil
let renderer = withRouting { rumbleByPad.removeValue(forKey: pad) }
renderer?.stop()
}
for (pad, controller) in want {
if let slot = slots[pad] {
guard slot.controller !== controller else { continue }
reset(slot.controller)
slot.controller = controller
withRouting { rumbleByPad[pad]?.retarget(controller) }
replay(slot)
} else {
slots[pad] = Slot(controller: controller)
let renderer = RumbleRenderer(policy: .session)
renderer.retarget(controller)
withRouting { rumbleByPad[pad] = renderer }
}
}
}
public func start() { public func start() {
guard !drainStarted else { return } guard !drainStarted else { return }
drainStarted = true drainStarted = true
@@ -88,19 +137,19 @@ public final class GamepadFeedback {
// rumble/HID latency low while leaving the lock free between polls. // rumble/HID latency low while leaving the lock free between polls.
// //
// Rumble is idempotent state, so drain the plane DRY and apply only the newest // Rumble is idempotent state, so drain the plane DRY and apply only the newest
// level. The old one-datagram-per-cycle shape let a burst outpace the ~125 Hz // level PER PAD. The old one-datagram-per-cycle shape let a burst outpace the
// drain: levels rendered up to ~130 ms late through the core's 16-deep queue, // ~125 Hz drain: levels rendered up to ~130 ms late through the core's 16-deep
// and its drop-newest overflow could shed a stop while stale nonzero states // queue, and its drop-newest overflow could shed a stop while stale nonzero
// queued ahead of it buzzing until the host's next 500 ms refresh. // states queued ahead of it buzzing until the host's next 500 ms refresh.
var newest: (low: UInt16, high: UInt16, ttl: UInt32)? var newestByPad: [UInt8: (low: UInt16, high: UInt16, ttl: UInt32)] = [:]
var rumbleBurst = 0 var rumbleBurst = 0
while rumbleBurst < 64, !flag.isStopped, while rumbleBurst < 64, !flag.isStopped,
let r = try connection.nextRumble2(timeoutMs: 0) { let r = try connection.nextRumble2(timeoutMs: 0) {
if r.pad == 0 { newest = (r.low, r.high, r.ttlMs) } newestByPad[UInt8(truncatingIfNeeded: r.pad)] = (r.low, r.high, r.ttlMs)
rumbleBurst += 1 rumbleBurst += 1
} }
if let n = newest { for (pad, n) in newestByPad {
self?.rumble.apply(low: n.low, high: n.high, ttlMs: n.ttl) self?.routeRumble(pad: pad, low: n.low, high: n.high, ttlMs: n.ttl)
} }
// Drain a BOUNDED burst of hidout events so sustained 0xCD traffic (a game writing // Drain a BOUNDED burst of hidout events so sustained 0xCD traffic (a game writing
// per-frame LED/trigger reports) can't spin here or block stop() past one cycle. // per-frame LED/trigger reports) can't spin here or block stop() past one cycle.
@@ -126,7 +175,7 @@ public final class GamepadFeedback {
thread.start() thread.start()
} }
/// Stop the drain and silence the motors. Blocks until the drain thread exits ( one /// Stop the drain and silence every pad's motors. Blocks until the drain thread exits ( one
/// poll cycle) call off the main actor, before `connection.close()`. /// poll cycle) call off the main actor, before `connection.close()`.
public func stop() { public func stop() {
flag.stop() flag.stop()
@@ -134,17 +183,32 @@ public final class GamepadFeedback {
drainDone.wait() drainDone.wait()
drainStarted = false drainStarted = false
} }
rumble.stop() let renderers = withRouting { () -> [RumbleRenderer] in
// Drop the retarget subscription and the dead session's cached feedback a let r = Array(rumbleByPad.values)
// controller change after teardown must not replay this session's triggers/LEDs. rumbleByPad.removeAll()
Task { @MainActor in return r
self.activeSub = nil
self.lastLight = nil
self.lastPlayerBits = nil
self.lastTrigger = [nil, nil]
self.reset(self.target)
self.target = nil
} }
for r in renderers { r.stop() }
// Drop the subscription and every dead pad's cached feedback a controller change after
// teardown must not replay this session's triggers/LEDs.
Task { @MainActor in
self.forwardedSub = nil
for slot in self.slots.values { self.reset(slot.controller) }
self.slots.removeAll()
}
}
/// Route one rumble envelope to its pad's renderer (drain thread). An update for a pad with no
/// live renderer one that just left the forwarded set is dropped.
private func routeRumble(pad: UInt8, low: UInt16, high: UInt16, ttlMs: UInt32) {
let renderer = withRouting { rumbleByPad[pad] }
renderer?.apply(low: low, high: high, ttlMs: ttlMs)
}
private func withRouting<R>(_ body: () -> R) -> R {
routingLock.lock()
defer { routingLock.unlock() }
return body()
} }
private func render(_ ev: PunktfunkConnection.HidOutputEvent) { private func render(_ ev: PunktfunkConnection.HidOutputEvent) {
@@ -157,40 +221,37 @@ public final class GamepadFeedback {
private func apply(_ ev: PunktfunkConnection.HidOutputEvent) { private func apply(_ ev: PunktfunkConnection.HidOutputEvent) {
switch ev { switch ev {
case let .led(pad, r, g, b): case let .led(pad, r, g, b):
guard pad == 0 else { return } guard let slot = slots[pad] else { return }
lastLight = (r, g, b) slot.lastLight = (r, g, b)
target?.light?.color = GCColor( slot.controller?.light?.color = GCColor(
red: Float(r) / 255, green: Float(g) / 255, blue: Float(b) / 255) red: Float(r) / 255, green: Float(g) / 255, blue: Float(b) / 255)
case let .playerLEDs(pad, bits): case let .playerLEDs(pad, bits):
guard pad == 0 else { return } guard let slot = slots[pad] else { return }
lastPlayerBits = bits slot.lastPlayerBits = bits
target?.playerIndex = Self.playerIndex(forBits: bits) slot.controller?.playerIndex = Self.playerIndex(forBits: bits)
case let .triggerEffect(pad, which, effect): case let .triggerEffect(pad, which, effect):
guard pad == 0, which < 2 else { return } guard which < 2, let slot = slots[pad] else { return }
let parsed = DualSenseTriggerEffect.parse(effect) let parsed = DualSenseTriggerEffect.parse(effect)
lastTrigger[Int(which)] = parsed slot.lastTrigger[Int(which)] = parsed
if let trigger = adaptiveTrigger(which) { if let trigger = adaptiveTrigger(slot.controller, which) {
parsed.apply(to: trigger) parsed.apply(to: trigger)
} }
} }
} }
/// Replay a pad's cached feedback onto its (swapped-in) controller so a re-plug looks the same.
@MainActor @MainActor
private func retarget(_ controller: GCController?) { private func replay(_ slot: Slot) {
guard controller !== target else { return } if let (r, g, b) = slot.lastLight {
reset(target) slot.controller?.light?.color = GCColor(
target = controller
rumble.retarget(controller)
// Replay the session's feedback state so a swapped-in controller looks the same.
if let (r, g, b) = lastLight {
controller?.light?.color = GCColor(
red: Float(r) / 255, green: Float(g) / 255, blue: Float(b) / 255) red: Float(r) / 255, green: Float(g) / 255, blue: Float(b) / 255)
} }
if let bits = lastPlayerBits { if let bits = slot.lastPlayerBits {
controller?.playerIndex = Self.playerIndex(forBits: bits) slot.controller?.playerIndex = Self.playerIndex(forBits: bits)
} }
for which in 0..<2 { for which in 0..<2 {
if let effect = lastTrigger[which], let trigger = adaptiveTrigger(UInt8(which)) { if let effect = slot.lastTrigger[which],
let trigger = adaptiveTrigger(slot.controller, UInt8(which)) {
effect.apply(to: trigger) effect.apply(to: trigger)
} }
} }
@@ -207,8 +268,8 @@ public final class GamepadFeedback {
} }
@MainActor @MainActor
private func adaptiveTrigger(_ which: UInt8) -> GCDualSenseAdaptiveTrigger? { private func adaptiveTrigger(_ controller: GCController?, _ which: UInt8) -> GCDualSenseAdaptiveTrigger? {
guard let ds = target?.extendedGamepad as? GCDualSenseGamepad else { return nil } guard let ds = controller?.extendedGamepad as? GCDualSenseGamepad else { return nil }
return which == 0 ? ds.leftTrigger : ds.rightTrigger return which == 0 ? ds.leftTrigger : ds.rightTrigger
} }
} }
@@ -1,14 +1,18 @@
// Controller discovery + selection, app-lifetime. One GamepadManager (`.shared`) watches // Controller discovery + selection, app-lifetime. One GamepadManager (`.shared`) watches
// GCController connect/disconnect from launch, so the Settings page shows live controller // GCController connect/disconnect from launch, so the Settings page shows live controller
// state without a session, and the session components (GamepadCapture / GamepadFeedback) // state without a session, and the session components (GamepadCapture / GamepadFeedback)
// follow `active` exactly ONE physical controller is forwarded to the host, as pad 0. // follow `forwarded` every forwarded controller is streamed to the host, each on its own
// wire pad index (pf-client-core parity; up to `GamepadWire.maxPads`).
// //
// Selection: the user can pin a controller in Settings (persisted under // Selection (mirrors pf-client-core's `forwarded_ids` + slot model): with no pin, EVERY
// DefaultsKey.gamepadID); with no pin or the pinned one absent the most recently // extended controller is forwarded each assigned a stable lowest-free pad index held for
// connected extended gamepad wins. GCController has no stable hardware serial, so the pin // its forwarded lifetime, so a disconnect frees only its own index and never renumbers the
// is a fingerprint of vendorName|productCategory (+ a connect-order suffix for twins); // others. A pin (Settings, persisted under DefaultsKey.gamepadID) forwards ONLY that one pad
// identical twin controllers may swap a pin across reconnects, which the Settings footer // an explicit single-player choice. `active` stays the single "primary" pad (the pinned
// documents. // one, else the most recently connected extended gamepad) that the Settings / launcher / menu
// UI reads. GCController has no stable hardware serial, so the pin is a fingerprint of
// vendorName|productCategory (+ a connect-order suffix for twins); identical twin controllers
// may swap a pin across reconnects, which the Settings footer documents.
// //
// A singleton (not a SwiftUI environment object) because macOS shows Settings in its own // A singleton (not a SwiftUI environment object) because macOS shows Settings in its own
// `Settings{}` scene there is no common ancestor view to inject from. // `Settings{}` scene there is no common ancestor view to inject from.
@@ -60,9 +64,23 @@ public final class GamepadManager: ObservableObject {
/// Every detected controller, in connect order (Settings lists these). /// Every detected controller, in connect order (Settings lists these).
@Published public private(set) var controllers: [DiscoveredController] = [] @Published public private(set) var controllers: [DiscoveredController] = []
/// The one controller forwarded to the host (pad 0); nil when none qualifies. /// The single "primary" controller the pinned one, else the most recently connected
/// extended gamepad; nil when none qualifies. The Settings / launcher / menu UI and the
/// connect-time `resolveType` read this; the streaming input path uses `forwarded`.
@Published public private(set) var active: DiscoveredController? @Published public private(set) var active: DiscoveredController?
/// The controllers forwarded to the host this session, in wire-pad-index preference order
/// (pf-client-core's `forwarded_ids`): a pin forwards ONLY the pinned pad; Automatic forwards
/// every extended controller. GamepadCapture opens a slot per entry and GamepadFeedback routes
/// feedback back to it, each on the index from `padIndex(for:)`.
@Published public private(set) var forwarded: [DiscoveredController] = []
/// Stable wire pad index (0..<`GamepadWire.maxPads`) per forwarded controller, keyed by
/// GCController identity. Lowest-free, held while the controller stays forwarded a
/// disconnect frees only its own index so the others never renumber (pf-client-core's
/// `lowest_free_index`). Recomputed by `assignPadIndices` whenever `forwarded` changes.
private var padIndexByController: [ObjectIdentifier: UInt8] = [:]
/// The user's pinned controller fingerprint ("" = automatic). Persisted; updating it /// The user's pinned controller fingerprint ("" = automatic). Persisted; updating it
/// reselects immediately, so a Settings Picker can bind straight to this. /// reselects immediately, so a Settings Picker can bind straight to this.
@Published public var preferredID: String { @Published public var preferredID: String {
@@ -159,7 +177,52 @@ public final class GamepadManager: ObservableObject {
let candidates = controllers.filter(\.isExtended) let candidates = controllers.filter(\.isExtended)
// The pin wins when present; otherwise the most recently connected extended pad // The pin wins when present; otherwise the most recently connected extended pad
// (list is in connect order). A stale pin falls back to automatic. // (list is in connect order). A stale pin falls back to automatic.
active = candidates.last { $0.id == preferredID } ?? candidates.last let pinned = candidates.last { $0.id == preferredID }
active = pinned ?? candidates.last
// Forwarded set (pf-client-core's `forwarded_ids`): a pin forwards ONLY the pinned pad
// (explicit single-player); Automatic forwards every extended controller in connect order
// (oldestnewest), so a game's player numbers are stable across hot-plug churn.
let next = pinned.map { [$0] } ?? candidates
// Update the pad-index assignment BEFORE publishing `forwarded`: @Published emits in
// `willSet`, so GamepadCapture/GamepadFeedback reconcile against `padIndex(for:)` the
// instant this assignment lands a stale map here would skip a newly-forwarded pad.
assignPadIndices(for: next)
forwarded = next
}
/// Assign each forwarded controller a stable wire pad index (lowest-free, held while it stays
/// forwarded) mirrors pf-client-core's slot model, where a disconnect frees only its own
/// index and the others keep theirs. A controller already holding an index keeps it across the
/// churn; a slot beyond `GamepadWire.maxPads` goes unassigned (that pad is not forwarded).
private func assignPadIndices(for next: [DiscoveredController]) {
let live = Set(next.map { ObjectIdentifier($0.controller) })
padIndexByController = padIndexByController.filter { live.contains($0.key) }
for dc in next {
let key = ObjectIdentifier(dc.controller)
guard padIndexByController[key] == nil,
let free = Self.lowestFreeIndex(Set(padIndexByController.values)) else { continue }
padIndexByController[key] = free
}
}
/// The lowest wire pad index not already taken, or nil when all `GamepadWire.maxPads` are in
/// use (pf-client-core's `lowest_free_index`).
private static func lowestFreeIndex(_ taken: Set<UInt8>) -> UInt8? {
(0..<UInt8(GamepadWire.maxPads)).first { !taken.contains($0) }
}
/// The wire pad index a forwarded controller streams on, or nil when it isn't forwarded.
public func padIndex(for controller: DiscoveredController) -> UInt8? {
padIndexByController[ObjectIdentifier(controller.controller)]
}
/// Drop every pad-index assignment and recompute from the current forwarded set called when
/// a streaming session begins so the assignment starts fresh (a controller pinned before the
/// session forwards as pad 0, not whatever index it held for the Settings list). pf-client-core
/// assigns indices at slot-open time; this reproduces that session-scoped start.
public func resetForwardingAssignment() {
padIndexByController.removeAll()
reselect()
} }
private static func describe(_ c: GCController, id: String) -> DiscoveredController { private static func describe(_ c: GCController, id: String) -> DiscoveredController {
@@ -1,10 +1,14 @@
// The gamepad wire contract shared by capture (GamepadCapture), feedback (GamepadFeedback), // The gamepad wire contract shared by capture (GamepadCapture), feedback (GamepadFeedback),
// and the tests button bits, axis ids, and the touchpad/motion unit conversions. // and the tests the pad count, button bits, axis ids, and the touchpad/motion unit conversions.
import Foundation import Foundation
/// The gamepad wire contract (mirrors `punktfunk_core::input::gamepad`). /// The gamepad wire contract (mirrors `punktfunk_core::input::gamepad`).
public enum GamepadWire { public enum GamepadWire {
/// Gamepads addressable on the wire the pad index rides the low byte of `flags` on every
/// per-pad event, 0...15 (`punktfunk_core::input::MAX_PADS`).
public static let maxPads: Int = 16
public static let dpadUp: UInt32 = 0x0001 public static let dpadUp: UInt32 = 0x0001
public static let dpadDown: UInt32 = 0x0002 public static let dpadDown: UInt32 = 0x0002
public static let dpadLeft: UInt32 = 0x0004 public static let dpadLeft: UInt32 = 0x0004
@@ -85,6 +85,12 @@ public final class InputCapture {
/// its Esc suppression need it in both states). /// its Esc suppression need it in both states).
private var cmdKeysDown: Set<UInt32> = [] private var cmdKeysDown: Set<UInt32> = []
/// Physical Control/Option/Shift keys currently held (Windows VKs, both L/R sides). iPad only:
/// the Q release chord is recognized from the HID stream here (iOS has no NSEvent monitor,
/// like the toggle), so it needs the live modifier state tracked in both forwarding states,
/// exactly like `cmdKeysDown`, and flushed by `releaseAll` when GC delivery stops.
private var chordModifiersDown: Set<UInt32> = []
/// While true, mouse/keyboard flow to the host and key NSEvents are swallowed /// While true, mouse/keyboard flow to the host and key NSEvents are swallowed
/// locally; while false the user is interacting with the local UI (dragging the /// locally; while false the user is interacting with the local UI (dragging the
/// window, clicking the HUD) and nothing is forwarded. Main-queue only. /// window, clicking the HUD) and nothing is forwarded. Main-queue only.
@@ -119,6 +125,21 @@ public final class InputCapture {
public var onDisconnect: (() -> Void)? public var onDisconnect: (() -> Void)?
public var onCycleStats: (() -> Void)? public var onCycleStats: (() -> Void)?
#if os(iOS)
/// Windows VKs of the three modifier classes in the Q release chord, both L/R sides:
/// control (0xA2/0xA3), option (0xA4/0xA5), shift (0xA0/0xA1). Used to sift the HID key stream.
private static let chordModifierVKs: Set<UInt32> = [0xA2, 0xA3, 0xA4, 0xA5, 0xA0, 0xA1]
/// Whether Control AND Option AND Shift are all currently held (either side of each counts)
/// the modifier precondition for the iPad Q release chord.
private var hasReleaseChordModifiers: Bool {
let m = chordModifiersDown
return (m.contains(0xA2) || m.contains(0xA3)) // control
&& (m.contains(0xA4) || m.contains(0xA5)) // option
&& (m.contains(0xA0) || m.contains(0xA1)) // shift
}
#endif
/// Fired when a newer InputCapture takes the process-global GC handler slots (the /// Fired when a newer InputCapture takes the process-global GC handler slots (the
/// singletons hold ONE handler each): the preempted owner must drop its capture /// singletons hold ONE handler each): the preempted owner must drop its capture
/// state its handlers are gone, so it would otherwise sit "captured" with dead /// state its handlers are gone, so it would otherwise sit "captured" with dead
@@ -294,6 +315,7 @@ public final class InputCapture {
/// in another app would otherwise stay "held" here forever hijacking Esc). /// in another app would otherwise stay "held" here forever hijacking Esc).
private func releaseAll() { private func releaseAll() {
cmdKeysDown.removeAll() cmdKeysDown.removeAll()
chordModifiersDown.removeAll()
suppressedVK = nil suppressedVK = nil
for vk in pressedVKs { for vk in pressedVKs {
connection.send(.key(vk, down: false)) connection.send(.key(vk, down: false))
@@ -576,6 +598,13 @@ public final class InputCapture {
self.cmdKeysDown.remove(vk) self.cmdKeysDown.remove(vk)
} }
} }
#if os(iOS)
// Track Control/Option/Shift for the Q release chord below in both forwarding
// states (like `cmdKeysDown`) so a modifier held before capture engaged still counts.
if Self.chordModifierVKs.contains(vk) {
if pressed { self.chordModifiersDown.insert(vk) } else { self.chordModifiersDown.remove(vk) }
}
#endif
// The toggle's Esc checked before the forwarding gate, because in the // The toggle's Esc checked before the forwarding gate, because in the
// engage direction forwarding is already true when this fires. // engage direction forwarding is already true when this fires.
if vk == self.suppressedVK { if vk == self.suppressedVK {
@@ -592,6 +621,18 @@ public final class InputCapture {
} }
#endif #endif
guard self.forwarding else { return } guard self.forwarding else { return }
#if os(iOS)
// Q releases the captured mouse/keyboard (cross-client parity the same combo the
// macOS keyDown monitor handles). Recognized only while forwarding (nothing to release
// otherwise). The Q is latched (`suppressedVK`) so its keyUp can't type into the host;
// the modifiers were forwarded as they went down and are flushed by the release
// path (setCaptured(false) releaseAll). VK 0x51 is layout-independent (physical Q).
if pressed, vk == 0x51, self.hasReleaseChordModifiers {
self.suppressedVK = 0x51
self.onReleaseCapture?()
return
}
#endif
// Release direction of the toggle: GC's Esc-down can beat the NSEvent // Release direction of the toggle: GC's Esc-down can beat the NSEvent
// monitor never type Esc into the host while is held ( is reserved). // monitor never type Esc into the host while is held ( is reserved).
if vk == 0x1B, !self.cmdKeysDown.isEmpty { if vk == 0x1B, !self.cmdKeysDown.isEmpty {
@@ -11,6 +11,15 @@ public enum DefaultsKey {
public static let streamWidth = "punktfunk.width" public static let streamWidth = "punktfunk.width"
public static let streamHeight = "punktfunk.height" public static let streamHeight = "punktfunk.height"
public static let streamHz = "punktfunk.hz" public static let streamHz = "punktfunk.hz"
/// Match-window resolution policy (design/midstream-resolution-resize.md D1/D2): when on, the
/// stream mode FOLLOWS the session view the connect asks for the view's pixel size and a
/// mid-session resize (a windowed macOS window, an iPad Stage Manager / Split View scene)
/// renegotiates the host's virtual display + encoder (`PunktfunkConnection.requestMode`), so a
/// windowed session streams native-resolution pixels instead of scaling. Off (default): the
/// explicit `streamWidth`/`streamHeight` are used and never auto-resized (a fullscreen session
/// is native either way, so this degenerates to Auto-native there). Read per session by the
/// stream views' `MatchWindowFollower`.
public static let matchWindow = "punktfunk.matchWindow"
public static let compositor = "punktfunk.compositor" public static let compositor = "punktfunk.compositor"
public static let gamepadType = "punktfunk.gamepadType" public static let gamepadType = "punktfunk.gamepadType"
public static let gamepadID = "punktfunk.gamepadID" public static let gamepadID = "punktfunk.gamepadID"
@@ -88,6 +97,12 @@ public enum DefaultsKey {
/// layout (the console launcher, gamepad-navigable settings, a coverflow-style library) /// layout (the console launcher, gamepad-navigable settings, a coverflow-style library)
/// whenever a gamepad is connected. On by default; see `GamepadUIEnvironment.isActive`. /// whenever a gamepad is connected. On by default; see `GamepadUIEnvironment.isActive`.
public static let gamepadUIEnabled = "punktfunk.gamepadUIEnabled" public static let gamepadUIEnabled = "punktfunk.gamepadUIEnabled"
/// Auto-wake on connect: when connecting to a saved host that isn't advertising on mDNS, fire
/// Wake-on-LAN and, if the dial fails, wait for it to come back before retrying (the "Waking"
/// overlay). On by default. Turn off if a host that's already on just isn't seen on mDNS (a
/// routed/VPN host), so connects go straight through instead of waiting out the wake timeout.
/// The explicit "Wake Host" action stays available regardless. Read by ContentView.startSession.
public static let autoWake = "punktfunk.autoWake"
} }
extension Notification.Name { extension Notification.Name {
@@ -0,0 +1,153 @@
// Match-window resize follower (design/midstream-resolution-resize.md D1/D2, client C3).
//
// The presenting view feeds this its PHYSICAL-PIXEL size on every layout; it debounces to
// resize-end, spaces requests 1 s apart, and asks the connection to switch the host's virtual
// display + encoder to match (`PunktfunkConnection.requestMode`) so a windowed macOS session or
// an iPad Stage Manager / Split View scene streams native-resolution pixels instead of scaling.
// The decode/present side needs nothing: VideoToolbox recreates its session on the keyframe-derived
// format-description change (the first new-mode AU is an IDR with fresh parameter sets).
//
// The trigger discipline is the shared cross-client one (mirrors the session binary's
// `resize_decision`): physical pixels rounded DOWN to even (the host rejects odd dimensions) and
// clamped 320×200; debounce to resize-end; 1 s between requests; skip a size equal to the live
// mode; and request each distinct size at most once which both stops re-asking a rejected size
// and keeps a host-side rollback (accepted, rebuild failed, corrective ack restored the old mode)
// from looping request rollback request.
import Foundation
/// The pure, side-effect-free core of the Match-window trigger so the normalize/skip discipline
/// is unit-tested without a live connection or a UI (`MatchWindowTests`).
public enum MatchWindow {
/// Even-floor + clamp a physical-pixel size to a host-valid mode dimension: the host's
/// `validate_dimensions` rejects odd sizes, and we never ask below 320×200.
public static func normalize(widthPx: Int, heightPx: Int) -> (width: UInt32, height: UInt32) {
let evenClamp: (Int, UInt32) -> UInt32 = { px, minimum in
let even = UInt32(max(px, 0)) / 2 * 2
return max(even, minimum)
}
return (evenClamp(widthPx, 320), evenClamp(heightPx, 200))
}
/// Whether to request `target` now (the debounce has already settled; spacing is the caller's
/// timer): `nil` to skip equal to the live mode, or already requested once (a rejected size /
/// a host rollback must not loop). `target` is expected already-[normalize]d.
public static func request(
target: (width: UInt32, height: UInt32),
current: (width: UInt32, height: UInt32),
lastRequested: (width: UInt32, height: UInt32)?
) -> (width: UInt32, height: UInt32)? {
if target.width == current.width, target.height == current.height { return nil }
if let lr = lastRequested, lr.width == target.width, lr.height == target.height { return nil }
return target
}
}
/// Owns the debounce timer + serialization state and drives `PunktfunkConnection.requestMode` from
/// the stream view's layout callbacks. Main-actor: the views feed it on the main thread and it reads
/// the connection's live mode there. Enabled per session from the `matchWindow` setting.
@MainActor
public final class MatchWindowFollower {
private weak var connection: PunktfunkConnection?
private let debounce: TimeInterval
private let minSpacing: TimeInterval
private var enabled: Bool
private var work: DispatchWorkItem?
private var pendingSize: (width: Int, height: Int)?
private var lastRequested: (width: UInt32, height: UInt32)?
private var lastRequestAt: Date?
/// The last size we reported via [`onResizeTarget`] dedups the per-layout stream of a drag so
/// the UI is notified once per distinct target, and reset to `nil` when the window is back in
/// sync with the live mode (so a later resize re-reports).
private var lastSteered: (width: UInt32, height: UInt32)?
/// Fired (on the main actor) the instant the window starts differing from the live mode i.e.
/// a resize is under way and a `Reconfigure` for `(width, height)` is imminent. Drives the
/// resize overlay's INSTANT feedback (blur + spinner) BEFORE the debounced request leaves; the
/// overlay clears when a decoded frame reaches this size (or on a timeout). Deduped per target.
public var onResizeTarget: ((_ width: UInt32, _ height: UInt32) -> Void)?
/// `debounce` = quiet time after the last size event before requesting (Win32 gets
/// `WM_EXITSIZEMOVE` for free; we debounce). `minSpacing` = floor between accepted requests
/// (a full host pipeline rebuild each). Defaults match the other clients.
public init(
connection: PunktfunkConnection,
enabled: Bool,
debounce: TimeInterval = 0.4,
minSpacing: TimeInterval = 1.0
) {
self.connection = connection
self.enabled = enabled
self.debounce = debounce
self.minSpacing = minSpacing
}
/// Turn following on/off live (a mid-session settings change; off cancels a pending request).
public func setEnabled(_ on: Bool) {
enabled = on
if !on {
work?.cancel()
work = nil
pendingSize = nil
lastSteered = nil
}
}
/// Feed the presenting view's current PHYSICAL-PIXEL size (its `bounds` × the backing/display
/// scale). Called from every layout pass; coalesced by the debounce so a drag-resize sends one
/// request at its end, never one per frame.
public func noteSize(widthPx: Int, heightPx: Int) {
guard enabled else { return }
pendingSize = (widthPx, heightPx)
schedule()
reportSteering(widthPx: widthPx, heightPx: heightPx)
}
/// Report the resize overlay's START signal (deduped): the moment the normalized window size
/// differs from the live mode we're steering toward a new size. No connection / no negotiated
/// mode yet nothing to compare against, skip.
private func reportSteering(widthPx: Int, heightPx: Int) {
guard let connection else { return }
let target = MatchWindow.normalize(widthPx: widthPx, heightPx: heightPx)
let mode = connection.currentMode()
guard mode.width > 0, mode.height > 0 else { return }
if target.width == mode.width, target.height == mode.height {
lastSteered = nil // back in sync a later change re-reports
return
}
if lastSteered?.width == target.width, lastSteered?.height == target.height { return }
lastSteered = target
onResizeTarget?(target.width, target.height)
}
private func schedule() {
work?.cancel()
let item = DispatchWorkItem { [weak self] in self?.fire() }
work = item
DispatchQueue.main.asyncAfter(deadline: .now() + debounce, execute: item)
}
private func fire() {
guard enabled, let connection, let size = pendingSize else { return }
// 1 s spacing: a request went out recently re-arm the debounce and retry later rather
// than fire early (keeps at most ~one request outstanding the accept ack round-trips in
// milliseconds, ahead of the host's rebuild).
if let last = lastRequestAt, Date().timeIntervalSince(last) < minSpacing {
schedule()
return
}
let target = MatchWindow.normalize(widthPx: size.width, heightPx: size.height)
let mode = connection.currentMode()
pendingSize = nil
guard let req = MatchWindow.request(
target: target,
current: (mode.width, mode.height),
lastRequested: lastRequested
) else { return }
// Keep the current refresh Match-window follows SIZE, not rate.
connection.requestMode(width: req.width, height: req.height, refreshHz: mode.refreshHz)
lastRequested = req
lastRequestAt = Date()
}
}
@@ -124,7 +124,16 @@ float2 chromaUV(texture2d<float> lumaTex, texture2d<float> chromaTex, float2 uv)
float3 sampleRgb(texture2d<float> lumaTex, texture2d<float> chromaTex, float2 uv, float3 sampleRgb(texture2d<float> lumaTex, texture2d<float> chromaTex, float2 uv,
constant CscUniform& csc) { constant CscUniform& csc) {
constexpr sampler s(filter::linear, address::clamp_to_edge); constexpr sampler s(filter::linear, address::clamp_to_edge);
float3 yuv = float3(catmullRomLuma(lumaTex, s, uv), #ifdef PF_BILINEAR_LUMA
// DEBUG (PUNKTFUNK_BILINEAR_LUMA=1): plain bilinear luma Catmull-Rom OFF. A/B lever to see if
// the bicubic overshoot contributes to edge fringing. NOTE: at a true 1:1 present both paths
// reduce to the identity texel, so if this toggle VISIBLY changes the picture, the present is
// NOT 1:1 (there's a resample); if it looks identical, the fringing is upstream (codec/source/OS).
float lumaY = lumaTex.sample(s, uv).r;
#else
float lumaY = catmullRomLuma(lumaTex, s, uv);
#endif
float3 yuv = float3(lumaY,
chromaTex.sample(s, chromaUV(lumaTex, chromaTex, uv)).rg); chromaTex.sample(s, chromaUV(lumaTex, chromaTex, uv)).rg);
return saturate(float3(dot(csc.r0.xyz, yuv) + csc.r0.w, return saturate(float3(dot(csc.r0.xyz, yuv) + csc.r0.w,
dot(csc.r1.xyz, yuv) + csc.r1.w, dot(csc.r1.xyz, yuv) + csc.r1.w,
@@ -250,7 +259,16 @@ public final class MetalVideoPresenter {
let pipelineHDR: MTLRenderPipelineState let pipelineHDR: MTLRenderPipelineState
let pipelineHDRToneMap: MTLRenderPipelineState? let pipelineHDRToneMap: MTLRenderPipelineState?
do { do {
let library = try device.makeLibrary(source: shaderSource, options: nil) // DEBUG A/B lever: PUNKTFUNK_BILINEAR_LUMA=1 compiles the shader with Catmull-Rom OFF
// (plain bilinear luma) by prepending a #define ahead of the source. Default (unset) is
// the normal bicubic path. Read at presenter creation set it in the environment and
// relaunch to flip; the log line confirms which path built.
let bilinearLuma = ProcessInfo.processInfo.environment["PUNKTFUNK_BILINEAR_LUMA"] == "1"
let source = (bilinearLuma ? "#define PF_BILINEAR_LUMA 1\n" : "") + shaderSource
if bilinearLuma {
presenterLog.info("stage2: PUNKTFUNK_BILINEAR_LUMA=1 — Catmull-Rom luma DISABLED (bilinear)")
}
let library = try device.makeLibrary(source: source, options: nil)
let vtx = library.makeFunction(name: "pf_vtx") let vtx = library.makeFunction(name: "pf_vtx")
let sdr = MTLRenderPipelineDescriptor() let sdr = MTLRenderPipelineDescriptor()
sdr.vertexFunction = vtx sdr.vertexFunction = vtx
@@ -590,8 +608,17 @@ public final class MetalVideoPresenter {
let sig = "\(Int(decoded.width))x\(Int(decoded.height))\(Int(drawable.width))x\(Int(drawable.height))|hdr\(hdrActive ? 1 : 0)" let sig = "\(Int(decoded.width))x\(Int(decoded.height))\(Int(drawable.width))x\(Int(drawable.height))|hdr\(hdrActive ? 1 : 0)"
if sig != lastSizeSig { if sig != lastSizeSig {
lastSizeSig = sig lastSizeSig = sig
// Explicit verdict: is the shader presenting 1:1 (decoded == drawable) or resampling? The
// scale ratio makes a residual match-window mismatch obvious. If this says 1:1 but the
// picture is still soft, the resample is downstream of us (macOS compositor a scaled
// display mode, or a fractional-pixel window position), not the shader.
let sx = decoded.width > 0 ? drawable.width / decoded.width : 0
let sy = decoded.height > 0 ? drawable.height / decoded.height : 0
let verdict = decoded == drawable
? "1:1 (no resample)"
: String(format: "RESAMPLE scale=%.4fx%.4f", sx, sy)
let msg = let msg =
"stage2: decoded \(Int(decoded.width))x\(Int(decoded.height)) → drawable \(Int(drawable.width))x\(Int(drawable.height)) hdr=\(hdrActive)" "stage2: decoded \(Int(decoded.width))x\(Int(decoded.height)) → drawable \(Int(drawable.width))x\(Int(drawable.height)) [\(verdict)] hdr=\(hdrActive)"
presenterLog.info("\(msg, privacy: .public)") presenterLog.info("\(msg, privacy: .public)")
} }
} }
@@ -0,0 +1,99 @@
// Swift wrapper around the punktfunk-core C ABI's post-loss re-anchor gate
// (`punktfunk_reanchor_gate_*`, ABI v6). The shared Rust gate (crates/punktfunk-core/src/reanchor.rs)
// is what the Linux/Windows desktop pump and the Android client use directly; the Swift clients reach
// it across the C ABI so the freeze-until-reanchor policy is defined ONCE for every platform.
//
// Why a freeze at all: after unrecoverable loss the host keeps sending delta frames that reference a
// picture the client never got. Hardware decoders (VideoToolbox included) don't reliably error on
// that they CONCEAL, returning a gray/garbage frame with a success status. Presenting those is the
// visible "gray flash with motion" of the loss reports. The gate withholds concealed frames and holds
// the last good picture on glass until a PROVEN clean re-anchor lands an IDR (wire `FLAG_SOF`), an
// RFI recovery anchor (`USER_FLAG_RECOVERY_ANCHOR`), or the 2nd of two intra-refresh recovery marks
// (`USER_FLAG_RECOVERY_POINT`) with a bounded backstop so a lost re-anchor can never freeze forever.
// See punktfunk-planning design/client-reanchor-freeze-parity.md.
//
// Threading: one gate per session. Its calls arrive from two threads the pump thread (`arm` on a
// frame-index gap / a submit failure, `poll` per iteration) and a VideoToolbox decode thread
// (`onDecoded` per decoded frame, `onNoOutput` on a decode error). The raw Rust gate is a plain
// struct behind an opaque pointer with no internal synchronization, so every call is serialized under
// `lock` here the calls are cheap field updates, so contention is negligible. `@unchecked Sendable`:
// the lock enforces the contract.
import Foundation
import PunktfunkCore
final class ReanchorGate: @unchecked Sendable {
private let lock = NSLock()
/// The opaque `ReanchorGate *`. `var` so `reseed` can swap it at session start. Never NULL
/// (`punktfunk_reanchor_gate_new` never returns NULL).
private var ptr: OpaquePointer
/// Seed the baseline with the connection's current `framesDropped` so the first `poll` doesn't
/// read the session's starting drop count as a fresh loss.
init(framesDropped: UInt64) {
ptr = punktfunk_reanchor_gate_new(framesDropped)
}
deinit { punktfunk_reanchor_gate_free(ptr) }
/// Re-anchor the drop-count baseline to `framesDropped` for a (re)started session. The gate is
/// created in the pipeline's init (before a connection exists, seeded 0); `start` calls this once
/// the live connection's count is known so a mid-life connection's non-zero baseline isn't
/// mistaken for loss on the first poll.
func reseed(framesDropped: UInt64) {
lock.lock()
defer { lock.unlock() }
punktfunk_reanchor_gate_free(ptr)
ptr = punktfunk_reanchor_gate_new(framesDropped)
}
/// Arm the freeze: a loss was detected (a frame-index gap, or a decoder wedge). Zeroes the
/// recovery-mark count and (re)sets the backstop deadline.
func arm() {
lock.lock()
punktfunk_reanchor_gate_arm(ptr)
lock.unlock()
}
/// Fold one decoded frame. `flags` is the AU's wire `user_flags`. Returns true to PRESENT the
/// frame, false to WITHHOLD it as a post-loss concealment (hold the last good picture). Pass
/// `decoderKeyframe: false` VideoToolbox doesn't flag IDRs, so the wire `FLAG_SOF` covers it.
func onDecoded(flags: UInt32, decoderKeyframe: Bool = false) -> Bool {
lock.lock()
defer { lock.unlock() }
var present = false
_ = punktfunk_reanchor_gate_on_decoded(ptr, flags, decoderKeyframe, &present)
return present
}
/// A received AU produced no decoded frame (a VideoToolbox decode error). Returns true when the
/// no-output streak has tripped (the gate armed the freeze) and the caller should throttled
/// request a keyframe.
func onNoOutput() -> Bool {
lock.lock()
defer { lock.unlock() }
var requestKf = false
_ = punktfunk_reanchor_gate_on_no_output(ptr, &requestKf)
return requestKf
}
/// Periodic fold of the session's `framesDropped` plus the overdue backstop. Returns true when the
/// caller should throttled request a keyframe (a drop-count climb armed a fresh freeze, or the
/// freeze is overdue and re-asks while it keeps holding).
func poll(framesDropped: UInt64) -> Bool {
lock.lock()
defer { lock.unlock() }
var requestKf = false
_ = punktfunk_reanchor_gate_poll(ptr, framesDropped, &requestKf)
return requestKf
}
/// Whether the gate is currently withholding concealed frames (frozen on the last good picture).
var isHolding: Bool {
lock.lock()
defer { lock.unlock() }
var holding = false
_ = punktfunk_reanchor_gate_is_holding(ptr, &holding)
return holding
}
}
@@ -0,0 +1,63 @@
// Resize-in-progress indicator state (design/midstream-resolution-resize.md client UX).
//
// A mid-stream resize takes the host 0.32 s to rebuild its virtual display + encoder, and the
// first new-mode frame is an IDR that the decoder re-inits on. Rather than let the stream scale
// (stretch/blur) to the changing window during that gap, the client EMBRACES the delay: it shows a
// deliberate blur + spinner the instant a resize starts and clears it the instant the sharp
// new-resolution frame is on screen so the wait reads as intentional, not as lag.
//
// This is driven ENTIRELY by signals the client already has (no new protocol):
// * START the Match-window follower reports the size it is steering toward (instant, on the
// first resize layout, before the debounced request even leaves).
// * END the decode pipeline reports each new-mode IDR's dimensions; when they reach the target
// the new picture is here.
// * TIMEOUT the safety net for a switch that never delivers the exact target: the host rejected
// it (gamescope), capped it to an advertised mode, or a corrective ack landed a different size.
//
// Pure + side-effect-free so the transition logic is unit-tested without a live session or UI
// (`ResizeIndicatorTests`); `SessionModel` owns an instance and mirrors `active` into a @Published.
import Foundation
/// The pure state of the resize overlay. `now` is a monotonic time in seconds (the caller passes
/// `ProcessInfo.processInfo.systemUptime` or a test clock).
public struct ResizeIndicator {
/// Whether the blur + spinner should be shown.
public private(set) var active = false
/// The size the follower is steering toward cleared once a decoded frame reaches it.
private var target: (width: UInt32, height: UInt32)?
/// When the current `active` span began the timeout is measured from here.
private var since: TimeInterval?
/// How long to keep the overlay up if the target frame never arrives (rejected / capped switch).
public var timeout: TimeInterval
public init(timeout: TimeInterval = 2.5) { self.timeout = timeout }
/// The follower is steering toward `width`×`height` a resize is under way. Show the overlay now
/// (instant feedback). Called only for a genuine change (the follower skips a target equal to the
/// live mode), possibly many times as a drag moves through sizes; the timeout re-arms whenever the
/// target actually changes so a slow drag never trips it mid-gesture.
public mutating func steering(width: UInt32, height: UInt32, now: TimeInterval) {
if !active || target?.width != width || target?.height != height {
since = now
}
target = (width, height)
active = true
}
/// A decoded frame arrived at `width`×`height` (a new-mode IDR). Clears the overlay once it
/// matches the steered target the sharp new-resolution picture is on glass.
public mutating func decoded(width: UInt32, height: UInt32) {
guard active, let t = target, t.width == width, t.height == height else { return }
active = false
since = nil
}
/// Timeout safety net: stop showing the overlay once `timeout` has elapsed with no matching frame
/// (a rejected or host-capped switch never delivers the exact target).
public mutating func tick(now: TimeInterval) {
guard active, let s = since, now - s >= timeout else { return }
active = false
since = nil
}
}
@@ -70,6 +70,15 @@ final class SessionPresenter {
private var stage2Link: CADisplayLink? private var stage2Link: CADisplayLink?
private var metalLayer: CAMetalLayer? private var metalLayer: CAMetalLayer?
private var connection: PunktfunkConnection? private var connection: PunktfunkConnection?
/// The decoded frame's REAL pixel dimensions (ground truth, pushed by the view from the pump's
/// `onDecodedSize` new-mode-IDR callback). Used for the aspect-fit in `layout` in preference to
/// `connection.currentMode()`, which (a) lags a mid-stream resize it only updates on the
/// `Reconfigured` ack, and a resize-END produces no bounds change to re-run `layout` afterward
/// and (b) can disagree with what the host actually DELIVERED (Windows corrective-ack falls back
/// to an advertised mode). The pixels we're drawing are the only correct aspect source; a wrong
/// one here is the "black bars + stretched" resize artifact. nil until the first frame `layout`
/// falls back to `currentMode()`. Main-thread only.
private var contentSize: CGSize?
/// Start the presenter for `connection`. `baseLayer` is the view's AVSampleBufferDisplayLayer: /// Start the presenter for `connection`. `baseLayer` is the view's AVSampleBufferDisplayLayer:
/// stage-1 enqueues into it; stage-2 leaves it idle and composites an opaque CAMetalLayer /// stage-1 enqueues into it; stage-2 leaves it idle and composites an opaque CAMetalLayer
@@ -85,7 +94,8 @@ final class SessionPresenter {
displayMeter: LatencyMeter? = nil, displayMeter: LatencyMeter? = nil,
makeDisplayLink: (AnyObject, Selector) -> CADisplayLink, makeDisplayLink: (AnyObject, Selector) -> CADisplayLink,
onFrame: (@Sendable (AccessUnit) -> Void)?, onFrame: (@Sendable (AccessUnit) -> Void)?,
onSessionEnd: (@Sendable () -> Void)? onSessionEnd: (@Sendable () -> Void)?,
onDecodedSize: (@Sendable (Int, Int) -> Void)? = nil
) { ) {
stop() stop()
self.connection = connection self.connection = connection
@@ -128,12 +138,14 @@ final class SessionPresenter {
link.add(to: .main, forMode: .common) link.add(to: .main, forMode: .common)
stage2Link = link stage2Link = link
syncFrameRate(hz: connection.currentMode().refreshHz) syncFrameRate(hz: connection.currentMode().refreshHz)
pipeline.start(connection: connection, onFrame: onFrame, onSessionEnd: onSessionEnd) pipeline.start(
connection: connection, onFrame: onFrame, onSessionEnd: onSessionEnd,
onDecodedSize: onDecodedSize)
} else { } else {
let pump = StreamPump() let pump = StreamPump()
pump.start( pump.start(
connection: connection, layer: baseLayer, connection: connection, layer: baseLayer,
onFrame: onFrame, onSessionEnd: onSessionEnd) onFrame: onFrame, onSessionEnd: onSessionEnd, onDecodedSize: onDecodedSize)
self.pump = pump self.pump = pump
} }
} }
@@ -181,22 +193,45 @@ final class SessionPresenter {
guard let metalLayer, let connection else { return } guard let metalLayer, let connection else { return }
let mode = connection.currentMode() let mode = connection.currentMode()
syncFrameRate(hz: mode.refreshHz) // track a mid-session Reconfigure's new refresh syncFrameRate(hz: mode.refreshHz) // track a mid-session Reconfigure's new refresh
let fit: CGRect = (mode.width > 0 && mode.height > 0) // Aspect source: the ACTUAL decoded dims when known (survives a lagging `currentMode()` and a
? AVMakeRect( // host that delivered a different size than requested), else the negotiated mode. The shader
aspectRatio: CGSize(width: Int(mode.width), height: Int(mode.height)), // stretches the frame across the WHOLE drawable, so this rect's aspect is the only thing that
insideRect: bounds) // keeps the picture undistorted a stale aspect here is the post-resize black-bars+stretch.
: bounds let aspect: CGSize? = {
if let c = contentSize, c.width > 0, c.height > 0 { return c }
if mode.width > 0, mode.height > 0 {
return CGSize(width: Int(mode.width), height: Int(mode.height))
}
return nil
}()
let fit: CGRect = aspect.map { AVMakeRect(aspectRatio: $0, insideRect: bounds) } ?? bounds
// Snap the sublayer frame to the BACKING PIXEL GRID. AVMakeRect centers the aspect-fit rect,
// so its origin/size are usually fractional points; a metal sublayer whose frame doesn't land
// on whole device pixels is RESAMPLED by the macOS/UIKit compositor during composite a
// uniform "everything looks soft" blur even when the drawable itself is pixel-exact 1:1
// (verified via the stage2 "[1:1 (no resample)]" log while the picture was still soft). Round
// origin AND size to device pixels so the composite is a true 1:1 blit. Idempotent when the
// frame is already aligned (e.g. fullscreen fit == integer bounds), so it's a no-op there.
let scale = contentsScale > 0 ? contentsScale : 1
let snapped = CGRect(
x: (fit.origin.x * scale).rounded() / scale,
y: (fit.origin.y * scale).rounded() / scale,
width: (fit.width * scale).rounded() / scale,
height: (fit.height * scale).rounded() / scale)
// No implicit resize animation; contentsScale tracks the view's backing/display scale. // No implicit resize animation; contentsScale tracks the view's backing/display scale.
CATransaction.begin() CATransaction.begin()
CATransaction.setDisableActions(true) CATransaction.setDisableActions(true)
metalLayer.contentsScale = contentsScale metalLayer.contentsScale = contentsScale
metalLayer.frame = fit metalLayer.frame = snapped
CATransaction.commit() CATransaction.commit()
// Hand the resulting pixel size to the render thread (it must not read layer geometry // Hand the resulting pixel size to the render thread (it must not read layer geometry
// cross-thread) this is what the presenter sizes its drawable to. // cross-thread) this is what the presenter sizes its drawable to. Uses the SNAPPED size so
// the drawable's texel count equals the on-screen device-pixel count exactly (1 texel 1
// device pixel); with the frame snapped, this equals the pre-snap rounded value, so the
// decodeddrawable 1:1 the log confirmed is preserved.
stage2?.setDrawableTarget(CGSize( stage2?.setDrawableTarget(CGSize(
width: (fit.width * contentsScale).rounded(), width: (snapped.width * scale).rounded(),
height: (fit.height * contentsScale).rounded())) height: (snapped.height * scale).rounded()))
#if os(tvOS) #if os(tvOS)
// Push the display's live EDR headroom alongside: > 1 means the TV is composited in an // Push the display's live EDR headroom alongside: > 1 means the TV is composited in an
// HDR mode (the session's AVDisplayManager request landed see StreamViewIOS), and HDR // HDR mode (the session's AVDisplayManager request landed see StreamViewIOS), and HDR
@@ -206,10 +241,20 @@ final class SessionPresenter {
#endif #endif
} }
/// Record the decoded frame's real dimensions (the view hops the pump's `onDecodedSize` to main
/// and calls this) so `layout` aspect-fits to what's actually on screen instead of the possibly-
/// stale `currentMode()`. Only stores the caller re-runs `layout` right after, because a
/// resize-END produces no bounds change to trigger one. No-op on a zero/unchanged size.
func setContentSize(_ size: CGSize) {
guard size.width > 0, size.height > 0, size != contentSize else { return }
contentSize = size
}
/// Stop the active pump/pipeline ( one poll timeout; stage-2 joins its pump) and detach the /// Stop the active pump/pipeline ( one poll timeout; stage-2 joins its pump) and detach the
/// stage-2 layer + link. Does not close the connection that stays with whoever owns it. /// stage-2 layer + link. Does not close the connection that stays with whoever owns it.
/// Idempotent. /// Idempotent.
func stop() { func stop() {
contentSize = nil // a new session re-derives it from its first frame
pump?.stop() pump?.stop()
pump = nil pump = nil
stage2Link?.invalidate() stage2Link?.invalidate()
@@ -259,6 +259,10 @@ public final class Stage2Pipeline {
private let endToEndMeter: LatencyMeter? private let endToEndMeter: LatencyMeter?
private let displayMeter: LatencyMeter? private let displayMeter: LatencyMeter?
private let recovery = KeyframeRecovery() private let recovery = KeyframeRecovery()
/// Post-loss freeze-until-reanchor gate (shared core policy via the C ABI). Created here seeded 0;
/// `start` reseeds it to the live connection's drop count. Captured by the decoder callbacks
/// (which withhold concealed frames) and driven by the pump (arm on a gap, poll per iteration).
private let gate = ReanchorGate(framesDropped: 0)
private var token = StopFlag() private var token = StopFlag()
private var offsetNs: Int64 = 0 private var offsetNs: Int64 = 0
/// Signalled when the pump thread exits, so `stop()` can join it (bounded) before `decoder.reset()` /// Signalled when the pump thread exits, so `stop()` can join it (bounded) before `decoder.reset()`
@@ -306,21 +310,29 @@ public final class Stage2Pipeline {
let ring = ring let ring = ring
let recovery = recovery let recovery = recovery
let renderSignal = renderSignal let renderSignal = renderSignal
let gate = gate
self.decoder = VideoDecoder( self.decoder = VideoDecoder(
onDecoded: { frame in onDecoded: { frame in
// Decode stage = receiveddecoded, both client CLOCK_REALTIME (offset 0 no // Decode stage = receiveddecoded, both client CLOCK_REALTIME (offset 0 no
// skew applies). Stamped at decode completion, so it covers every decoded frame, // skew applies). Stamped at decode completion, so it covers every decoded frame,
// including ones the newest-wins ring drops before present. // including ones the re-anchor gate withholds or the newest-wins ring drops.
decodeMeter?.record( decodeMeter?.record(
ptsNs: UInt64(frame.receivedNs), atNs: frame.decodedNs, offsetNs: 0) ptsNs: UInt64(frame.receivedNs), atNs: frame.decodedNs, offsetNs: 0)
// Freeze-until-reanchor: WITHHOLD a decoder-concealed post-loss frame (the gray/
// garbage VideoToolbox returns Ok for a reference-missing delta) don't submit it,
// so the CAMetalLayer keeps its last good drawable on glass. The gate lifts (returns
// present) on a proven clean re-anchor (IDR / RFI anchor / 2nd recovery mark) or the
// bounded backstop. decoderKeyframe=false: VT doesn't flag IDRs, the wire FLAG_SOF does.
guard gate.onDecoded(flags: frame.flags) else { return }
ring.submit(frame) ring.submit(frame)
// FRAME ARRIVAL is the render trigger (never the display link see the header). // FRAME ARRIVAL is the render trigger (never the display link see the header).
renderSignal.signal() renderSignal.signal()
}, },
// Async decode failure (a bad P-frame referencing a lost/corrupt IDR): the pump resets to // Async decode failure (a bad P-frame referencing a lost/corrupt IDR): fold it into the
// re-gate on the next IDR, and we ask the host to send one now (infinite GOP it wouldn't // gate's no-output streak (which arms the freeze after a short run, matching the desktop),
// and when that trips ask the host for a fresh IDR now (infinite GOP it wouldn't
// otherwise come soon). Throttled in KeyframeRecovery. // otherwise come soon). Throttled in KeyframeRecovery.
onDecodeError: { _ in recovery.request() }) onDecodeError: { _ in if gate.onNoOutput() { recovery.request() } })
} }
/// Start pulling AUs into the decoder. MAIN THREAD. `onFrame` fires per AU at receipt (the /// Start pulling AUs into the decoder. MAIN THREAD. `onFrame` fires per AU at receipt (the
@@ -329,10 +341,12 @@ public final class Stage2Pipeline {
public func start( public func start(
connection: PunktfunkConnection, connection: PunktfunkConnection,
onFrame: (@Sendable (AccessUnit) -> Void)?, onFrame: (@Sendable (AccessUnit) -> Void)?,
onSessionEnd: (@Sendable () -> Void)? onSessionEnd: (@Sendable () -> Void)?,
onDecodedSize: (@Sendable (Int, Int) -> Void)? = nil
) { ) {
offsetNs = connection.clockOffsetNs offsetNs = connection.clockOffsetNs
recovery.bind(connection) // arm host-keyframe recovery for this session recovery.bind(connection) // arm host-keyframe recovery for this session
gate.reseed(framesDropped: connection.framesDropped()) // baseline the freeze to this session
token = StopFlag() // fresh token per start a stop is permanent (like StreamPump) token = StopFlag() // fresh token per start a stop is permanent (like StreamPump)
// Configure the decoder's chroma + the layer's initial colorimetry before the first frame. The // Configure the decoder's chroma + the layer's initial colorimetry before the first frame. The
@@ -347,9 +361,13 @@ public final class Stage2Pipeline {
let recovery = recovery let recovery = recovery
let presenter = presenter let presenter = presenter
let pumpStopped = pumpStopped let pumpStopped = pumpStopped
let reanchorGate = gate
let thread = Thread { let thread = Thread {
defer { pumpStopped.signal() } // let stop() join the pump (bounded) before decoder.reset() defer { pumpStopped.signal() } // let stop() join the pump (bounded) before decoder.reset()
var format: CMVideoFormatDescription? var format: CMVideoFormatDescription?
// Report coded dims to the resize overlay only on a CHANGE (new-mode IDR), not per
// loss-recovery IDR at the same size (see StreamPump).
var lastDecodedDims: CMVideoDimensions?
var lastFramesDropped = connection.framesDropped() var lastFramesDropped = connection.framesDropped()
// Persistent recovery WANT, not a one-shot edge (see StreamPump for the full rationale): // Persistent recovery WANT, not a one-shot edge (see StreamPump for the full rationale):
// keep asking until an IDR lands so a request swallowed by the throttle is re-sent. // keep asking until an IDR lands so a request swallowed by the throttle is re-sent.
@@ -375,6 +393,9 @@ public final class Stage2Pipeline {
awaitingIDR = true awaitingIDR = true
} }
if awaitingIDR { recovery.request() } if awaitingIDR { recovery.request() }
// Freeze backstop: a drop-count climb arms the gate (in case the frame-index gap
// below was itself lost), and an overdue freeze re-asks for the re-anchor.
if reanchorGate.poll(framesDropped: dropped) { recovery.request() }
// Drain HDR mastering metadata (0xCE) and hand it to the PRESENTER ( CAEDRMetadata). // Drain HDR mastering metadata (0xCE) and hand it to the PRESENTER ( CAEDRMetadata).
// Polled UNCONDITIONALLY (not gated on connection.isHDR, the fixed Welcome flag): the // Polled UNCONDITIONALLY (not gated on connection.isHDR, the fixed Welcome flag): the
// host sends 0xCE only for HDR, INCLUDING a mid-session SDRHDR transition (a game // host sends 0xCE only for HDR, INCLUDING a mid-session SDRHDR transition (a game
@@ -384,9 +405,21 @@ public final class Stage2Pipeline {
presenter.setHdrMeta(meta) presenter.setHdrMeta(meta)
} }
guard let au = try connection.nextAU(timeoutMs: 100) else { return true } guard let au = try connection.nextAU(timeoutMs: 100) else { return true }
// Loss recovery (RFI): a forward frame-index gap fires a throttled reference-
// frame-invalidation request so an RFI-capable host (AMD LTR / NVENC) recovers
// with a cheap clean P-frame instead of a full IDR. The framesDropped-driven
// recovery above stays the backstop for when the recovery frame itself is lost.
// The same gap is the earliest, most precise signal to ARM the display freeze
// the following concealed frames are withheld until a clean re-anchor.
if connection.noteFrameIndexGap(au.frameIndex) { reanchorGate.arm() }
onFrame?(au) onFrame?(au)
if let f = connection.videoCodec.formatDescription(fromKeyframe: au.data) { if let f = connection.videoCodec.formatDescription(fromKeyframe: au.data) {
format = f // refreshed on every IDR (mode changes included) format = f // refreshed on every IDR (mode changes included)
let dims = CMVideoFormatDescriptionGetDimensions(f)
if lastDecodedDims?.width != dims.width || lastDecodedDims?.height != dims.height {
lastDecodedDims = dims
onDecodedSize?(Int(dims.width), Int(dims.height))
}
awaitingIDR = false // a fresh IDR re-anchored decode recovery complete awaitingIDR = false // a fresh IDR re-anchored decode recovery complete
} }
guard let f = format, !token.isStopped else { return true } guard let f = format, !token.isStopped else { return true }
@@ -21,12 +21,18 @@ final class StreamPump {
connection: PunktfunkConnection, connection: PunktfunkConnection,
layer: AVSampleBufferDisplayLayer, layer: AVSampleBufferDisplayLayer,
onFrame: (@Sendable (AccessUnit) -> Void)?, onFrame: (@Sendable (AccessUnit) -> Void)?,
onSessionEnd: (@Sendable () -> Void)? onSessionEnd: (@Sendable () -> Void)?,
onDecodedSize: (@Sendable (Int, Int) -> Void)? = nil
) { ) {
let token = token let token = token
// Coalesced host keyframe requests (100 ms throttle see KeyframeRecovery). // Coalesced host keyframe requests (100 ms throttle see KeyframeRecovery).
let recovery = KeyframeRecovery() let recovery = KeyframeRecovery()
recovery.bind(connection) recovery.bind(connection)
// Post-loss freeze-until-reanchor (shared core policy via the C ABI). Stage-1 has no per-frame
// decode callback, so the gate is folded at ENQUEUE (from the AU's wire flags): a withheld
// frame is still enqueued but flagged DoNotDisplay so the layer's decoder keeps the reference
// chain fed while the last GOOD picture stays on glass until a clean re-anchor lifts it.
let gate = ReanchorGate(framesDropped: connection.framesDropped())
// The layer is non-Sendable but its enqueue/flush are documented thread-safe, and after // The layer is non-Sendable but its enqueue/flush are documented thread-safe, and after
// this point only the pump thread drives it assert that so the @Sendable Thread closure // this point only the pump thread drives it assert that so the @Sendable Thread closure
// may capture it. // may capture it.
@@ -35,6 +41,9 @@ final class StreamPump {
let thread = Thread { let thread = Thread {
var format: CMVideoFormatDescription? var format: CMVideoFormatDescription?
// Report the coded dims to the resize overlay only when they CHANGE (a new-mode IDR),
// not on every loss-recovery IDR at the same size so it fires once per real switch.
var lastDecodedDims: CMVideoDimensions?
var lastFramesDropped = connection.framesDropped() var lastFramesDropped = connection.framesDropped()
// Recovery is a persistent WANT, not a one-shot edge: set it on detected loss (or a // Recovery is a persistent WANT, not a one-shot edge: set it on detected loss (or a
// decoder reset), retry the throttled request EVERY iteration, and clear it only when a // decoder reset), retry the throttled request EVERY iteration, and clear it only when a
@@ -73,12 +82,26 @@ final class StreamPump {
awaitingIDR = true awaitingIDR = true
} }
if awaitingIDR { recovery.request() } if awaitingIDR { recovery.request() }
// Freeze backstop: a drop-count climb arms the gate (should the frame-index gap
// below be lost too), and an overdue freeze re-asks for the re-anchor.
if gate.poll(framesDropped: dropped) { recovery.request() }
guard let au = try connection.nextAU(timeoutMs: 100) else { return true } guard let au = try connection.nextAU(timeoutMs: 100) else { return true }
// Loss recovery (RFI): a forward frame-index gap fires a throttled reference-
// frame-invalidation request so an RFI-capable host (AMD LTR / NVENC) recovers
// with a cheap clean P-frame instead of a full IDR. The framesDropped-driven
// recovery above stays the backstop for when the recovery frame itself is lost.
// The same gap is the earliest, most precise signal to ARM the display freeze.
if connection.noteFrameIndexGap(au.frameIndex) { gate.arm() }
onFrame?(au) onFrame?(au)
let idrFormat = connection.videoCodec.formatDescription(fromKeyframe: au.data) let idrFormat = connection.videoCodec.formatDescription(fromKeyframe: au.data)
if let f = idrFormat { if let f = idrFormat {
format = f // refreshed on every IDR (mode changes included) format = f // refreshed on every IDR (mode changes included)
let dims = CMVideoFormatDescriptionGetDimensions(f)
if lastDecodedDims?.width != dims.width || lastDecodedDims?.height != dims.height {
lastDecodedDims = dims
onDecodedSize?(Int(dims.width), Int(dims.height))
}
if awaitingIDR { if awaitingIDR {
let ms = Int(Date().timeIntervalSince(awaitingSince) * 1000) let ms = Int(Date().timeIntervalSince(awaitingSince) * 1000)
pumpLog.notice("video: recovery IDR received — resumed after \(ms, privacy: .public) ms") pumpLog.notice("video: recovery IDR received — resumed after \(ms, privacy: .public) ms")
@@ -93,6 +116,7 @@ final class StreamPump {
// delta into a failed layer can't recover it. // delta into a failed layer can't recover it.
if !wasFailed { pumpLog.warning("video: display layer .failed — flushing + re-anchoring") } if !wasFailed { pumpLog.warning("video: display layer .failed — flushing + re-anchoring") }
layer.flush() layer.flush()
gate.arm() // a wedged decoder is a loss freeze until the re-anchor
if idrFormat == nil { if idrFormat == nil {
format = nil format = nil
awaitingIDR = true awaitingIDR = true
@@ -103,6 +127,13 @@ final class StreamPump {
let sample = connection.videoCodec.sampleBuffer(au: au, format: f), let sample = connection.videoCodec.sampleBuffer(au: au, format: f),
!token.isStopped // don't enqueue a stale frame after a restart !token.isStopped // don't enqueue a stale frame after a restart
else { return true } else { return true }
// Freeze-until-reanchor: while holding, WITHHOLD this concealed post-loss frame by
// flagging it DoNotDisplay the layer still decodes it (keeping the reference
// chain fed) but shows the last GOOD picture until a clean re-anchor lifts the
// gate. Folded from the AU's wire flags (stage-1 has no decode callback).
if !gate.onDecoded(flags: au.flags) {
StreamPump.setDoNotDisplay(sample)
}
layer.enqueue(sample) layer.enqueue(sample)
return true return true
} catch { } catch {
@@ -119,6 +150,21 @@ final class StreamPump {
thread.start() thread.start()
} }
/// Flag a sample decode-but-don't-display (`kCMSampleAttachmentKey_DoNotDisplay`). Used to
/// withhold decoder-concealed post-loss frames while the re-anchor gate holds: the layer keeps
/// its reference chain fed without flipping the frozen picture. No-op if the attachments array
/// can't be materialized (then the frame just displays the freeze degrades to the old behavior).
private static func setDoNotDisplay(_ sample: CMSampleBuffer) {
guard let attachments = CMSampleBufferGetSampleAttachmentsArray(
sample, createIfNecessary: true), CFArrayGetCount(attachments) > 0
else { return }
let dict = unsafeBitCast(CFArrayGetValueAtIndex(attachments, 0), to: CFMutableDictionary.self)
CFDictionarySetValue(
dict,
Unmanaged.passUnretained(kCMSampleAttachmentKey_DoNotDisplay).toOpaque(),
Unmanaged.passUnretained(kCFBooleanTrue).toOpaque())
}
/// Stop pumping ( one poll timeout). Does not close the connection. /// Stop pumping ( one poll timeout). Does not close the connection.
func stop() { func stop() {
token.stop() token.stop()
@@ -27,19 +27,40 @@ public struct ReadyFrame: @unchecked Sendable {
/// True when the stream is HDR (BT.2020 PQ): the buffer is 10-bit P010 and the presenter must /// True when the stream is HDR (BT.2020 PQ): the buffer is 10-bit P010 and the presenter must
/// configure EDR + BT.2020 PQ output. Derived from the decoded buffer's pixel format. /// configure EDR + BT.2020 PQ output. Derived from the decoded buffer's pixel format.
public let isHDR: Bool public let isHDR: Bool
/// The AU's wire `user_flags` (`AccessUnit.flags`), threaded through the decode via the frame
/// context so the re-anchor gate can classify this decoded frame (IDR / RFI anchor / recovery
/// mark) at present time the async decode callback has no other access to it. 0 when unknown.
public let flags: UInt32
}
/// Per-frame context threaded through the VideoToolbox frame refcon: the AU's receipt instant (for
/// the decode-stage meter) and its wire `user_flags` (for the re-anchor gate). Retained across the
/// async decode and reclaimed exactly once by the output callback for every frame VideoToolbox
/// accepts, or by `decode`'s error branch for a frame `DecodeFrame` rejected outright (the callback
/// then never fires). A tiny per-frame allocation, the price of smuggling two values (a 64-bit
/// instant plus the flags) through the single `void*` a bit-pattern scalar can't hold.
private final class FrameContext {
let receivedNs: Int64
let flags: UInt32
init(receivedNs: Int64, flags: UInt32) {
self.receivedNs = receivedNs
self.flags = flags
}
} }
/// The C output callback can't capture context, so VideoToolbox hands it the refcon we set at /// The C output callback can't capture context, so VideoToolbox hands it the refcon we set at
/// session creation a pointer back to the owning `VideoDecoder`. The per-frame refcon carries /// session creation a pointer back to the owning `VideoDecoder`. The per-frame refcon is the
/// the AU's `receivedNs` as a pointer bit pattern (a scalar smuggled through the C void*, never /// retained `FrameContext` set at submit; reclaim it here (balancing `passRetained`) and unpack the
/// dereferenced) so the decode stage can be computed against decode-completion. /// AU's receipt instant (for the decode stage) and wire flags (for the re-anchor gate).
private let decoderOutputCallback: VTDecompressionOutputCallback = { private let decoderOutputCallback: VTDecompressionOutputCallback = {
refcon, frameRefcon, status, _, imageBuffer, pts, _ in refcon, frameRefcon, status, _, imageBuffer, pts, _ in
guard let refcon else { return } guard let refcon else { return }
let receivedNs = frameRefcon.map { Int64(Int(bitPattern: $0)) } ?? 0 let ctx = frameRefcon.map { Unmanaged<FrameContext>.fromOpaque($0).takeRetainedValue() }
Unmanaged<VideoDecoder>.fromOpaque(refcon) Unmanaged<VideoDecoder>.fromOpaque(refcon)
.takeUnretainedValue() .takeUnretainedValue()
.handleDecoded(status: status, imageBuffer: imageBuffer, pts: pts, receivedNs: receivedNs) .handleDecoded(
status: status, imageBuffer: imageBuffer, pts: pts,
receivedNs: ctx?.receivedNs ?? 0, flags: ctx?.flags ?? 0)
} }
/// Owns a `VTDecompressionSession` rebuilt whenever the format description changes (every IDR / /// Owns a `VTDecompressionSession` rebuilt whenever the format description changes (every IDR /
@@ -117,16 +138,21 @@ public final class VideoDecoder: @unchecked Sendable {
let sample = codec.sampleBuffer(au: au, format: newFormat) let sample = codec.sampleBuffer(au: au, format: newFormat)
else { lock.unlock(); return false } else { lock.unlock(); return false }
var infoOut = VTDecodeInfoFlags() var infoOut = VTDecodeInfoFlags()
// The AU's receipt instant + wire flags ride through as a retained context; the output
// callback reclaims it. Retain immediately before submit so no early return can leak it.
let ctx = FrameContext(receivedNs: au.receivedNs, flags: au.flags)
let refcon = Unmanaged.passRetained(ctx).toOpaque()
let status = VTDecompressionSessionDecodeFrame( let status = VTDecompressionSessionDecodeFrame(
session, session,
sampleBuffer: sample, sampleBuffer: sample,
flags: [._EnableAsynchronousDecompression], flags: [._EnableAsynchronousDecompression],
// The AU's receipt instant rides through as a bit pattern (nil for 0 the output frameRefcon: refcon,
// callback maps that back to 0); the callback needs it to stamp the decode stage.
frameRefcon: UnsafeMutableRawPointer(bitPattern: Int(au.receivedNs)),
infoFlagsOut: &infoOut) infoFlagsOut: &infoOut)
lock.unlock() lock.unlock()
if status != noErr { if status != noErr {
// DecodeFrame rejected the frame outright the output callback will NOT fire, so
// reclaim the context here (balancing passRetained) to avoid leaking it.
Unmanaged<FrameContext>.fromOpaque(refcon).release()
onDecodeError(status) onDecodeError(status)
return false return false
} }
@@ -231,9 +257,10 @@ public final class VideoDecoder: @unchecked Sendable {
} }
/// VT thread. Stamp decode-completion and enqueue, or report the error. `receivedNs` is the /// VT thread. Stamp decode-completion and enqueue, or report the error. `receivedNs` is the
/// AU's receipt instant threaded through the frame refcon (0 = unknown). /// AU's receipt instant and `flags` its wire `user_flags`, both threaded through the frame refcon
/// (0 = unknown).
fileprivate func handleDecoded( fileprivate func handleDecoded(
status: OSStatus, imageBuffer: CVImageBuffer?, pts: CMTime, receivedNs: Int64 status: OSStatus, imageBuffer: CVImageBuffer?, pts: CMTime, receivedNs: Int64, flags: UInt32
) { ) {
guard status == noErr, let imageBuffer else { guard status == noErr, let imageBuffer else {
onDecodeError(status) onDecodeError(status)
@@ -259,6 +286,6 @@ public final class VideoDecoder: @unchecked Sendable {
onDecoded( onDecoded(
ReadyFrame( ReadyFrame(
ptsNs: ptsNs, receivedNs: receivedNs, decodedNs: decodedNs, ptsNs: ptsNs, receivedNs: receivedNs, decodedNs: decodedNs,
pixelBuffer: imageBuffer, isHDR: isHDR)) pixelBuffer: imageBuffer, isHDR: isHDR, flags: flags))
} }
} }
@@ -87,6 +87,8 @@ public struct StreamView: NSViewRepresentable {
private let onDisconnectRequest: (() -> Void)? private let onDisconnectRequest: (() -> Void)?
private let onFrame: (@Sendable (AccessUnit) -> Void)? private let onFrame: (@Sendable (AccessUnit) -> Void)?
private let onSessionEnd: (@Sendable () -> Void)? private let onSessionEnd: (@Sendable () -> Void)?
private let onResizeTarget: ((UInt32, UInt32) -> Void)?
private let onDecodedSize: (@Sendable (Int, Int) -> Void)?
private let endToEndMeter: LatencyMeter? private let endToEndMeter: LatencyMeter?
private let decodeMeter: LatencyMeter? private let decodeMeter: LatencyMeter?
private let displayMeter: LatencyMeter? private let displayMeter: LatencyMeter?
@@ -108,6 +110,8 @@ public struct StreamView: NSViewRepresentable {
onDisconnectRequest: (() -> Void)? = nil, onDisconnectRequest: (() -> Void)? = nil,
onFrame: (@Sendable (AccessUnit) -> Void)? = nil, onFrame: (@Sendable (AccessUnit) -> Void)? = nil,
onSessionEnd: (@Sendable () -> Void)? = nil, onSessionEnd: (@Sendable () -> Void)? = nil,
onResizeTarget: ((UInt32, UInt32) -> Void)? = nil,
onDecodedSize: (@Sendable (Int, Int) -> Void)? = nil,
endToEndMeter: LatencyMeter? = nil, endToEndMeter: LatencyMeter? = nil,
decodeMeter: LatencyMeter? = nil, decodeMeter: LatencyMeter? = nil,
displayMeter: LatencyMeter? = nil displayMeter: LatencyMeter? = nil
@@ -118,6 +122,8 @@ public struct StreamView: NSViewRepresentable {
self.onDisconnectRequest = onDisconnectRequest self.onDisconnectRequest = onDisconnectRequest
self.onFrame = onFrame self.onFrame = onFrame
self.onSessionEnd = onSessionEnd self.onSessionEnd = onSessionEnd
self.onResizeTarget = onResizeTarget
self.onDecodedSize = onDecodedSize
self.endToEndMeter = endToEndMeter self.endToEndMeter = endToEndMeter
self.decodeMeter = decodeMeter self.decodeMeter = decodeMeter
self.displayMeter = displayMeter self.displayMeter = displayMeter
@@ -131,6 +137,8 @@ public struct StreamView: NSViewRepresentable {
view.endToEndMeter = endToEndMeter view.endToEndMeter = endToEndMeter
view.decodeMeter = decodeMeter view.decodeMeter = decodeMeter
view.displayMeter = displayMeter view.displayMeter = displayMeter
view.onResizeTarget = onResizeTarget
view.onDecodedSize = onDecodedSize
view.start(connection: connection, onFrame: onFrame, onSessionEnd: onSessionEnd) view.start(connection: connection, onFrame: onFrame, onSessionEnd: onSessionEnd)
return view return view
} }
@@ -142,6 +150,8 @@ public struct StreamView: NSViewRepresentable {
view.endToEndMeter = endToEndMeter view.endToEndMeter = endToEndMeter
view.decodeMeter = decodeMeter view.decodeMeter = decodeMeter
view.displayMeter = displayMeter view.displayMeter = displayMeter
view.onResizeTarget = onResizeTarget
view.onDecodedSize = onDecodedSize
// SwiftUI reuses the NSView across state changes repoint the pump only when the // SwiftUI reuses the NSView across state changes repoint the pump only when the
// connection identity actually changed. // connection identity actually changed.
if view.connection !== connection { if view.connection !== connection {
@@ -165,6 +175,15 @@ public final class StreamLayerView: NSView {
/// stage-1 StreamPump displayLayer path as the Metal-unavailable / DEBUG fallback. /// stage-1 StreamPump displayLayer path as the Metal-unavailable / DEBUG fallback.
private let presenter = SessionPresenter() private let presenter = SessionPresenter()
public private(set) var connection: PunktfunkConnection? public private(set) var connection: PunktfunkConnection?
/// Match-window resize follower (C3) non-nil while a session is active AND the `matchWindow`
/// setting is on (DEFAULT on, for pixel-exact windowed streaming); fed the view's physical-pixel
/// size on every relayout so the host mode tracks the window (1:1, no presenter resample).
private var matchFollower: MatchWindowFollower?
/// Last decoded frame size fed into the presenter's aspect-fit. A new-mode IDR after a resize
/// re-fits the metal sublayer to the REAL content aspect here `layout()` only re-runs on a
/// bounds change and a resize-END has none, so without this the layer keeps its pre-resize aspect
/// and the shader stretches the new frame into it (black bars + squish). Main-thread only.
private var lastDecodedContentSize: CGSize?
private let cursorCapture = CursorCapture() private let cursorCapture = CursorCapture()
private var inputCapture: InputCapture? private var inputCapture: InputCapture?
private var appObservers: [NSObjectProtocol] = [] private var appObservers: [NSObjectProtocol] = []
@@ -201,6 +220,13 @@ public final class StreamLayerView: NSView {
/// view can't do that itself (the connection's owner disconnects). /// view can't do that itself (the connection's owner disconnects).
public var onDisconnectRequest: (() -> Void)? public var onDisconnectRequest: (() -> Void)?
/// Resize overlay signals (design/midstream-resolution-resize.md client UX): `onResizeTarget`
/// (main thread, via the follower) fires the instant the window starts steering toward a new
/// size; `onDecodedSize` (PUMP thread) fires when a new-mode IDR's dims land. The owner drives
/// the blur+spinner from these set before `start()`.
public var onResizeTarget: ((UInt32, UInt32) -> Void)?
public var onDecodedSize: (@Sendable (Int, Int) -> Void)?
/// Main-thread only. False = input capture disabled outright (UI layered over the /// Main-thread only. False = input capture disabled outright (UI layered over the
/// stream); flipping to true auto-engages once. /// stream); flipping to true auto-engages once.
public var captureEnabled = true { public var captureEnabled = true {
@@ -618,6 +644,10 @@ public final class StreamLayerView: NSView {
// (explicit VTDecompressionSession decode + a CAMetalLayer/display-link present) by // (explicit VTDecompressionSession decode + a CAMetalLayer/display-link present) by
// default, the stage-1 pump as the Metal-missing / DEBUG fallback. The link comes from // default, the stage-1 pump as the Metal-missing / DEBUG fallback. The link comes from
// NSView.displayLink so it tracks the display this view is on. // NSView.displayLink so it tracks the display this view is on.
// Intercept the pump's coded-dims callback: re-fit the metal sublayer to the real content
// aspect (main thread) BEFORE forwarding to the owner's overlay END-signal. Fires only on a
// size CHANGE (first frame + each resolved resize), so this is rare, not per-frame.
let overlayDecodedSize = onDecodedSize
presenter.start( presenter.start(
connection: connection, connection: connection,
baseLayer: displayLayer, baseLayer: displayLayer,
@@ -626,15 +656,39 @@ public final class StreamLayerView: NSView {
displayMeter: displayMeter, displayMeter: displayMeter,
makeDisplayLink: { displayLink(target: $0, selector: $1) }, makeDisplayLink: { displayLink(target: $0, selector: $1) },
onFrame: onFrame, onFrame: onFrame,
onSessionEnd: onSessionEnd) onSessionEnd: onSessionEnd,
onDecodedSize: { [weak self] w, h in // resize overlay END signal (new-mode IDR dims)
DispatchQueue.main.async { self?.noteDecodedContentSize(width: w, height: h) }
overlayDecodedSize?(w, h)
})
// Match-window (C3): when ON, follow the window's pixel size so a windowed session streams
// 1:1 (pixel-exact) instead of the presenter resampling a fixed-mode frame into a
// non-matching window. The first real `layout()` feeds the initial size, so the stream
// converges to the window even though the connect used the explicit/display mode; entering
// fullscreen reports the full-display px, restoring a native-res 1:1 present there too.
// OPT-IN `?? false` matches the Settings toggle (which also defaults off); an unset
// default keeps the explicit mode.
let follower = MatchWindowFollower(
connection: connection,
enabled: UserDefaults.standard.object(forKey: DefaultsKey.matchWindow) as? Bool ?? false)
follower.onResizeTarget = onResizeTarget // resize overlay START signal (instant, on the follower)
matchFollower = follower
layoutPresenter() layoutPresenter()
requestAutoCapture() // entering a session is the deliberate "capture me" moment requestAutoCapture() // entering a session is the deliberate "capture me" moment
} }
/// Aspect-fit the stage-2 metal sublayer to the view; refresh contentsScale on a /// Aspect-fit the stage-2 metal sublayer to the view; refresh contentsScale on a
/// retinanon-retina move (see SessionPresenter.layout). /// retinanon-retina move (see SessionPresenter.layout). Also feeds the Match-window follower
/// the view's physical-pixel size (bounds backing), so a window resize / retina move follows.
private func layoutPresenter() { private func layoutPresenter() {
presenter.layout(in: bounds, contentsScale: window?.backingScaleFactor ?? 1) presenter.layout(in: bounds, contentsScale: window?.backingScaleFactor ?? 1)
// Feed the follower only once in a window (backing scale is real then) and with real
// bounds a pre-window layout would report point-sized dimensions.
if window != nil, bounds.width > 0, bounds.height > 0 {
let px = convertToBacking(bounds).size
matchFollower?.noteSize(
widthPx: Int(px.width.rounded()), heightPx: Int(px.height.rounded()))
}
} }
public override func viewDidChangeBackingProperties() { public override func viewDidChangeBackingProperties() {
@@ -642,6 +696,18 @@ public final class StreamLayerView: NSView {
layoutPresenter() // backing scale changed (e.g. moved to a non-retina display) layoutPresenter() // backing scale changed (e.g. moved to a non-retina display)
} }
/// A new decoded size landed (a new-mode IDR after a resize, or the session's first frame): push
/// it to the presenter's aspect-fit and re-layout NOW. A resize-END triggers no `layout()`, so
/// this is what makes the metal sublayer track the new content aspect instead of stretching the
/// new frame into the pre-resize box. Deduped so a same-size repeat is a no-op. Main thread.
private func noteDecodedContentSize(width: Int, height: Int) {
let size = CGSize(width: width, height: height)
guard size.width > 0, size.height > 0, size != lastDecodedContentSize else { return }
lastDecodedContentSize = size
presenter.setContentSize(size)
layoutPresenter()
}
/// Stop pumping ( one poll timeout). Does not close the connection that stays with /// Stop pumping ( one poll timeout). Does not close the connection that stays with
/// whoever owns it (PunktfunkConnection.close() is safe alongside a draining pump). /// whoever owns it (PunktfunkConnection.close() is safe alongside a draining pump).
public func stop() { public func stop() {
@@ -650,6 +716,8 @@ public final class StreamLayerView: NSView {
inputCapture?.stop() inputCapture?.stop()
inputCapture = nil inputCapture = nil
presenter.stop() presenter.stop()
matchFollower = nil
lastDecodedContentSize = nil // the next session re-derives it from its first frame
connection = nil connection = nil
} }
@@ -24,7 +24,9 @@
// (== locked): GCMouse forwards only WHILE locked, the UIKit indirect path (motion, buttons AND // (== locked): GCMouse forwards only WHILE locked, the UIKit indirect path (motion, buttons AND
// scroll) only while NOT locked so a pointer that emits both channels under lock can't double-send. // scroll) only while NOT locked so a pointer that emits both channels under lock can't double-send.
// Hardware keyboard forwarding shares InputCapture with macOS auto-engaged when streaming // Hardware keyboard forwarding shares InputCapture with macOS auto-engaged when streaming
// starts, toggles (detected from the HID stream; there is no NSEvent monitor here). // starts, toggles and Q releases (both detected from the HID stream; there is no NSEvent
// monitor here). Q is the cross-client Ctrl+Alt+Shift+Q it un-captures so the Magic Keyboard
// trackpad drives the local iPad UI again.
// //
// The public type is named StreamView like its macOS twin (each is platform-gated), so // The public type is named StreamView like its macOS twin (each is platform-gated), so
// the SwiftUI app layer is identical on both platforms. // the SwiftUI app layer is identical on both platforms.
@@ -53,6 +55,8 @@ public struct StreamView: UIViewControllerRepresentable {
private let onCaptureChange: ((Bool) -> Void)? private let onCaptureChange: ((Bool) -> Void)?
private let onFrame: (@Sendable (AccessUnit) -> Void)? private let onFrame: (@Sendable (AccessUnit) -> Void)?
private let onSessionEnd: (@Sendable () -> Void)? private let onSessionEnd: (@Sendable () -> Void)?
private let onResizeTarget: ((UInt32, UInt32) -> Void)?
private let onDecodedSize: (@Sendable (Int, Int) -> Void)?
private let endToEndMeter: LatencyMeter? private let endToEndMeter: LatencyMeter?
private let decodeMeter: LatencyMeter? private let decodeMeter: LatencyMeter?
private let displayMeter: LatencyMeter? private let displayMeter: LatencyMeter?
@@ -68,6 +72,8 @@ public struct StreamView: UIViewControllerRepresentable {
onDisconnectRequest: (() -> Void)? = nil, onDisconnectRequest: (() -> Void)? = nil,
onFrame: (@Sendable (AccessUnit) -> Void)? = nil, onFrame: (@Sendable (AccessUnit) -> Void)? = nil,
onSessionEnd: (@Sendable () -> Void)? = nil, onSessionEnd: (@Sendable () -> Void)? = nil,
onResizeTarget: ((UInt32, UInt32) -> Void)? = nil,
onDecodedSize: (@Sendable (Int, Int) -> Void)? = nil,
endToEndMeter: LatencyMeter? = nil, endToEndMeter: LatencyMeter? = nil,
decodeMeter: LatencyMeter? = nil, decodeMeter: LatencyMeter? = nil,
displayMeter: LatencyMeter? = nil displayMeter: LatencyMeter? = nil
@@ -77,6 +83,8 @@ public struct StreamView: UIViewControllerRepresentable {
self.onCaptureChange = onCaptureChange self.onCaptureChange = onCaptureChange
self.onFrame = onFrame self.onFrame = onFrame
self.onSessionEnd = onSessionEnd self.onSessionEnd = onSessionEnd
self.onResizeTarget = onResizeTarget
self.onDecodedSize = onDecodedSize
self.endToEndMeter = endToEndMeter self.endToEndMeter = endToEndMeter
self.decodeMeter = decodeMeter self.decodeMeter = decodeMeter
self.displayMeter = displayMeter self.displayMeter = displayMeter
@@ -89,6 +97,8 @@ public struct StreamView: UIViewControllerRepresentable {
controller.endToEndMeter = endToEndMeter controller.endToEndMeter = endToEndMeter
controller.decodeMeter = decodeMeter controller.decodeMeter = decodeMeter
controller.displayMeter = displayMeter controller.displayMeter = displayMeter
controller.onResizeTarget = onResizeTarget
controller.onDecodedSize = onDecodedSize
controller.start(connection: connection, onFrame: onFrame, onSessionEnd: onSessionEnd) controller.start(connection: connection, onFrame: onFrame, onSessionEnd: onSessionEnd)
return controller return controller
} }
@@ -99,6 +109,8 @@ public struct StreamView: UIViewControllerRepresentable {
controller.endToEndMeter = endToEndMeter controller.endToEndMeter = endToEndMeter
controller.decodeMeter = decodeMeter controller.decodeMeter = decodeMeter
controller.displayMeter = displayMeter controller.displayMeter = displayMeter
controller.onResizeTarget = onResizeTarget
controller.onDecodedSize = onDecodedSize
if controller.connection !== connection { if controller.connection !== connection {
controller.start(connection: connection, onFrame: onFrame, onSessionEnd: onSessionEnd) controller.start(connection: connection, onFrame: onFrame, onSessionEnd: onSessionEnd)
} }
@@ -147,6 +159,12 @@ public final class StreamViewController: StreamViewControllerBase {
/// Capture state at the last resign, restored on the next foreground otherwise the /// Capture state at the last resign, restored on the next foreground otherwise the
/// mouse/keyboard stay released after navigating out and nothing re-grabs them. /// mouse/keyboard stay released after navigating out and nothing re-grabs them.
private var wasCapturedOnResign = false private var wasCapturedOnResign = false
/// Match-window resize follower (C3) non-nil while a session is active AND the `matchWindow`
/// setting is on (DEFAULT on, for pixel-exact scene streaming); fed the view's physical-pixel
/// size from `viewDidLayoutSubviews` so an iPad Stage Manager / Split View scene resize
/// renegotiates the host mode (1:1, no presenter resample). iOS only (iPhone naturally no-ops
/// its fixed full-screen scene; tvOS drives display modes via AVDisplayManager instead).
private var matchFollower: MatchWindowFollower?
#endif #endif
/// Reads whether the scene's pointer is actually locked right now; nil = state /// Reads whether the scene's pointer is actually locked right now; nil = state
@@ -161,6 +179,18 @@ public final class StreamViewController: StreamViewControllerBase {
} }
var onCaptureChange: ((Bool) -> Void)? var onCaptureChange: ((Bool) -> Void)?
/// Resize-overlay START: forwarded to the Match-window follower so a scene resize drives the
/// blur+spinner the instant the window differs from the live mode (iOS only tvOS has no
/// follower). See `MatchWindowFollower.onResizeTarget`.
var onResizeTarget: ((UInt32, UInt32) -> Void)?
/// Resize-overlay END: the presenter reports the coded dims of each new-mode IDR here, so the
/// overlay clears when a frame at the requested size actually decodes.
var onDecodedSize: (@Sendable (Int, Int) -> Void)?
/// Last decoded size fed into the presenter's aspect-fit. A new-mode IDR (an iPad scene resize,
/// or a tvOS AVDisplayManager mode switch) re-fits the metal sublayer to the REAL content aspect
/// here `viewDidLayoutSubviews` only re-runs on a bounds change, which a resize-END lacks, so
/// without this the layer keeps its pre-resize aspect and stretches the new frame into it. Main.
private var lastDecodedContentSize: CGSize?
var captureEnabled = true { var captureEnabled = true {
didSet { didSet {
@@ -309,7 +339,19 @@ public final class StreamViewController: StreamViewControllerBase {
x: p.x, y: p.y, surfaceWidth: p.w, surfaceHeight: p.h) x: p.x, y: p.y, surfaceWidth: p.w, surfaceHeight: p.h)
} }
streamView.onPointerButton = { [weak self] button, down in streamView.onPointerButton = { [weak self] button, down in
guard let self, self.inputCapture?.gcMouseForwarding == false else { return } guard let self else { return }
// Released a trackpad/mouse click into the video RE-ENGAGES capture (the iPad
// analogue of macOS's `mouseDown engageCapture(fromClick:)`, and the click-mirror of
// the / Q keyboard toggles). Only the button-DOWN engages; that click is the local
// engage gesture, so it's suppressed toward the host (`fromClick`) and never forwarded
// its release is swallowed by InputCapture's suppress latch, whichever path delivers it.
// (Finger taps are untouched: touch always plays directly, so only the indirect pointer
// re-captures.) Captured already the absolute path forwards the button as before.
if !self.captured {
if down, self.captureEnabled { self.setCaptured(true, fromClick: true) }
return
}
guard self.inputCapture?.gcMouseForwarding == false else { return }
self.inputCapture?.sendMouseButton(button, pressed: down) self.inputCapture?.sendMouseButton(button, pressed: down)
} }
streamView.onScroll = { [weak self] dx, dy in streamView.onScroll = { [weak self] dx, dy in
@@ -322,16 +364,38 @@ public final class StreamViewController: StreamViewControllerBase {
guard let self else { return } guard let self else { return }
self.setCaptured(!self.captured) self.setCaptured(!self.captured)
} }
// Q (cross-client parity with macOS/Windows/Linux) releases the captured pointer +
// keyboard so the Magic Keyboard trackpad returns to driving the local iPad UI. Detected
// from the HID stream in InputCapture (no NSEvent monitor on iOS); unlike the toggle it
// only ever RELEASES re-pressing it while already released is a no-op (setCaptured guards).
capture.onReleaseCapture = { [weak self] in
self?.setCaptured(false)
}
capture.onPreempted = { [weak self] in capture.onPreempted = { [weak self] in
self?.setCaptured(false) self?.setCaptured(false)
} }
capture.start() capture.start()
inputCapture = capture inputCapture = capture
// Match-window (C3): when ON, follow the scene's pixel size so a resizable iPad scene
// streams 1:1 (pixel-exact) instead of the presenter resampling a fixed-mode frame into it.
// `viewDidLayoutSubviews` feeds it covers Stage Manager / Split View resizes and rotation.
// iPhone is a fixed full-screen scene, so this naturally no-ops (reports the device mode).
// OPT-IN `?? false` matches the Settings toggle (which also defaults off); an unset
// default keeps the explicit mode.
let follower = MatchWindowFollower(
connection: connection,
enabled: UserDefaults.standard.object(forKey: DefaultsKey.matchWindow) as? Bool ?? false)
follower.onResizeTarget = onResizeTarget
matchFollower = follower
#endif #endif
// Presenter choice + lifecycle live in SessionPresenter (shared with macOS): stage-2 // Presenter choice + lifecycle live in SessionPresenter (shared with macOS): stage-2
// (explicit VTDecompressionSession decode + a CAMetalLayer/display-link present) by // (explicit VTDecompressionSession decode + a CAMetalLayer/display-link present) by
// default, the stage-1 pump as the Metal-missing / DEBUG fallback. // default, the stage-1 pump as the Metal-missing / DEBUG fallback.
// Intercept the pump's coded-dims callback: re-fit the metal sublayer to the real content
// aspect (main thread) BEFORE forwarding to the owner's overlay END-signal. Fires only on a
// size CHANGE (first frame + each resolved resize), so this is rare, not per-frame.
let overlayDecodedSize = onDecodedSize
presenter.start( presenter.start(
connection: connection, connection: connection,
baseLayer: streamView.displayLayer, baseLayer: streamView.displayLayer,
@@ -340,7 +404,11 @@ public final class StreamViewController: StreamViewControllerBase {
displayMeter: displayMeter, displayMeter: displayMeter,
makeDisplayLink: { CADisplayLink(target: $0, selector: $1) }, makeDisplayLink: { CADisplayLink(target: $0, selector: $1) },
onFrame: onFrame, onFrame: onFrame,
onSessionEnd: onSessionEnd) onSessionEnd: onSessionEnd,
onDecodedSize: { [weak self] w, h in
DispatchQueue.main.async { self?.noteDecodedContentSize(width: w, height: h) }
overlayDecodedSize?(w, h)
})
layoutMetalLayer() layoutMetalLayer()
#if os(iOS) #if os(iOS)
@@ -376,6 +444,19 @@ public final class StreamViewController: StreamViewControllerBase {
) { [weak self] _ in ) { [weak self] _ in
self?.syncPointerLock() self?.syncPointerLock()
}) })
// The Stream menu's "Release Mouse" (Q) posts this the discoverable menu surface for
// the RELEASED state. While CAPTURED the combo is recognized from the HID stream in
// InputCapture (onReleaseCapture) before the menu sees it, so in practice this fires as a
// not-captured no-op (setCaptured guards it); wired for honesty + a non-GC fallback. Only the
// foreground-active scene's stream acts the iPad analogue of macOS's key-window guard, so a
// second Stage Manager scene isn't released out from under the user.
observers.append(NotificationCenter.default.addObserver(
forName: .punktfunkReleaseCapture, object: nil, queue: .main
) { [weak self] _ in
guard let self,
self.view.window?.windowScene?.activationState == .foregroundActive else { return }
self.setCaptured(false)
})
if captureEnabled { if captureEnabled {
setCaptured(true) // entering a session is the deliberate "capture me" moment setCaptured(true) // entering a session is the deliberate "capture me" moment
@@ -411,6 +492,7 @@ public final class StreamViewController: StreamViewControllerBase {
streamView.onPointerButton = nil streamView.onPointerButton = nil
streamView.onScroll = nil streamView.onScroll = nil
streamView.currentHostMode = nil streamView.currentHostMode = nil
matchFollower = nil
#endif #endif
#if os(tvOS) #if os(tvOS)
// Return the TV to the user's preferred mode the home screen must not stay in the // Return the TV to the user's preferred mode the home screen must not stay in the
@@ -419,12 +501,23 @@ public final class StreamViewController: StreamViewControllerBase {
sessionDisplayManager = nil sessionDisplayManager = nil
#endif #endif
presenter.stop() presenter.stop()
lastDecodedContentSize = nil // the next session re-derives it from its first frame
connection = nil connection = nil
} }
public override func viewDidLayoutSubviews() { public override func viewDidLayoutSubviews() {
super.viewDidLayoutSubviews() super.viewDidLayoutSubviews()
layoutMetalLayer() layoutMetalLayer()
#if os(iOS)
// Match-window (C3): feed the follower the view's physical-pixel size (points × scale).
let b = streamView.bounds
if b.width > 0, b.height > 0 {
let scale = renderScale
matchFollower?.noteSize(
widthPx: Int((b.width * scale).rounded()),
heightPx: Int((b.height * scale).rounded()))
}
#endif
#if os(tvOS) #if os(tvOS)
applyDisplayCriteriaIfNeeded() applyDisplayCriteriaIfNeeded()
#endif #endif
@@ -485,12 +578,28 @@ public final class StreamViewController: StreamViewControllerBase {
presenter.layout(in: streamView.bounds, contentsScale: renderScale) presenter.layout(in: streamView.bounds, contentsScale: renderScale)
} }
/// A new decoded size landed (a scene/mode resize's new IDR, or the first frame): push it to the
/// presenter's aspect-fit and re-layout NOW. A resize-END triggers no `viewDidLayoutSubviews`, so
/// this is what makes the metal sublayer track the new content aspect instead of stretching the
/// new frame into the pre-resize box. Deduped so a same-size repeat is a no-op. Main thread.
private func noteDecodedContentSize(width: Int, height: Int) {
let size = CGSize(width: width, height: height)
guard size.width > 0, size.height > 0, size != lastDecodedContentSize else { return }
lastDecodedContentSize = size
presenter.setContentSize(size)
layoutMetalLayer()
}
#if os(iOS) #if os(iOS)
private func setCaptured(_ on: Bool) { /// `fromClick` marks a click-driven engage (the released-state pointer click that re-captures):
/// that click's press/release are suppressed toward the host it's the local engage gesture,
/// not a host click exactly as macOS's `engageCapture(fromClick:)` does. Keyboard-driven
/// engages () pass false so a normal click still reaches the host.
private func setCaptured(_ on: Bool, fromClick: Bool = false) {
if on { if on {
// `connection != nil` is the session-active gate (presenter internals are opaque here). // `connection != nil` is the session-active gate (presenter internals are opaque here).
guard captureEnabled, !captured, connection != nil else { return } guard captureEnabled, !captured, connection != nil else { return }
inputCapture?.setForwarding(true) inputCapture?.setForwarding(true, suppressClick: fromClick)
captured = true captured = true
} else { } else {
guard captured else { return } guard captured else { return }
@@ -3,6 +3,7 @@
// player-LED-bits GCControllerPlayerIndex map. All pure functions. // player-LED-bits GCControllerPlayerIndex map. All pure functions.
import GameController import GameController
import PunktfunkCore
import XCTest import XCTest
@testable import PunktfunkKit @testable import PunktfunkKit
@@ -40,6 +41,43 @@ final class GamepadWireTests: XCTestCase {
XCTAssertEqual(GamepadWire.axisRT, 5) XCTAssertEqual(GamepadWire.axisRT, 5)
} }
func testPadIndexRidesFlagsOnEveryPerPadEvent() {
// The wire pad index is the low byte of `flags` (punktfunk_core::input) on button + axis.
let btn = PunktfunkInputEvent.gamepadButton(GamepadWire.a, down: true, pad: 3)
XCTAssertEqual(btn.kind, UInt8(PUNKTFUNK_INPUT_KIND_GAMEPAD_BUTTON.rawValue))
XCTAssertEqual(btn.code, GamepadWire.a)
XCTAssertEqual(btn.x, 1)
XCTAssertEqual(btn.flags, 3)
let axis = PunktfunkInputEvent.gamepadAxis(GamepadWire.axisRT, value: 200, pad: 5)
XCTAssertEqual(axis.kind, UInt8(PUNKTFUNK_INPUT_KIND_GAMEPAD_AXIS.rawValue))
XCTAssertEqual(axis.code, GamepadWire.axisRT)
XCTAssertEqual(axis.x, 200)
XCTAssertEqual(axis.flags, 5)
// Single-controller path stays byte-identical: pad 0 flags 0, exactly as before.
XCTAssertEqual(PunktfunkInputEvent.gamepadButton(GamepadWire.a, down: false, pad: 0).flags, 0)
XCTAssertEqual(PunktfunkInputEvent.gamepadAxis(GamepadWire.axisLSX, value: 0, pad: 0).flags, 0)
}
func testArrivalAndRemoveWireLayout() {
// GamepadArrival (kind 14): code = the GamepadType wire byte, flags = pad index.
let arrival = PunktfunkInputEvent.gamepadArrival(
pref: PunktfunkConnection.GamepadType.dualSense.rawValue, pad: 2)
XCTAssertEqual(arrival.kind, UInt8(PUNKTFUNK_INPUT_KIND_GAMEPAD_ARRIVAL.rawValue))
XCTAssertEqual(arrival.code, PunktfunkConnection.GamepadType.dualSense.rawValue) // 2
XCTAssertEqual(arrival.flags, 2)
// The GamepadType raw values ARE the GamepadPref wire bytes the host resolves.
XCTAssertEqual(PunktfunkConnection.GamepadType.xbox360.rawValue, 1)
XCTAssertEqual(PunktfunkConnection.GamepadType.dualSense.rawValue, 2)
XCTAssertEqual(PunktfunkConnection.GamepadType.xboxOne.rawValue, 3)
XCTAssertEqual(PunktfunkConnection.GamepadType.dualShock4.rawValue, 4)
// GamepadRemove (kind 13): flags = pad index (the core stamps the per-pad seq).
let remove = PunktfunkInputEvent.gamepadRemove(pad: 7)
XCTAssertEqual(remove.kind, UInt8(PUNKTFUNK_INPUT_KIND_GAMEPAD_REMOVE.rawValue))
XCTAssertEqual(remove.flags, 7)
// 16 addressable pads (punktfunk_core::input::MAX_PADS).
XCTAssertEqual(GamepadWire.maxPads, 16)
}
func testTouchpadConversionCorners() { func testTouchpadConversionCorners() {
// GC ±1 with +y up wire 0...65535 with origin top-left, +y down. // GC ±1 with +y up wire 0...65535 with origin top-left, +y down.
let topLeft = GamepadWire.touchpad(x: -1, y: 1) let topLeft = GamepadWire.touchpad(x: -1, y: 1)
@@ -0,0 +1,43 @@
// The Match-window trigger discipline (design/midstream-resolution-resize.md D2), as pure
// functions the same rules the session binary's `resize_decision` unit-tests: physical pixels
// even-floored and clamped 320×200, skip a size equal to the live mode, and request each
// distinct size at most once (so a rejected size / a host rollback can't loop).
import XCTest
@testable import PunktfunkKit
final class MatchWindowTests: XCTestCase {
func testNormalizeEvenFloorsAndClamps() {
// Odd pixels floor to even (the host rejects odd dimensions).
let a = MatchWindow.normalize(widthPx: 1001, heightPx: 601)
XCTAssertEqual(a.width, 1000)
XCTAssertEqual(a.height, 600)
// Already-even sizes pass through.
let b = MatchWindow.normalize(widthPx: 2560, heightPx: 1440)
XCTAssertEqual(b.width, 2560)
XCTAssertEqual(b.height, 1440)
// Tiny / zero clamp to the host floor.
let c = MatchWindow.normalize(widthPx: 100, heightPx: 80)
XCTAssertEqual(c.width, 320)
XCTAssertEqual(c.height, 200)
let z = MatchWindow.normalize(widthPx: 0, heightPx: -4)
XCTAssertEqual(z.width, 320)
XCTAssertEqual(z.height, 200)
}
func testRequestSkipsEqualAndAlreadyRequested() {
// A new size (different from the live mode, not yet requested) request it.
let r = MatchWindow.request(
target: (1000, 600), current: (1280, 720), lastRequested: (800, 500))
XCTAssertEqual(r?.width, 1000)
XCTAssertEqual(r?.height, 600)
// Equal to the live mode nothing to do.
XCTAssertNil(MatchWindow.request(
target: (1280, 720), current: (1280, 720), lastRequested: nil))
// Already requested once don't re-ask (covers a rejected size AND a host rollback:
// accepted rebuild failed corrective ack restored the old mode must not loop).
XCTAssertNil(MatchWindow.request(
target: (1000, 600), current: (1280, 720), lastRequested: (1000, 600)))
}
}
@@ -0,0 +1,52 @@
import XCTest
@testable import PunktfunkKit
final class ResizeIndicatorTests: XCTestCase {
func testInactiveUntilSteered() {
var r = ResizeIndicator()
XCTAssertFalse(r.active)
// A decoded frame with nothing pending is a no-op (session start / steady state).
r.decoded(width: 1920, height: 1080)
XCTAssertFalse(r.active)
}
func testSteeringActivatesAndDecodedTargetClears() {
var r = ResizeIndicator()
r.steering(width: 2560, height: 1440, now: 0)
XCTAssertTrue(r.active)
// A frame at a DIFFERENT size (the old mode still draining) doesn't clear it.
r.decoded(width: 1920, height: 1080)
XCTAssertTrue(r.active)
// The target frame lands clear.
r.decoded(width: 2560, height: 1440)
XCTAssertFalse(r.active)
}
func testTimeoutClearsWhenTargetNeverArrives() {
var r = ResizeIndicator(timeout: 2.5)
r.steering(width: 2560, height: 1440, now: 10)
r.tick(now: 12) // 2 s < timeout still up
XCTAssertTrue(r.active)
r.tick(now: 12.6) // 2.6 s timeout a rejected/capped switch clears
XCTAssertFalse(r.active)
}
func testDragReArmsTimeoutOnEachNewTarget() {
var r = ResizeIndicator(timeout: 2.5)
r.steering(width: 2000, height: 1200, now: 0)
r.steering(width: 2200, height: 1200, now: 2) // target changed since re-armed to 2
r.tick(now: 4) // only 2 s since the last change still up (drag isn't a timeout)
XCTAssertTrue(r.active)
r.tick(now: 4.6) // 2.6 s since the last change clears
XCTAssertFalse(r.active)
}
func testSteadyDragDoesNotResetTimeout() {
var r = ResizeIndicator(timeout: 2.5)
r.steering(width: 2560, height: 1440, now: 0)
r.steering(width: 2560, height: 1440, now: 1) // SAME target since stays 0
r.tick(now: 2.6) // 2.6 s since the ORIGINAL steer clears (not reset by the repeat)
XCTAssertFalse(r.active)
}
}
+33 -19
View File
@@ -264,21 +264,23 @@ pub fn show(
let page = adw::PreferencesPage::new(); let page = adw::PreferencesPage::new();
let stream = adw::PreferencesGroup::builder().title("Stream").build(); let stream = adw::PreferencesGroup::builder().title("Stream").build();
let res_names: Vec<String> = RESOLUTIONS // The D1 tri-state: Native, Match window (a virtual index 1, stored as the
// `match_window` flag), then the explicit sizes.
let res_names: Vec<String> = std::iter::once("Native display".to_string())
.chain(std::iter::once("Match window".to_string()))
.chain(
RESOLUTIONS
.iter() .iter()
.map(|&(w, h)| { .skip(1)
if w == 0 { .map(|&(w, h)| format!("{w} × {h}")),
"Native display".to_string() )
} else {
format!("{w} × {h}")
}
})
.collect(); .collect();
let res_row = ChoiceRow::new( let res_row = ChoiceRow::new(
&dialog, &dialog,
inline, inline,
"Resolution", "Resolution",
"The host creates a virtual output at exactly this size", "The host creates a virtual output at exactly this size — Match window follows \
the stream window, including mid-stream resizes",
&res_names.iter().map(String::as_str).collect::<Vec<_>>(), &res_names.iter().map(String::as_str).collect::<Vec<_>>(),
); );
let hz_names: Vec<String> = REFRESH let hz_names: Vec<String> = REFRESH
@@ -345,13 +347,14 @@ pub fn show(
stream.add(stats_row.widget()); stream.add(stats_row.widget());
let input = adw::PreferencesGroup::builder().title("Input").build(); let input = adw::PreferencesGroup::builder().title("Input").build();
// Which physical controller forwards as pad 0: automatic = the most recently connected // Controller forwarding: Automatic forwards EVERY real controller, each as its own pad
// real pad (Steam's virtual pad skipped). A pin is persisted by stable key // (Steam's virtual pad skipped); pinning one restricts the session to that single
// (`Settings::forward_pad`), so it survives restarts — and disconnects: an offline // controller (single-player). The pin is persisted by stable key (`Settings::forward_pad`),
// pinned pad keeps its entry here instead of silently snapping back to Automatic. // so it survives restarts — and disconnects: an offline pinned pad keeps its entry here
// instead of silently snapping back to Automatic.
let pads = gamepads.pads(); let pads = gamepads.pads();
let saved_pin = settings.borrow().forward_pad.clone(); let saved_pin = settings.borrow().forward_pad.clone();
let mut pad_names = vec!["Automatic (most recent)".to_string()]; let mut pad_names = vec!["Automatic (all controllers)".to_string()];
let mut pad_keys: Vec<String> = Vec::new(); let mut pad_keys: Vec<String> = Vec::new();
for p in &pads { for p in &pads {
let kind = p.kind_label(); let kind = p.kind_label();
@@ -377,7 +380,7 @@ pub fn show(
if pads.is_empty() { if pads.is_empty() {
"No controllers detected" "No controllers detected"
} else { } else {
"Exactly one controller is forwarded to the host" "All controllers are forwarded, each as its own player; pick one to force single-player"
}, },
&pad_names.iter().map(String::as_str).collect::<Vec<_>>(), &pad_names.iter().map(String::as_str).collect::<Vec<_>>(),
); );
@@ -470,10 +473,15 @@ pub fn show(
// Seed from the current settings. // Seed from the current settings.
{ {
let s = settings.borrow(); let s = settings.borrow();
let res_i = RESOLUTIONS let res_i = if s.match_window {
1
} else {
RESOLUTIONS
.iter() .iter()
.position(|&(w, h)| w == s.width && h == s.height) .position(|&(w, h)| w == s.width && h == s.height)
.unwrap_or(0); .map(|i| if i == 0 { 0 } else { i + 1 })
.unwrap_or(0)
};
res_row.set_selected(res_i as u32); res_row.set_selected(res_i as u32);
let hz_i = REFRESH.iter().position(|&r| r == s.refresh_hz).unwrap_or(0); let hz_i = REFRESH.iter().position(|&r| r == s.refresh_hz).unwrap_or(0);
hz_row.set_selected(hz_i as u32); hz_row.set_selected(hz_i as u32);
@@ -508,8 +516,14 @@ pub fn show(
dialog.add(&page); dialog.add(&page);
dialog.connect_closed(move |_| { dialog.connect_closed(move |_| {
let mut s = settings.borrow_mut(); let mut s = settings.borrow_mut();
let (w, h) = RESOLUTIONS[(res_row.selected() as usize).min(RESOLUTIONS.len() - 1)]; // Index 1 is the virtual "Match window" option; 0 = Native, 2.. = explicit.
(s.width, s.height) = (w, h); let res_i = (res_row.selected() as usize).min(RESOLUTIONS.len());
s.match_window = res_i == 1;
(s.width, s.height) = if res_i <= 1 {
(0, 0)
} else {
RESOLUTIONS[res_i - 1]
};
s.refresh_hz = REFRESH[(hz_row.selected() as usize).min(REFRESH.len() - 1)]; s.refresh_hz = REFRESH[(hz_row.selected() as usize).min(REFRESH.len() - 1)];
s.bitrate_kbps = (bitrate_row.value() * 1000.0) as u32; s.bitrate_kbps = (bitrate_row.value() * 1000.0) as u32;
s.gamepad = GAMEPADS[(pad_row.selected() as usize).min(GAMEPADS.len() - 1)].to_string(); s.gamepad = GAMEPADS[(pad_row.selected() as usize).min(GAMEPADS.len() - 1)].to_string();
+4
View File
@@ -145,6 +145,10 @@ pub fn run(target: Option<&str>) -> u8 {
trust::touch_last_used(&trust::hex(&fingerprint)); trust::touch_last_used(&trust::hex(&fingerprint));
})), })),
overlay: Some(Box::new(overlay)), overlay: Some(Box::new(overlay)),
window_size: crate::session_main::window_size(&settings_at_start),
// Latched at console start (like the stats tier above): toggling Match window in
// the console's settings screen applies from the next console launch.
match_window: crate::session_main::match_window(&settings_at_start),
}; };
let result = let result =
+28
View File
@@ -164,6 +164,32 @@ mod session_main {
} }
} }
/// The window's starting size under Match-window: the persisted last size, so the
/// first connect's mode already matches the glass; `None` (policy off / never
/// stored) = the 1280×720 default.
pub(crate) fn window_size(settings: &trust::Settings) -> Option<(u32, u32)> {
(settings.match_window && settings.last_window_w > 0 && settings.last_window_h > 0)
.then_some((settings.last_window_w, settings.last_window_h))
}
/// The Match-window policy hook for the presenter loop
/// (design/midstream-resolution-resize.md D1/D2): `Some(persist)` turns the
/// debounced resize→`Reconfigure` machinery on; the callback stores each resize-end's
/// logical window size (load-modify-save, like the console settings screen) so the
/// next launch opens at it.
pub(crate) fn match_window(settings: &trust::Settings) -> Option<Box<dyn FnMut(u32, u32)>> {
settings.match_window.then(|| {
Box::new(|w: u32, h: u32| {
let mut s = trust::Settings::load();
if (s.last_window_w, s.last_window_h) != (w, h) {
s.last_window_w = w;
s.last_window_h = h;
s.save();
}
}) as Box<dyn FnMut(u32, u32)>
})
}
/// One JSON status line on stdout (the shell parses these; strings hand-escaped via /// One JSON status line on stdout (the shell parses these; strings hand-escaped via
/// the minimal rules a reason string can need). `pub(crate)`: browse mode emits its /// the minimal rules a reason string can need). `pub(crate)`: browse mode emits its
/// failure through the same contract when spawned with `--json-status`. /// failure through the same contract when spawned with `--json-status`.
@@ -343,6 +369,8 @@ mod session_main {
overlay: Some(Box::new(pf_console_ui::SkiaOverlay::new())), overlay: Some(Box::new(pf_console_ui::SkiaOverlay::new())),
#[cfg(not(feature = "ui"))] #[cfg(not(feature = "ui"))]
overlay: None, overlay: None,
window_size: window_size(&settings),
match_window: match_window(&settings),
}; };
let outcome = let outcome =
+4 -11
View File
@@ -1,6 +1,6 @@
[package] [package]
name = "punktfunk-client-windows" name = "punktfunk-client-windows"
description = "Native Windows punktfunk/1 client — WinUI 3 (windows-reactor) shell, D3D11/SwapChainPanel present, FFmpeg decode, WASAPI audio, SDL3 gamepads" description = "Native Windows punktfunk/1 client — WinUI 3 (windows-reactor) shell, SDL3 gamepads; streaming runs in the spawned punktfunk-session binary"
version.workspace = true version.workspace = true
edition.workspace = true edition.workspace = true
rust-version.workspace = true rust-version.workspace = true
@@ -57,13 +57,10 @@ windows = { git = "https://github.com/microsoft/windows-rs", rev = "a4f7b2cb7c63
"Win32_UI_WindowsAndMessaging", "Win32_UI_WindowsAndMessaging",
] } ] }
# Video decode (same FFmpeg pin as the host/Linux client) — software HEVC on the GPU-less dev # FFmpeg — used only to enumerate which codecs this client can decode (probe::decodable_codecs),
# box; D3D11VA hardware decode is a follow-up for the real-GPU box. # advertised to the host on the speed-test connect. Same pin as the host/Linux client. (Real
# decode + present live in the spawned punktfunk-session binary.)
ffmpeg-next = "8" ffmpeg-next = "8"
opus = "0.3"
# Audio render + mic capture (the WASAPI analogue of the Linux client's PipeWire backend).
wasapi = "0.23"
# Gamepads: capture + feedback (full DualSense fidelity needs hidapi). SDL3 is cross-platform; # Gamepads: capture + feedback (full DualSense fidelity needs hidapi). SDL3 is cross-platform;
# built from source via the bundled CMake on Windows (no system SDL3). # built from source via the bundled CMake on Windows (no system SDL3).
@@ -71,12 +68,8 @@ sdl3 = { version = "0.18", features = ["build-from-source", "hidapi"] }
mdns-sd = "0.20" mdns-sd = "0.20"
async-channel = "2" async-channel = "2"
# The decoded-frame channel (session pump → render thread): crossbeam because the render loop
# blocks with `recv_timeout`, which async-channel has no sync analogue of.
crossbeam-channel = "0.5"
serde = { version = "1", features = ["derive"] } serde = { version = "1", features = ["derive"] }
serde_json = "1" serde_json = "1"
anyhow = "1"
tracing = "0.1" tracing = "0.1"
tracing-subscriber = { version = "0.3", features = ["env-filter"] } tracing-subscriber = { version = "0.3", features = ["env-filter"] }
+5 -205
View File
@@ -6,10 +6,7 @@
use super::style::*; use super::style::*;
use super::{AppCtx, Screen, Svc, Target}; use super::{AppCtx, Screen, Svc, Target};
use crate::discovery::DiscoveredHost; use crate::discovery::DiscoveredHost;
use crate::session::{self, SessionEvent, SessionParams, Stats}; use crate::trust::{self, KnownHost, KnownHosts};
use crate::trust::{self, KnownHost, KnownHosts, Settings};
use crate::video::DecoderPref;
use punktfunk_core::config::{CompositorPref, GamepadPref, Mode};
use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc; use std::sync::Arc;
use std::time::Duration; use std::time::Duration;
@@ -117,82 +114,6 @@ pub(crate) fn initiate_launch(
); );
} }
/// The mode to request: explicit settings, with `0` fields resolved to the native size/refresh
/// of the display our window is on (mirrors the Linux/Swift clients' native-display default).
pub(crate) fn resolve_mode(s: &Settings) -> Mode {
let mut mode = Mode {
width: s.width,
height: s.height,
refresh_hz: s.refresh_hz,
};
if mode.width == 0 || mode.refresh_hz == 0 {
if let Some((w, h, hz)) = current_display_mode() {
if mode.width == 0 {
(mode.width, mode.height) = (w, h);
}
if mode.refresh_hz == 0 {
mode.refresh_hz = hz;
}
}
}
// No display info (headless session, RDP oddities) — a sane floor.
if mode.width == 0 {
(mode.width, mode.height) = (1920, 1080);
}
if mode.refresh_hz == 0 {
mode.refresh_hz = 60;
}
mode
}
/// The current mode (physical pixels + refresh) of the display our window occupies:
/// `MonitorFromWindow` on the foreground window — ours, the user just clicked in it — then
/// `EnumDisplaySettingsW(ENUM_CURRENT_SETTINGS)` on that monitor's device. Defaults to the
/// primary display when we're not foreground (e.g. a scripted connect).
fn current_display_mode() -> Option<(u32, u32, u32)> {
use windows::core::PCWSTR;
use windows::Win32::Graphics::Gdi::{
EnumDisplaySettingsW, GetMonitorInfoW, MonitorFromWindow, DEVMODEW, ENUM_CURRENT_SETTINGS,
MONITORINFO, MONITORINFOEXW,
};
use windows::Win32::UI::WindowsAndMessaging::GetForegroundWindow;
unsafe {
let monitor = MonitorFromWindow(
GetForegroundWindow(),
windows::Win32::Graphics::Gdi::MONITOR_DEFAULTTOPRIMARY,
);
let mut info = MONITORINFOEXW::default();
info.monitorInfo.cbSize = std::mem::size_of::<MONITORINFOEXW>() as u32;
if !GetMonitorInfoW(
monitor,
&mut info as *mut MONITORINFOEXW as *mut MONITORINFO,
)
.as_bool()
{
return None;
}
let mut dm = DEVMODEW {
dmSize: std::mem::size_of::<DEVMODEW>() as u16,
..Default::default()
};
if !EnumDisplaySettingsW(
PCWSTR(info.szDevice.as_ptr()),
ENUM_CURRENT_SETTINGS,
&mut dm,
)
.as_bool()
{
return None;
}
// dmDisplayFrequency of 0/1 means "hardware default" — unusable as a mode request.
(dm.dmPelsWidth > 0 && dm.dmDisplayFrequency > 1).then_some((
dm.dmPelsWidth,
dm.dmPelsHeight,
dm.dmDisplayFrequency,
))
}
}
/// Tunables that differ between the normal connect and the no-PIN "request access" flow. /// Tunables that differ between the normal connect and the no-PIN "request access" flow.
/// `Default` is the normal connect: short handshake budget, persist *unpaired* on TOFU, and the /// `Default` is the normal connect: short handshake budget, persist *unpaired* on TOFU, and the
/// plain "Connecting" screen. /// plain "Connecting" screen.
@@ -220,9 +141,7 @@ pub(crate) struct ConnectOpts {
/// so it can't loop. /// so it can't loop.
wake_on_fail: bool, wake_on_fail: bool,
/// A library title id (`steam:570`, …) the host launches during the connect handshake — /// A library title id (`steam:570`, …) the host launches during the connect handshake —
/// the library page's tap-to-play. Spawn mode passes it as `--launch`; the legacy /// the library page's tap-to-play, passed to the spawned session child as `--launch`.
/// in-process path has no launch plumbing (it predates the library and is slated for
/// deletion).
launch: Option<String>, launch: Option<String>,
} }
@@ -265,128 +184,11 @@ fn connect_with(
opts: ConnectOpts, opts: ConnectOpts,
) { ) {
// Session-always: every stream runs in the spawned punktfunk-session Vulkan binary. // Session-always: every stream runs in the spawned punktfunk-session Vulkan binary.
// The in-process D3D11VA path below stays reachable via the "Streaming engine" connect_spawn(ctx, target, pin, set_screen, set_status, opts)
// setting / PUNKTFUNK_BUILTIN_STREAM=1 as the A/B baseline until its deletion.
if !super::use_builtin_stream(ctx) {
return connect_spawn(ctx, target, pin, set_screen, set_status, opts);
}
let s = ctx.settings.lock().unwrap().clone();
let gamepad_pref = match GamepadPref::from_name(&s.gamepad) {
Some(GamepadPref::Auto) | None => ctx.gamepad.auto_pref(),
Some(explicit) => explicit,
};
let handle = session::start(SessionParams {
host: target.addr.clone(),
port: target.port,
mode: resolve_mode(&s),
compositor: CompositorPref::from_name(&s.compositor).unwrap_or(CompositorPref::Auto),
gamepad: gamepad_pref,
bitrate_kbps: s.bitrate_kbps,
audio_channels: s.audio_channels,
mic_enabled: s.mic_enabled,
hdr_enabled: s.hdr_enabled,
decoder: DecoderPref::from_name(&s.decoder),
preferred_codec: s.preferred_codec(),
pin,
identity: ctx.identity.clone(),
connect_timeout: opts.connect_timeout,
});
set_status.call(String::new());
set_screen.call(if opts.awaiting_approval {
Screen::RequestAccess
} else {
Screen::Connecting
});
let tofu = pin.is_none();
let persist_paired = opts.persist_paired;
let cancel = opts.cancel;
let wake_on_fail = opts.wake_on_fail;
let ctx = ctx.clone();
let (shared, gamepad) = (ctx.shared.clone(), ctx.gamepad.clone());
let (ss, st) = (set_screen.clone(), set_status.clone());
let target = target.clone();
std::thread::spawn(move || loop {
let event = match handle.events.recv_blocking() {
Ok(e) => e,
Err(_) => {
gamepad.detach();
ss.call(Screen::Hosts);
break;
}
};
// A cancelled request-access connect that resolved late (the host approved or the park
// timed out after the user walked away): tear down silently. Cancel already returned the
// UI to the host list; dropping `event` (and with it any connector) closes the connection
// without popping a stream or a stray error over the screen a new session may own.
if cancel.as_ref().is_some_and(|c| c.load(Ordering::SeqCst)) {
break;
}
match event {
SessionEvent::Connected {
connector,
fingerprint,
..
} => {
if persist_paired || tofu {
// Request-access: the operator approved this device, so record the host as a
// trusted PAIRED host — future connects are then silent (rule 1), exactly like
// after a PIN ceremony. A plain TOFU connect persists it *unpaired* (pinned).
let mut k = KnownHosts::load();
k.upsert(KnownHost {
name: target.name.clone(),
addr: target.addr.clone(),
port: target.port,
fp_hex: trust::hex(&fingerprint),
paired: persist_paired,
last_used: None,
mac: target.mac.clone(),
});
let _ = k.save();
}
trust::touch_last_used(&trust::hex(&fingerprint));
gamepad.attach(connector.clone());
*shared.stats.lock().unwrap() = Stats::default(); // clear any prior session's numbers
*shared.handoff.lock().unwrap() =
Some((connector, handle.frames.clone(), handle.stop.clone()));
ss.call(Screen::Stream);
}
SessionEvent::Failed {
msg,
trust_rejected,
} => {
st.call(msg);
gamepad.detach();
if trust_rejected {
// Pinned-fingerprint mismatch / pairing required → re-pair via the PIN screen.
// The host ANSWERED, so this never takes the wake fallback.
*shared.target.lock().unwrap() = target.clone();
ss.call(Screen::Pair);
} else if wake_on_fail {
// The dial-first attempt to a non-advertising host failed — it may genuinely
// be asleep. NOW wake and wait (its resolved redial uses default opts, so a
// second failure lands on the host list, not back here).
wake_and_connect(&ctx, target.clone(), &ss, &st);
} else {
ss.call(Screen::Hosts);
}
break;
}
SessionEvent::Ended(err) => {
// `None` = the user ended the session themselves (the disconnect shortcut) —
// return to the host list silently; an error banner would read as a failure.
st.call(err.unwrap_or_default());
gamepad.detach();
ss.call(Screen::Hosts);
break;
}
SessionEvent::Stats(s) => *shared.stats.lock().unwrap() = s,
}
});
} }
/// Spawn-mode connect: run the stream in the punktfunk-session binary and translate its /// Spawn-mode connect: run the stream in the punktfunk-session binary and translate its
/// stdout contract into the same navigation the in-process event loop drove. The child /// stdout contract into the app's connect-flow navigation. The child
/// NEVER connects unpinned — a stored/ceremony pin, else the host's advertised /// NEVER connects unpinned — a stored/ceremony pin, else the host's advertised
/// fingerprint (TOFU: persisted once the child reports ready, which proves the host /// fingerprint (TOFU: persisted once the child reports ready, which proves the host
/// really holds that identity, mirroring the GTK shell); no fingerprint at all routes to /// really holds that identity, mirroring the GTK shell); no fingerprint at all routes to
@@ -723,9 +525,7 @@ pub(crate) fn request_access_page(
.on_click(move || { .on_click(move || {
// Return the UI immediately; trip the flag this request's event loop // Return the UI immediately; trip the flag this request's event loop
// captured so it tears down silently when the connect resolves (see // captured so it tears down silently when the connect resolves (see
// ConnectOpts::cancel). Spawn mode: killing the parked child IS the abort // ConnectOpts::cancel). Killing the parked session child IS the abort.
// (builtin mode's in-process connect is blocking with none — it just
// resolves/times out later).
if let Some(c) = ctx.shared.cancel.lock().unwrap().as_ref() { if let Some(c) = ctx.shared.cancel.lock().unwrap().as_ref() {
c.store(true, Ordering::SeqCst); c.store(true, Ordering::SeqCst);
} }
+2 -4
View File
@@ -1,8 +1,7 @@
//! The Shortcuts screen: a short note on the in-stream capture model plus a reference of the //! The Shortcuts screen: a short note on the in-stream capture model plus a reference of the
//! keyboard shortcuts — reached from the Shortcuts button on the host list. The Windows //! keyboard shortcuts — reached from the Shortcuts button on the host list. The Windows
//! counterpart of the GTK client's Keyboard Shortcuts window; the bindings themselves live in //! counterpart of the GTK client's Keyboard Shortcuts window; the bindings themselves live in
//! the session window (and [`crate::input`] for the legacy builtin path), so both clients //! the session window, so both clients document the same set.
//! document the same set.
use super::style::*; use super::style::*;
use super::Screen; use super::Screen;
@@ -10,8 +9,7 @@ use windows_reactor::*;
/// The in-stream keyboard shortcuts, in the GTK Shortcuts window's order: the chord, then what it /// The in-stream keyboard shortcuts, in the GTK Shortcuts window's order: the chord, then what it
/// does. Read-only — the keyboard bindings live in the session window (`pf-presenter`'s run /// does. Read-only — the keyboard bindings live in the session window (`pf-presenter`'s run
/// loop; the legacy builtin path's in [`crate::input`]), the controller chord in its gamepad /// loop), the controller chord in its gamepad service.
/// service.
const STREAM_SHORTCUTS: &[(&str, &str)] = &[ const STREAM_SHORTCUTS: &[(&str, &str)] = &[
("F11 / Alt+Enter", "Toggle fullscreen"), ("F11 / Alt+Enter", "Toggle fullscreen"),
( (
+49 -10
View File
@@ -189,14 +189,21 @@ fn status_row(online: Option<bool>, badge: &str, kind: Pill) -> Element {
/// The in-tile rename editor (ContentDialog can't hold a text field): name box + save/cancel. /// The in-tile rename editor (ContentDialog can't hold a text field): name box + save/cancel.
/// No tap-to-connect while editing — a click into the box would bubble `Tapped` to the region. /// No tap-to-connect while editing — a click into the box would bubble `Tapped` to the region.
/// `initial` seeds the text box's displayed value and is CONSTANT for the life of the edit — the
/// field is uncontrolled, its live value kept in `live` (read at Save). Driving a *controlled* box
/// from an always-deferred `AsyncSetState` round-trip fights the caret on fast typing and can drop
/// the last char if Save is clicked before the write lands; an uncontrolled box + a ref sidesteps
/// both (and skips a full-page re-render per keystroke). See the seed block in `hosts_page`.
fn rename_editor( fn rename_editor(
draft: &str, initial: &str,
fp: String, fp: String,
live: HookRef<String>,
set_rename: AsyncSetState<Option<(String, String)>>, set_rename: AsyncSetState<Option<(String, String)>>,
) -> Element { ) -> Element {
let commit = { let commit = {
let (fp, draft, sr) = (fp.clone(), draft.to_string(), set_rename.clone()); let (fp, live, sr) = (fp.clone(), live.clone(), set_rename.clone());
move || { move || {
let draft = live.borrow();
let name = draft.trim(); let name = draft.trim();
if !name.is_empty() { if !name.is_empty() {
let mut known = KnownHosts::load(); let mut known = KnownHosts::load();
@@ -209,12 +216,12 @@ fn rename_editor(
} }
}; };
let on_changed = { let on_changed = {
let sr = set_rename.clone(); let live = live.clone();
move |s: String| sr.call(Some((fp.clone(), s))) move |s: String| live.set(s)
}; };
card( card(
vstack(( vstack((
text_box(draft) text_box(initial)
.placeholder_text("Host name") .placeholder_text("Host name")
.on_text_changed(on_changed), .on_text_changed(on_changed),
hstack(( hstack((
@@ -240,6 +247,14 @@ pub(crate) fn hosts_page(props: &HostsProps, cx: &mut RenderCx) -> Element {
let set_screen = &props.svc.set_screen; let set_screen = &props.svc.set_screen;
let set_status = &props.svc.set_status; let set_status = &props.svc.set_status;
let (manual, set_manual) = cx.use_state(String::new()); let (manual, set_manual) = cx.use_state(String::new());
// The Add-host field's live value, read by Connect at click time. This page's `use_state` is
// unreliable as the click's source of truth: while the modal is open the page usually has no
// reason to re-render (you open it precisely because the host ISN'T being discovered, so no
// discovery tick fires), and the top-down reconcile skips this unchanged-props subtree — so a
// sync `set_manual` write never re-renders the Connect button to re-capture the address, and it
// would connect to the empty mount-time value. Mirror every keystroke into this stable ref (the
// pair-screen PIN pattern). `manual` still drives the text box's displayed value.
let manual_live = cx.use_ref(String::new());
// "Add host" modal open state lives in ROOT (see `HostsProps`). // "Add host" modal open state lives in ROOT (see `HostsProps`).
let show_add = props.show_add; let show_add = props.show_add;
let set_show_add = &props.set_show_add; let set_show_add = &props.set_show_add;
@@ -249,6 +264,18 @@ pub(crate) fn hosts_page(props: &HostsProps, cx: &mut RenderCx) -> Element {
let rename = props.rename.clone(); let rename = props.rename.clone();
let set_forget = &props.set_forget; let set_forget = &props.set_forget;
let set_rename = &props.set_rename; let set_rename = &props.set_rename;
// The live rename draft, read at Save time (see `rename_editor`). Root `rename` carries only the
// INITIAL name, so it no longer round-trips per keystroke. Seed the draft each time the rename
// TARGET changes (start, cancel, or a switch to another host).
let rename_draft = cx.use_ref(String::new());
let rename_seed = cx.use_ref(Option::<String>::None);
{
let active = rename.as_ref().map(|(fp, _)| fp.clone());
if *rename_seed.borrow() != active {
rename_draft.set(rename.as_ref().map(|(_, n)| n.clone()).unwrap_or_default());
rename_seed.set(active);
}
}
let hover = Hover { let hover = Hover {
current: props.hover.clone(), current: props.hover.clone(),
set: props.set_hover.clone(), set: props.set_hover.clone(),
@@ -393,8 +420,13 @@ pub(crate) fn hosts_page(props: &HostsProps, cx: &mut RenderCx) -> Element {
for k in &known.hosts { for k in &known.hosts {
// Rust 2021 (no let-chains): match the "this tile is being renamed" case explicitly. // Rust 2021 (no let-chains): match the "this tile is being renamed" case explicitly.
if matches!(&rename, Some((fp, _)) if fp == &k.fp_hex) { if matches!(&rename, Some((fp, _)) if fp == &k.fp_hex) {
let (fp, draft) = rename.clone().unwrap(); let (fp, initial) = rename.clone().unwrap();
tiles.push(rename_editor(&draft, fp, set_rename.clone())); tiles.push(rename_editor(
&initial,
fp,
rename_draft.clone(),
set_rename.clone(),
));
continue; continue;
} }
let target = Target { let target = Target {
@@ -595,14 +627,15 @@ pub(crate) fn hosts_page(props: &HostsProps, cx: &mut RenderCx) -> Element {
// field). The scrim border fills the cell and is hit-testable, so it blocks the page behind; // field). The scrim border fills the cell and is hit-testable, so it blocks the page behind;
// it closes only via Cancel/Connect (a scrim tap would bubble `Tapped` up from the card too). // it closes only via Cancel/Connect (a scrim tap would bubble `Tapped` up from the card too).
let connect_manual = { let connect_manual = {
let (ctx2, ss, st, text, sa) = ( let (ctx2, ss, st, live, sa) = (
ctx.clone(), ctx.clone(),
set_screen.clone(), set_screen.clone(),
set_status.clone(), set_status.clone(),
manual.clone(), manual_live.clone(),
set_show_add.clone(), set_show_add.clone(),
); );
move || { move || {
let text = live.borrow();
let text = text.trim(); let text = text.trim();
if text.is_empty() { if text.is_empty() {
return; return;
@@ -640,7 +673,13 @@ pub(crate) fn hosts_page(props: &HostsProps, cx: &mut RenderCx) -> Element {
text_box(manual) text_box(manual)
.header("Address") .header("Address")
.placeholder_text("192.168.1.20 or my-pc.local") .placeholder_text("192.168.1.20 or my-pc.local")
.on_text_changed(move |s| set_manual.call(s)) .on_text_changed({
let live = manual_live.clone();
move |s: String| {
live.set(s.clone());
set_manual.call(s);
}
})
.margin(edges(0.0, 6.0, 0.0, 0.0)), .margin(edges(0.0, 6.0, 0.0, 0.0)),
hstack(( hstack((
button("Connect") button("Connect")
+1 -1
View File
@@ -23,7 +23,7 @@ pub(crate) fn licenses_page(set_screen: &AsyncSetState<Screen>) -> Element {
let app_card = card( let app_card = card(
vstack(( vstack((
text_block("punktfunk").font_size(15.0).semibold(), text_block("Punktfunk").font_size(15.0).semibold(),
text_block("Licensed under MIT OR Apache-2.0, at your option.") text_block("Licensed under MIT OR Apache-2.0, at your option.")
.font_size(12.0) .font_size(12.0)
.wrap() .wrap()
+15 -42
View File
@@ -1,8 +1,8 @@
//! The WinUI 3 (windows-reactor) application shell. //! The WinUI 3 (windows-reactor) application shell.
//! //!
//! Declarative React-like model: this root component routes on a `Screen` value held in //! Declarative React-like model: this root component routes on a `Screen` value held in
//! `use_async_state` so background threads (discovery, the session pump) can drive navigation. //! `use_async_state` so background threads (discovery, the spawned session's stdout reader) can
//! Each screen lives in its own submodule: //! drive navigation. Each screen lives in its own submodule:
//! //!
//! * [`hosts`] — saved/discovered/manual host list, plus per-host forget + speed test //! * [`hosts`] — saved/discovered/manual host list, plus per-host forget + speed test
//! * [`connect`] — the trust gate and session lifecycle glue (connect / request-access flows) //! * [`connect`] — the trust gate and session lifecycle glue (connect / request-access flows)
@@ -10,7 +10,7 @@
//! * [`speed`] — the per-host network speed test (probe burst over the real data plane) //! * [`speed`] — the per-host network speed test (probe burst over the real data plane)
//! * [`settings`] — persisted preferences · [`licenses`] — the license notices screen · //! * [`settings`] — persisted preferences · [`licenses`] — the license notices screen ·
//! [`help`] — the in-stream keyboard-shortcuts reference (reached from the host list) //! [`help`] — the in-stream keyboard-shortcuts reference (reached from the host list)
//! * [`stream`] — the live stream: `SwapChainPanel` + D3D11 presenter + HUD overlay //! * [`stream`] — the stream status card (the stream itself runs in the spawned session window)
//! * [`style`] — the shared look (cards, pills, monograms), following the windows-reactor //! * [`style`] — the shared look (cards, pills, monograms), following the windows-reactor
//! gallery: Mica backdrop, a centred max-width column, theme brushes (`ThemeRef`) //! gallery: Mica backdrop, a centred max-width column, theme brushes (`ThemeRef`)
//! //!
@@ -19,9 +19,6 @@
//! marks it dirty and re-renders it; an `AsyncSetState` written from a background thread does //! marks it dirty and re-renders it; an `AsyncSetState` written from a background thread does
//! NOT (the child is pruned when its props are unchanged) — so everything thread-driven //! NOT (the child is pruned when its props are unchanged) — so everything thread-driven
//! (discovery, HUD stats, speed-test results) is held as *root* state and passed down as props. //! (discovery, HUD stats, speed-test results) is held as *root* state and passed down as props.
//! The present + decoded-frame handoff crosses to the UI thread through a `Mutex` side-channel
//! and thread-locals (the windows-reactor SwapChainPanel sample's pattern), since the per-frame
//! present must not go through state/rerender.
mod connect; mod connect;
mod help; mod help;
@@ -36,7 +33,6 @@ mod style;
use crate::discovery::{self, DiscoveredHost}; use crate::discovery::{self, DiscoveredHost};
use crate::gamepad::GamepadService; use crate::gamepad::GamepadService;
use crate::session::Stats;
use crate::trust::{KnownHosts, Settings}; use crate::trust::{KnownHosts, Settings};
use hosts::HostsProps; use hosts::HostsProps;
use punktfunk_core::client::NativeClient; use punktfunk_core::client::NativeClient;
@@ -45,7 +41,6 @@ use std::collections::HashMap;
use std::sync::atomic::AtomicBool; use std::sync::atomic::AtomicBool;
use std::sync::{Arc, Mutex}; use std::sync::{Arc, Mutex};
use std::time::Duration; use std::time::Duration;
use stream::StreamProps;
use windows_reactor::*; use windows_reactor::*;
#[derive(Clone, PartialEq)] #[derive(Clone, PartialEq)]
@@ -88,7 +83,7 @@ pub(crate) struct Target {
} }
/// Stable app services handed to the page components as props. Each routed screen that uses /// Stable app services handed to the page components as props. Each routed screen that uses
/// hooks (`hosts_page`/`pair_page`/`stream_page`/`speed_page`) is mounted as its own /// hooks (`hosts_page`/`pair_page`/`speed_page`/`library_page`) is mounted as its own
/// `component(...)`, so its hooks live in an isolated slot list — calling them on the shared /// `component(...)`, so its hooks live in an isolated slot list — calling them on the shared
/// parent `cx` would change the hook order whenever the screen changes (reactor's /// parent `cx` would change the hook order whenever the screen changes (reactor's
/// Rules-of-Hooks guard aborts). /// Rules-of-Hooks guard aborts).
@@ -115,18 +110,12 @@ impl PartialEq for Svc {
} }
} }
/// Cross-thread handoff from the session pump (off-thread) to the stream page (UI thread): /// Cross-thread shell state driven off the UI thread: the current target, the live spawned
/// the connector (input sends), the decoded-frame channel (render thread), and the session's /// session child (Disconnect/Cancel kill it) and its latest stats line, plus the connect-flow
/// stop flag (the disconnect shortcut trips it). /// cancel flag and the discovery/library/speed-test generation guards.
#[derive(Default)] #[derive(Default)]
pub(crate) struct Shared { pub(crate) struct Shared {
#[allow(clippy::type_complexity)]
pub(crate) handoff:
Mutex<Option<(Arc<NativeClient>, crate::session::FrameRx, Arc<AtomicBool>)>>,
pub(crate) target: Mutex<Target>, pub(crate) target: Mutex<Target>,
/// Latest stream stats, written by the session's event loop and mirrored into reactor state
/// by the HUD poll thread to drive the overlay.
pub(crate) stats: Mutex<Stats>,
/// The live session child (spawn mode) — the status page's Disconnect and the /// The live session child (spawn mode) — the status page's Disconnect and the
/// request-access Cancel kill it. A FRESH handle is installed per spawn. /// request-access Cancel kill it. A FRESH handle is installed per spawn.
pub(crate) session: Mutex<crate::spawn::SessionChild>, pub(crate) session: Mutex<crate::spawn::SessionChild>,
@@ -157,14 +146,6 @@ pub struct AppCtx {
pub(crate) shared: Arc<Shared>, pub(crate) shared: Arc<Shared>,
} }
/// The legacy in-process streaming path (SwapChainPanel + D3D11VA) instead of the
/// spawned punktfunk-session window: the `PUNKTFUNK_BUILTIN_STREAM=1` env override — a
/// developer A/B knob only (the former Settings "Streaming engine" pick is gone), removed
/// with the legacy path once the Vulkan session is fully validated.
pub(crate) fn use_builtin_stream(_ctx: &AppCtx) -> bool {
std::env::var_os("PUNKTFUNK_BUILTIN_STREAM").is_some_and(|v| v == "1")
}
pub fn run(identity: (String, String), gamepad: GamepadService) -> windows_reactor::Result<()> { pub fn run(identity: (String, String), gamepad: GamepadService) -> windows_reactor::Result<()> {
let ctx = Arc::new(AppCtx { let ctx = Arc::new(AppCtx {
identity, identity,
@@ -302,10 +283,9 @@ fn root(cx: &mut RenderCx, ctx: &Arc<AppCtx>) -> Element {
} }
}); });
// HUD sample: the session event loop writes `shared.stats` and the input hooks track capture // HUD sample: the spawned session child's latest `stats:` line, mirrored into root state so
// state; this poll thread mirrors both into root state so the stream page gets them as a // the stream status page gets it as a *prop* (thread-driven state must be root state — see the
// *prop* (thread-driven state must be root state — see the module docs). The compare in // module docs). The compare in `AsyncSetState::call` makes the idle case free.
// `AsyncSetState::call` makes the idle case free.
cx.use_effect((), { cx.use_effect((), {
let shared = ctx.shared.clone(); let shared = ctx.shared.clone();
let set_hud = set_hud.clone(); let set_hud = set_hud.clone();
@@ -315,10 +295,6 @@ fn root(cx: &mut RenderCx, ctx: &Arc<AppCtx>) -> Element {
.spawn(move || loop { .spawn(move || loop {
std::thread::sleep(std::time::Duration::from_millis(400)); std::thread::sleep(std::time::Duration::from_millis(400));
set_hud.call(stream::HudSample { set_hud.call(stream::HudSample {
stats: *shared.stats.lock().unwrap(),
captured: crate::input::is_captured(),
visible: crate::input::hud_visible(),
present: crate::render::present_stats(),
stats_line: shared.stats_line.lock().unwrap().clone(), stats_line: shared.stats_line.lock().unwrap().clone(),
}); });
}) })
@@ -525,16 +501,13 @@ fn root(cx: &mut RenderCx, ctx: &Arc<AppCtx>) -> Element {
state: library, state: library,
}, },
), ),
// Spawn mode (the default): the stream runs in the punktfunk-session child's own // The stream runs in the punktfunk-session child's own window; this screen is a
// window; this screen is a status page (no hooks — inline is sound). The legacy // status page (no hooks — inline is sound).
// in-process SwapChainPanel page stays behind the "Streaming engine" setting / Screen::Stream => stream::session_page(ctx, &hud),
// PUNKTFUNK_BUILTIN_STREAM=1.
Screen::Stream if !use_builtin_stream(ctx) => stream::session_page(ctx, &hud),
Screen::Stream => component(stream::stream_page, StreamProps { svc, hud }),
}; };
// The Stream screen owns the SwapChainPanel + per-frame present; never wrap it in an animated // The Stream screen is a plain status card (the session child owns the real stream window);
// opacity/offset layer. Everything else slides + fades in on navigation. // it's shown without the navigation entrance tween. Everything else slides + fades in.
if matches!(screen, Screen::Stream) { if matches!(screen, Screen::Stream) {
return body; return body;
} }
+20 -4
View File
@@ -14,21 +14,28 @@ pub(crate) fn pair_page(props: &Svc, cx: &mut RenderCx) -> Element {
let set_screen = &props.set_screen; let set_screen = &props.set_screen;
let set_status = &props.set_status; let set_status = &props.set_status;
let (code, set_code) = cx.use_state(String::new()); let (code, set_code) = cx.use_state(String::new());
// The PIN's live value, read directly by the click handler. This page's props (`Svc`) never
// change, and root wraps every screen in an animated `border` that compares equal once the
// entrance tween settles — so the top-down reconcile `can_skip_update`s this subtree and never
// re-renders the pair component off its *local* `use_state`. A button rebuilt only at mount
// would forever capture the empty mount-time PIN (pairing then fails as a "wrong PIN"). Mirror
// every keystroke into this stable ref instead, so the click reads exactly what was typed.
let live_pin = cx.use_ref(String::new());
let target = ctx.shared.target.lock().unwrap().clone(); let target = ctx.shared.target.lock().unwrap().clone();
let pair_btn = { let pair_btn = {
let (ctx2, ss, st, code2, target2) = ( let (ctx2, ss, st, live, target2) = (
ctx.clone(), ctx.clone(),
set_screen.clone(), set_screen.clone(),
set_status.clone(), set_status.clone(),
code.clone(), live_pin.clone(),
target.clone(), target.clone(),
); );
button("Pair & Connect") button("Pair & Connect")
.accent() .accent()
.icon(Symbol::Accept) .icon(Symbol::Accept)
.on_click(move || { .on_click(move || {
let pin = code2.trim().to_string(); let pin = live.borrow().trim().to_string();
let (ctx3, ss, st, target3) = let (ctx3, ss, st, target3) =
(ctx2.clone(), ss.clone(), st.clone(), target2.clone()); (ctx2.clone(), ss.clone(), st.clone(), target2.clone());
std::thread::spawn(move || { std::thread::spawn(move || {
@@ -109,7 +116,16 @@ pub(crate) fn pair_page(props: &Svc, cx: &mut RenderCx) -> Element {
text_box(code) text_box(code)
.placeholder_text("PIN") .placeholder_text("PIN")
.font_size(28.0) .font_size(28.0)
.on_text_changed(move |s| set_code.call(s)), .on_text_changed({
let live = live_pin.clone();
move |s: String| {
// Record the live value for the click handler (the source of truth for the
// PIN), and mirror it into `code` so the field stays correct if anything ever
// does re-render this page (theme/DPI change).
live.set(s.clone());
set_code.call(s);
}
}),
hstack((pair_btn, cancel_btn)).spacing(8.0), hstack((pair_btn, cancel_btn)).spacing(8.0),
text_block( text_block(
"Don\u{2019}t have a PIN? Request access instead and approve this device on the host \ "Don\u{2019}t have a PIN? Request access instead and approve this device on the host \
+44 -21
View File
@@ -136,29 +136,37 @@ pub(crate) fn settings_page(
let s = ctx.settings.lock().unwrap().clone(); let s = ctx.settings.lock().unwrap().clone();
// --- Display --------------------------------------------------------------------------- // --- Display ---------------------------------------------------------------------------
// The D1 tri-state: Native, Match window (a virtual index 1, stored as the
// `match_window` flag), then the explicit sizes.
let (res_names, res_i) = { let (res_names, res_i) = {
let names: Vec<String> = RESOLUTIONS let names: Vec<String> = std::iter::once("Native display".to_string())
.chain(std::iter::once("Match window".to_string()))
.chain(
RESOLUTIONS
.iter() .iter()
.map(|&(w, h)| { .skip(1)
if w == 0 { .map(|&(w, h)| format!("{w} \u{00D7} {h}")),
"Native display".into() )
} else {
format!("{w} \u{00D7} {h}")
}
})
.collect(); .collect();
let i = RESOLUTIONS let i = if s.match_window {
1
} else {
RESOLUTIONS
.iter() .iter()
.position(|&(w, h)| w == s.width && h == s.height) .position(|&(w, h)| w == s.width && h == s.height)
.unwrap_or(0); .map(|i| if i == 0 { 0 } else { i + 1 })
.unwrap_or(0)
};
(names, i) (names, i)
}; };
let res_combo = setting_combo(ctx, "Resolution", res_names, res_i, |s, i| { let res_combo = setting_combo(ctx, "Resolution", res_names, res_i, |s, i| {
(s.width, s.height) = RESOLUTIONS[i]; s.match_window = i == 1;
(s.width, s.height) = if i <= 1 { (0, 0) } else { RESOLUTIONS[i - 1] };
}) })
.tooltip( .tooltip(
"The host creates a virtual display at exactly this size. \u{201C}Native display\u{201D} \ "The host creates a virtual display at exactly this size. \u{201C}Native display\u{201D} \
resolves to the monitor this window is on at connect.", resolves to the monitor this window is on at connect; \u{201C}Match window\u{201D} \
follows the stream window, including mid-stream resizes.",
); );
let (hz_names, hz_i) = { let (hz_names, hz_i) = {
let names: Vec<String> = REFRESH let names: Vec<String> = REFRESH
@@ -259,12 +267,13 @@ pub(crate) fn settings_page(
); );
// --- Input ----------------------------------------------------------------------------- // --- Input -----------------------------------------------------------------------------
// Which physical controller forwards as pad 0: automatic = the most recently connected. // Controller forwarding: Automatic forwards EVERY real controller, each as its own pad;
// Persisted by stable key (`Settings::forward_pad`, GTK parity) so the pin survives // pinning one restricts the session to that single controller (single-player). Persisted
// restarts AND reaches the spawned session binary, whose service applies the same key. // by stable key (`Settings::forward_pad`, GTK parity) so the pin survives restarts AND
// reaches the spawned session binary, whose service applies the same key.
let pads = ctx.gamepad.pads(); let pads = ctx.gamepad.pads();
let (fwd_names, fwd_i) = { let (fwd_names, fwd_i) = {
let mut names = vec!["Automatic (most recent)".to_string()]; let mut names = vec!["Automatic (all controllers)".to_string()];
names.extend(pads.iter().map(|p| { names.extend(pads.iter().map(|p| {
let kind = p.kind_label(); let kind = p.kind_label();
if kind.is_empty() { if kind.is_empty() {
@@ -293,16 +302,16 @@ pub(crate) fn settings_page(
} else { } else {
keys.get(sel - 1).cloned() keys.get(sel - 1).cloned()
}; };
// Apply live (the in-process service, legacy builtin streams) and persist — // Apply live to the gamepad service and persist — the spawned session
// the spawned session reads `forward_pad` at connect. // reads `forward_pad` at connect.
svc.set_pinned(key.clone()); svc.set_pinned(key.clone());
let mut s = ctx2.settings.lock().unwrap(); let mut s = ctx2.settings.lock().unwrap();
s.forward_pad = key.unwrap_or_default(); s.forward_pad = key.unwrap_or_default();
s.save(); s.save();
}) })
.tooltip( .tooltip(
"Exactly one controller is forwarded to the host; \u{201C}Automatic\u{201D} \ "Every connected controller is forwarded, each as its own player. Pick one \
picks the most recently connected.", to force single-player \u{2014} only it reaches the host.",
) )
}; };
let (pad_names, pad_i) = presets(GAMEPADS, |v| { let (pad_names, pad_i) = presets(GAMEPADS, |v| {
@@ -361,6 +370,16 @@ pub(crate) fn settings_page(
"Adds \u{201C}Browse library\u{2026}\u{201D} to paired hosts \u{2014} pick a game and it \ "Adds \u{201C}Browse library\u{2026}\u{201D} to paired hosts \u{2014} pick a game and it \
launches in the stream. Mirrors the Apple client's toggle.", launches in the stream. Mirrors the Apple client's toggle.",
); );
// App identity + version at the top of the About card (the WinUI Settings convention; the About
// screen previously showed no version at all). CARGO_PKG_VERSION is the workspace version, baked
// in at compile time.
let about_identity = vstack((
text_block("Punktfunk").font_size(20.0).semibold(),
text_block(concat!("Version ", env!("CARGO_PKG_VERSION")))
.font_size(12.0)
.foreground(ThemeRef::SecondaryText),
))
.spacing(2.0);
// The selected section's content — per-control guidance lives on hover tooltips, so the // The selected section's content — per-control guidance lives on hover tooltips, so the
// card is just the controls. // card is just the controls.
@@ -395,7 +414,11 @@ pub(crate) fn settings_page(
), ),
"about" => ( "about" => (
"About", "About",
settings_card(vec![library_toggle.into(), licenses_button.into()]), settings_card(vec![
about_identity.into(),
library_toggle.into(),
licenses_button.into(),
]),
), ),
_ => ( _ => (
"Display", "Display",
+1 -1
View File
@@ -4,7 +4,7 @@
use super::style::*; use super::style::*;
use super::{Screen, Svc}; use super::{Screen, Svc};
use crate::session::run_speed_probe; use crate::probe::run_speed_probe;
use windows_reactor::*; use windows_reactor::*;
/// Speed-test lifecycle. Held as ROOT state (the probe worker completes it via /// Speed-test lifecycle. Held as ROOT state (the probe worker completes it via
+7 -311
View File
@@ -1,163 +1,20 @@
//! The stream page: a `SwapChainPanel` whose composition swapchain is created (and bound) once on //! The stream status page: streams run in the spawned `punktfunk-session` child's own window,
//! the UI thread, then handed — presenter and all — to the dedicated render thread //! so the shell shows a status card in the app's card language — host header, the child's live
//! ([`crate::render`]), which presents decoded frames at stream cadence. The page itself only //! `stats:` line as a chip row + stage lines, the in-window shortcuts, and a Disconnect.
//! forwards panel size/DPI changes and draws the status-chip HUD overlay (mode · decode path ·
//! HDR · fps/goodput · end-to-end latency + stage equation · capture hint).
use super::style::{edges, uniform}; use super::style::{edges, uniform};
use super::Svc;
use crate::present::Presenter;
use crate::render::{self, RenderThread};
use crate::session::Stats;
use punktfunk_core::client::NativeClient;
use punktfunk_core::config::Mode;
use std::cell::RefCell;
use std::sync::Arc; use std::sync::Arc;
use windows_reactor::*; use windows_reactor::*;
/// One HUD refresh: the latest session stats, the input hooks' capture state, and the render /// One HUD refresh: the session child's latest formatted `stats:` line, mirrored into root state
/// thread's display-side window. Mirrored into root state by the poll thread (`pf-hud`) and /// by the poll thread (`pf-hud`) and passed down as a prop.
/// passed down as a prop.
#[derive(Clone, Default, PartialEq)] #[derive(Clone, Default, PartialEq)]
pub(crate) struct HudSample { pub(crate) struct HudSample {
pub(crate) stats: Stats, /// The session child's latest formatted `stats:` line, for the status page. Empty before the
pub(crate) captured: bool, /// child's first stats window.
/// Whether the stats overlay should be shown — the Settings default at stream start, then
/// whatever Ctrl+Alt+Shift+S last set (see [`crate::input::hud_visible`]). Carried in the
/// sample so a live toggle changes the sample and re-renders the page (the stream page is a
/// child component — only a changed prop re-renders it).
pub(crate) visible: bool,
/// The render thread's glass-side window (presents/s, skips, end-to-end p50/p95, display
/// stage p50) — see [`crate::render::present_stats`].
pub(crate) present: crate::render::PresentStats,
/// Spawn mode: the session child's latest formatted `stats:` line, for the status
/// page. Empty in builtin mode / before the first window.
pub(crate) stats_line: String, pub(crate) stats_line: String,
} }
/// Props for the stream page: the services plus the live HUD sample that drives the overlay
/// (compared by value, so each new sample re-renders the overlay).
#[derive(Clone)]
pub(crate) struct StreamProps {
pub(crate) svc: Svc,
pub(crate) hud: HudSample,
}
impl PartialEq for StreamProps {
fn eq(&self, other: &Self) -> bool {
self.svc == other.svc && self.hud == other.hud
}
}
thread_local! {
/// Frames + host clock offset, stashed by the mount effect for `on_mounted` (which fires
/// later, once the native panel exists).
static PENDING: RefCell<Option<(crate::session::FrameRx, std::sync::Arc<std::sync::atomic::AtomicI64>)>> = const { RefCell::new(None) };
/// The live render thread; stopped + joined by the unmount cleanup (before panel teardown).
static RENDER: RefCell<Option<RenderThread>> = const { RefCell::new(None) };
}
/// The app window's DPI (96 when the window can't be found — then DIPs == pixels). Reactor's
/// `on_resize` reports DIPs and exposes no CompositionScale, so the window DPI is the scale.
fn window_dpi() -> u32 {
use windows::Win32::UI::HiDpi::GetDpiForWindow;
use windows::Win32::UI::WindowsAndMessaging::FindWindowW;
unsafe {
FindWindowW(None, windows::core::w!("Punktfunk"))
.ok()
.map(|h| GetDpiForWindow(h))
.filter(|d| *d > 0)
.unwrap_or(96)
}
}
pub(crate) fn stream_page(props: &StreamProps, cx: &mut RenderCx) -> Element {
let ctx = &props.svc.ctx;
// Take the connector + frames handoff once on mount; keep the connector alive (and for input)
// in a use_ref, stash frames for `on_mounted`, install the input hooks. The cleanup stops the
// render thread FIRST (it must not present into a panel that's tearing down), then removes
// the input hooks.
let connector_ref = cx.use_ref::<Option<Arc<NativeClient>>>(None);
cx.use_effect_with_cleanup((), {
let shared = ctx.shared.clone();
let (inhibit, show_stats) = {
let s = ctx.settings.lock().unwrap();
(s.inhibit_shortcuts, s.show_stats)
};
let connector_ref = connector_ref.clone();
move || {
if let Some((connector, frames, stop)) = shared.handoff.lock().unwrap().take() {
let mode = connector.mode();
let clock_offset = connector.clock_offset_shared();
connector_ref.set(Some(connector.clone()));
PENDING.with(|c| *c.borrow_mut() = Some((frames, clock_offset)));
crate::input::install(connector, mode, inhibit, show_stats, stop);
}
Some(|| {
RENDER.with(|c| {
if let Some(mut rt) = c.borrow_mut().take() {
rt.stop_and_join();
}
});
PENDING.with(|c| c.borrow_mut().take());
crate::input::uninstall();
})
}
});
let mode = connector_ref.borrow().as_ref().map(|c| c.mode());
let host = ctx.shared.target.lock().unwrap().name.clone();
let mut layers: Vec<Element> = vec![swap_chain_panel()
.on_mounted(|panel| {
// Placeholder size — the first `on_resize` (fired after the first layout pass)
// resizes to the panel's real pixel size.
let dpi = window_dpi();
match Presenter::new(1280, 720, dpi) {
Ok(p) => {
if let Err(e) = panel.set_swap_chain(p.swap_chain()) {
tracing::error!(error = %e, "set_swap_chain");
return;
}
if let Some((frames, clock_offset)) = PENDING.with(|c| c.borrow_mut().take()) {
let shared = render::RenderShared::new(1280, 720, dpi);
RENDER.with(|cell| {
*cell.borrow_mut() =
Some(render::spawn(p, frames, shared, clock_offset));
});
tracing::info!(dpi, "stream presenter bound — render thread started");
}
}
Err(e) => tracing::error!(error = %e, "create presenter"),
}
})
.on_resize(|w, h| {
// DIPs → physical pixels; the presenter maps back via SetMatrixTransform.
let dpi = window_dpi();
let px = |v: f64| (v * f64::from(dpi) / 96.0).round() as u32;
RENDER.with(|cell| {
if let Some(rt) = cell.borrow().as_ref() {
rt.shared().set_dpi(dpi);
rt.shared().set_size(px(w), px(h));
}
});
})
.into()];
// The overlay follows the LIVE visibility (Settings default, then Ctrl+Alt+Shift+S): the page
// re-renders on every HUD sample (~400 ms), so a toggle takes effect promptly mid-stream.
if props.hud.visible {
layers.push(hud_overlay(&props.hud, mode, &host));
}
// Flash the shortcut key set for the first few seconds of every session, regardless of the
// HUD setting — so "how do I get back out" is answered the moment the stream comes up (parity
// with the GTK client's stream-start hint). Uptime drives it, so it needs no timer/state: the
// HUD poll re-renders the page each second and the banner drops once the session passes the
// threshold.
if props.hud.stats.uptime_secs < START_HINT_SECS {
layers.push(start_hint());
}
grid(layers).into()
}
/// Spawn mode's Stream screen: the stream runs in the punktfunk-session child's own /// Spawn mode's Stream screen: the stream runs in the punktfunk-session child's own
/// window, so the shell shows a status card in the app's card language — monogram + /// window, so the shell shows a status card in the app's card language — monogram +
/// host header, the child's live `stats:` line as a chip row + stage lines, the /// host header, the child's live `stats:` line as a chip row + stage lines, the
@@ -277,164 +134,3 @@ pub(crate) fn session_page(ctx: &Arc<super::AppCtx>, hud: &HudSample) -> Element
.vertical_alignment(VerticalAlignment::Center) .vertical_alignment(VerticalAlignment::Center)
.into() .into()
} }
/// How long the stream-start shortcut banner stays up (seconds of session uptime).
const START_HINT_SECS: u32 = 6;
/// The stream-start shortcut banner: the full client key set on a translucent pill, bottom-centre,
/// shown for [`START_HINT_SECS`] at the start of every session (see the call site). Independent of
/// the stats overlay, so it appears even with the HUD turned off.
fn start_hint() -> Element {
border(
text_block(
"Click the stream to capture \u{00B7} Ctrl+Alt+Shift+Q releases \u{00B7} \
Ctrl+Alt+Shift+D disconnects \u{00B7} Ctrl+Alt+Shift+S stats \u{00B7} F11 fullscreen",
)
.font_size(12.0)
.semibold()
.foreground(Color::rgb(235, 235, 235)),
)
.background(Color::rgb(0, 0, 0))
.corner_radius(10.0)
.padding(edges(14.0, 8.0, 14.0, 8.0))
.opacity(0.82)
.horizontal_alignment(HorizontalAlignment::Center)
.vertical_alignment(VerticalAlignment::Bottom)
.margin(edges(0.0, 0.0, 0.0, 28.0))
.into()
}
/// A small chip for the dark HUD: coloured text on a translucent dark fill.
fn hud_chip(text: &str, color: Color) -> Border {
border(
text_block(text)
.font_size(11.0)
.semibold()
.foreground(color),
)
.background(Color::rgb(38, 38, 38))
.corner_radius(8.0)
.padding(edges(8.0, 2.0, 8.0, 2.0))
}
/// The negotiated wire codec's display name (`quic::CODEC_*` bit → label).
fn codec_name(bits: u8) -> &'static str {
match bits {
punktfunk_core::quic::CODEC_H264 => "H.264",
punktfunk_core::quic::CODEC_AV1 => "AV1",
_ => "HEVC",
}
}
/// `mm:ss` (or `h:mm:ss`) session time.
fn fmt_uptime(secs: u32) -> String {
let (h, m, s) = (secs / 3600, secs / 60 % 60, secs % 60);
if h > 0 {
format!("{h}:{m:02}:{s:02}")
} else {
format!("{m}:{s:02}")
}
}
/// The streaming HUD overlay (top-right), unified stats vocabulary (design/stats-unification.md):
/// a chip row (mode · codec · decode path · HDR), a stream line (received fps · goodput ·
/// presenter fps), the end-to-end headline (capture→on-glass p50/p95, host-clock corrected), the
/// stage equation (= host + network + decode + display when the host reports 0xCF timings, else
/// the combined = host+network + decode + display; stage p50s), a session line
/// (host · time · loss/skips), and the shortcut hints. Layered over the `SwapChainPanel` in the
/// same grid cell.
fn hud_overlay(hud: &HudSample, mode: Option<Mode>, host: &str) -> Element {
let stats = &hud.stats;
let present = &hud.present;
let res = mode
.map(|m| format!("{}\u{00D7}{}@{}", m.width, m.height, m.refresh_hz))
.unwrap_or_else(|| "\u{2014}".into());
let mut chips: Vec<Element> = vec![
hud_chip(&res, Color::rgb(235, 235, 235)).into(),
hud_chip(codec_name(stats.codec), Color::rgb(180, 190, 255)).into(),
];
chips.push(if stats.hardware {
hud_chip("GPU decode", Color::rgb(120, 220, 150)).into()
} else {
hud_chip("CPU decode", Color::rgb(240, 190, 90)).into()
});
if stats.hdr {
chips.push(hud_chip("HDR", Color::rgb(255, 205, 90)).into());
}
// Received fps + goodput, plus the presenter's own rate (Moonlight's "Rendering frame rate"
// analog — how often the display actually gets a new frame).
let stream_line = format!(
"{:.0} fps \u{00B7} {:.1} Mb/s \u{00B7} display {} fps",
stats.fps, stats.mbps, present.fps
);
// The headline: end-to-end capture→displayed, measured directly post-Present (never the sum
// of the stage percentiles). `(same-host clock)` flags an uncorrected clock (offset == 0:
// same host, or the host skipped the skew handshake).
let mut e2e_line = format!(
"end-to-end {:.1} ms p50 \u{00B7} {:.1} p95 \u{00B7} capture\u{2192}on-glass",
present.e2e_p50_ms, present.e2e_p95_ms
);
if stats.same_host {
e2e_line.push_str(" (same-host clock)");
}
// The equation: the stages tile the headline interval per frame; the window p50s only
// approximately sum (percentiles aren't additive). With per-AU 0xCF host timings the opaque
// `host+network` term splits into `host` (host capture→sent) + `network` (the remainder);
// an old host emits none and the combined term stays.
let stage_line = if stats.split {
format!(
"= host {:.1} + network {:.1} + decode {:.1} + display {:.1}",
stats.host_ms, stats.net_ms, stats.decode_ms, present.display_p50_ms
)
} else {
format!(
"= host+network {:.1} + decode {:.1} + display {:.1}",
stats.hostnet_ms, stats.decode_ms, present.display_p50_ms
)
};
let mut session_bits: Vec<String> = Vec::new();
if !host.is_empty() {
session_bits.push(host.to_string());
}
// `lost` = unrecoverable network drops (session-cumulative); `skipped` = the render thread's
// newest-wins drops last window (expected when the stream outpaces the display).
session_bits.push(fmt_uptime(stats.uptime_secs));
session_bits.push(format!("{} lost", stats.dropped));
if present.skipped > 0 {
session_bits.push(format!("{} skipped", present.skipped));
}
let session_line = session_bits.join(" \u{00B7} ");
let hint = if hud.captured {
"Ctrl+Alt+Shift+Q releases the mouse \u{00B7} Ctrl+Alt+Shift+D disconnects \u{00B7} \
Ctrl+Alt+Shift+S stats \u{00B7} F11 fullscreen"
} else {
"Click the stream to capture \u{00B7} Ctrl+Alt+Shift+D disconnects \u{00B7} \
Ctrl+Alt+Shift+S stats \u{00B7} F11 fullscreen"
};
let dim = |t: &str| {
text_block(t)
.font_size(11.0)
.foreground(Color::rgb(210, 210, 210))
};
border(
vstack((
hstack(chips).spacing(6.0),
dim(&stream_line),
dim(&e2e_line),
dim(&stage_line),
dim(&session_line),
text_block(hint)
.font_size(11.0)
.foreground(Color::rgb(150, 150, 150)),
))
.spacing(6.0),
)
.background(Color::rgb(0, 0, 0))
.corner_radius(10.0)
.padding(uniform(10.0))
.opacity(0.82)
.horizontal_alignment(HorizontalAlignment::Right)
.vertical_alignment(VerticalAlignment::Top)
.margin(uniform(12.0))
.into()
}
-308
View File
@@ -1,308 +0,0 @@
//! Audio: playback (decoded PCM → a WASAPI shared-mode render stream) and the microphone
//! uplink (WASAPI capture → Opus → 0xCB datagrams, the inverse of the host's virtual mic).
//!
//! The WASAPI analogue of the Linux client's PipeWire backend. Playback mirrors the host's
//! virtual-mic producer's adaptive jitter buffer: the session pump pushes 5 ms Opus-decoded
//! chunks on the network clock; the WASAPI render thread pulls whole event-driven quanta on
//! the device clock. Prime to ~3 quanta before producing, cap the ring so latency stays
//! bounded, re-prime after a real drain.
//!
//! WASAPI objects are COM-apartment-bound and not `Send`, so they live on a dedicated thread
//! (the same discipline as the host's `wasapi_cap`); only the channel + stop flag + join
//! handle cross the boundary.
use anyhow::{anyhow, Context, Result};
use punktfunk_core::client::NativeClient;
use std::collections::VecDeque;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::mpsc::{Receiver, SyncSender, TrySendError};
use std::sync::Arc;
use std::time::Duration;
use wasapi::{DeviceEnumerator, Direction, SampleType, StreamMode, WaveFormat};
const SAMPLE_RATE: usize = 48_000;
/// The microphone uplink stays stereo (the host's virtual mic is stereo). The render path is
/// multichannel — its channel count + block align are runtime, driven by the host-resolved layout.
const CHANNELS: usize = 2;
/// Mic frames are 20 ms (960 samples/channel) — any size ≤ 120 ms is fine host-side.
const MIC_FRAME: usize = 960;
pub struct AudioPlayer {
pcm_tx: SyncSender<Vec<f32>>,
stop: Arc<AtomicBool>,
thread: Option<std::thread::JoinHandle<()>>,
}
impl AudioPlayer {
/// Spawn the WASAPI render thread for `channels` (2/6/8, canonical wire order
/// FL FR FC LFE RL RR SL SR). Failure (no render endpoint on this box) is survivable — the
/// caller streams video-only.
pub fn spawn(channels: u8) -> Result<AudioPlayer> {
// 64 × 5 ms = 320 ms of slack between the pump and the WASAPI loop.
let (pcm_tx, pcm_rx) = std::sync::mpsc::sync_channel::<Vec<f32>>(64);
let stop = Arc::new(AtomicBool::new(false));
let (ready_tx, ready_rx) = std::sync::mpsc::sync_channel::<Result<()>>(1);
let stop_t = stop.clone();
let thread = std::thread::Builder::new()
.name("punktfunk-audio".into())
.spawn(move || {
if let Err(e) = render_thread(pcm_rx, stop_t, ready_tx, channels) {
tracing::warn!(error = format!("{e:#}"), "audio playback thread ended");
}
})
.context("spawn audio thread")?;
match ready_rx.recv_timeout(Duration::from_secs(3)) {
Ok(Ok(())) => {
tracing::info!(channels, "WASAPI render: 48 kHz f32 (default endpoint)");
Ok(AudioPlayer {
pcm_tx,
stop,
thread: Some(thread),
})
}
Ok(Err(e)) => Err(e),
Err(_) => Err(anyhow!(
"wasapi render init timed out (no render endpoint?)"
)),
}
}
/// Queue one interleaved f32 chunk (in the session's channel layout). Drops the chunk if the
/// WASAPI side is wedged (the renderer conceals the gap; never block the session pump).
pub fn push(&self, pcm: Vec<f32>) {
if let Err(TrySendError::Disconnected(_)) = self.pcm_tx.try_send(pcm) {
// Thread already dead — Drop will reap it; nothing to do per-chunk.
}
}
}
impl Drop for AudioPlayer {
fn drop(&mut self) {
self.stop.store(true, Ordering::SeqCst);
if let Some(t) = self.thread.take() {
let _ = t.join();
}
}
}
fn render_thread(
pcm_rx: Receiver<Vec<f32>>,
stop: Arc<AtomicBool>,
ready: SyncSender<Result<()>>,
channels: u8,
) -> Result<()> {
if let Err(e) = wasapi::initialize_mta()
.ok()
.context("CoInitializeEx (MTA)")
{
let _ = ready.send(Err(e));
return Ok(());
}
let res = (|| -> Result<()> {
// F32LE interleaved: channels × 4 bytes/sample. Stereo (channels == 2) is byte-identical
// to the old fixed path (mask 0x3, block align 8).
let block_align = channels as usize * 4;
let device = DeviceEnumerator::new()
.context("DeviceEnumerator")?
.get_default_device(&Direction::Render)
.context("default render endpoint")?;
let mut audio_client = device.get_iaudioclient().context("IAudioClient")?;
// The explicit dwChannelMask is the wire order (FL FR FC LFE RL RR SL SR); 5.1 = 0x3F,
// 7.1 = 0x63F. WASAPI delivers channels in ascending mask-bit order, which equals the wire
// order, so the render mapping is the identity — no permute. `autoconvert` (below) lets the
// audio engine downmix when the endpoint has fewer speakers.
let desired = WaveFormat::new(
32,
32,
&SampleType::Float,
SAMPLE_RATE,
channels as usize,
Some(punktfunk_core::audio::wasapi_channel_mask(channels)),
);
let (default_period, _min_period) =
audio_client.get_device_period().context("device period")?;
let mode = StreamMode::EventsShared {
autoconvert: true,
buffer_duration_hns: default_period,
};
audio_client
.initialize_client(&desired, &Direction::Render, &mode)
.context("initialize render client")?;
let h_event = audio_client.set_get_eventhandle().context("event handle")?;
let render_client = audio_client
.get_audiorenderclient()
.context("IAudioRenderClient")?;
audio_client.start_stream().context("start render stream")?;
let _ = ready.send(Ok(()));
// Adaptive jitter buffer, in f32-byte units (same shape as the host's virtual mic).
let mut ring: VecDeque<u8> = VecDeque::new();
let mut primed = false;
let mut out = Vec::new(); // per-quantum scratch, reused across iterations
while !stop.load(Ordering::Relaxed) {
if h_event.wait_for_event(100).is_err() {
continue;
}
// Drain everything the pump has queued into the ring.
while let Ok(chunk) = pcm_rx.try_recv() {
for s in chunk {
ring.extend(s.to_le_bytes());
}
}
let avail_frames = audio_client
.get_available_space_in_frames()
.context("available space")? as usize;
if avail_frames == 0 {
continue;
}
let want_bytes = avail_frames * block_align;
// Prime to ~3 quanta; cap at ~1 quantum of slack beyond that; re-prime on drain.
let target = (3 * want_bytes).clamp(720 * block_align, 9600 * block_align);
let cap = target.max(want_bytes) + want_bytes;
if ring.len() > cap {
ring.drain(..ring.len() - cap);
}
if !primed && ring.len() >= target {
primed = true;
}
out.clear();
out.resize(want_bytes, 0);
if primed {
let n = ring.len().min(want_bytes);
for (dst, b) in out.iter_mut().zip(ring.drain(..n)) {
*dst = b;
}
}
if ring.is_empty() {
primed = false;
}
render_client
.write_to_device(avail_frames, &out, None)
.context("write_to_device")?;
}
audio_client.stop_stream().ok();
Ok(())
})();
if let Err(ref e) = res {
let _ = ready.send(Err(anyhow!("{e:#}")));
}
res
}
/// The microphone uplink: capture the default input device, Opus-encode 20 ms chunks, ship
/// them as 0xCB datagrams into the host's virtual mic source.
pub struct MicStreamer {
stop: Arc<AtomicBool>,
thread: Option<std::thread::JoinHandle<()>>,
}
impl MicStreamer {
pub fn spawn(connector: Arc<NativeClient>) -> Result<MicStreamer> {
let stop = Arc::new(AtomicBool::new(false));
let stop_t = stop.clone();
let thread = std::thread::Builder::new()
.name("punktfunk-mic".into())
.spawn(move || {
if let Err(e) = mic_thread(&connector, stop_t) {
tracing::warn!(error = format!("{e:#}"), "mic uplink thread ended");
}
})
.context("spawn mic thread")?;
Ok(MicStreamer {
stop,
thread: Some(thread),
})
}
}
impl Drop for MicStreamer {
fn drop(&mut self) {
self.stop.store(true, Ordering::SeqCst);
if let Some(t) = self.thread.take() {
let _ = t.join();
}
}
}
fn mic_thread(connector: &Arc<NativeClient>, stop: Arc<AtomicBool>) -> Result<()> {
wasapi::initialize_mta()
.ok()
.context("CoInitializeEx (MTA)")?;
let mut encoder = opus::Encoder::new(
SAMPLE_RATE as u32,
opus::Channels::Stereo,
opus::Application::Voip,
)
.map_err(|e| anyhow!("opus encoder: {e}"))?;
let _ = encoder.set_bitrate(opus::Bitrate::Bits(64_000));
let device = DeviceEnumerator::new()
.context("DeviceEnumerator")?
.get_default_device(&Direction::Capture)
.context("default capture endpoint (no microphone?)")?;
let mut audio_client = device.get_iaudioclient().context("IAudioClient")?;
let desired = WaveFormat::new(32, 32, &SampleType::Float, SAMPLE_RATE, CHANNELS, None);
let (default_period, _min_period) =
audio_client.get_device_period().context("device period")?;
let mode = StreamMode::EventsShared {
autoconvert: true,
buffer_duration_hns: default_period,
};
audio_client
.initialize_client(&desired, &Direction::Capture, &mode)
.context("initialize capture client")?;
let h_event = audio_client.set_get_eventhandle().context("event handle")?;
let capture_client = audio_client
.get_audiocaptureclient()
.context("IAudioCaptureClient")?;
audio_client
.start_stream()
.context("start capture stream")?;
let mut bytes: VecDeque<u8> = VecDeque::new();
let mut ring: VecDeque<f32> = VecDeque::new();
let mut out = vec![0u8; 4000];
let mut seq = 0u32;
while !stop.load(Ordering::Relaxed) {
if h_event.wait_for_event(100).is_err() {
continue;
}
loop {
match capture_client.get_next_packet_size() {
Ok(Some(0)) | Ok(None) => break,
Ok(Some(_n)) => {
capture_client
.read_from_device_to_deque(&mut bytes)
.context("read capture")?;
}
Err(e) => return Err(anyhow!("get_next_packet_size: {e}")),
}
}
let whole = (bytes.len() / 4) * 4;
for c in bytes.drain(..whole).collect::<Vec<u8>>().chunks_exact(4) {
ring.push_back(f32::from_le_bytes([c[0], c[1], c[2], c[3]]));
}
// Ship every complete 20 ms stereo frame.
while ring.len() >= MIC_FRAME * CHANNELS {
let pcm: Vec<f32> = ring.drain(..MIC_FRAME * CHANNELS).collect();
match encoder.encode_float(&pcm, &mut out) {
Ok(len) => {
let pts = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_nanos() as u64)
.unwrap_or(0);
let _ = connector.send_mic(seq, pts, out[..len].to_vec());
seq = seq.wrapping_add(1);
}
Err(e) => tracing::debug!(error = %e, "opus mic encode"),
}
}
}
audio_client.stop_stream().ok();
Ok(())
}
+6 -46
View File
@@ -74,30 +74,26 @@ fn pref_for_type(t: sdl3::gamepad::GamepadType) -> GamepadPref {
} }
enum Ctl { enum Ctl {
Attach(Arc<NativeClient>),
Detach,
Pin(Option<String>), Pin(Option<String>),
} }
#[derive(Clone)] #[derive(Clone)]
pub struct GamepadService { pub struct GamepadService {
pads: Arc<Mutex<Vec<PadInfo>>>, pads: Arc<Mutex<Vec<PadInfo>>>,
active: Arc<Mutex<Option<PadInfo>>>,
// `Arc<Mutex<…>>` (not a bare `Sender`, which is `!Sync`) so the service is `Sync` — the // `Arc<Mutex<…>>` (not a bare `Sender`, which is `!Sync`) so the service is `Sync` — the
// WinUI app shares it across the UI thread and the session-pump thread (attach/detach). // WinUI app shares it across the UI thread and the settings-pin path.
ctl: Arc<Mutex<Sender<Ctl>>>, ctl: Arc<Mutex<Sender<Ctl>>>,
} }
impl GamepadService { impl GamepadService {
pub fn start() -> GamepadService { pub fn start() -> GamepadService {
let pads = Arc::new(Mutex::new(Vec::new())); let pads = Arc::new(Mutex::new(Vec::new()));
let active = Arc::new(Mutex::new(None));
let (ctl, ctl_rx) = std::sync::mpsc::channel(); let (ctl, ctl_rx) = std::sync::mpsc::channel();
let (p, a) = (pads.clone(), active.clone()); let p = pads.clone();
if let Err(e) = std::thread::Builder::new() if let Err(e) = std::thread::Builder::new()
.name("punktfunk-gamepad".into()) .name("punktfunk-gamepad".into())
.spawn(move || { .spawn(move || {
if let Err(e) = run(&p, &a, &ctl_rx) { if let Err(e) = run(&p, &ctl_rx) {
tracing::warn!(error = %e, "gamepad service ended — pads disabled"); tracing::warn!(error = %e, "gamepad service ended — pads disabled");
} }
}) })
@@ -106,7 +102,6 @@ impl GamepadService {
} }
GamepadService { GamepadService {
pads, pads,
active,
ctl: Arc::new(Mutex::new(ctl)), ctl: Arc::new(Mutex::new(ctl)),
} }
} }
@@ -116,33 +111,13 @@ impl GamepadService {
self.pads.lock().unwrap().clone() self.pads.lock().unwrap().clone()
} }
pub fn active(&self) -> Option<PadInfo> {
self.active.lock().unwrap().clone()
}
/// Pin the forwarded controller by stable key (`PadInfo::key`) — `None` = automatic. /// Pin the forwarded controller by stable key (`PadInfo::key`) — `None` = automatic.
/// The pin survives the pad disconnecting: it re-applies the moment a matching /// The pin survives the pad disconnecting: it re-applies the moment a matching
/// controller shows up again (same semantics as `pf-client-core`'s service). /// controller shows up again (same semantics as `pf-client-core`'s service). The spawned
/// `punktfunk-session` binary owns the actual forwarding; this persists the selection.
pub fn set_pinned(&self, key: Option<String>) { pub fn set_pinned(&self, key: Option<String>) {
let _ = self.ctl.lock().unwrap().send(Ctl::Pin(key)); let _ = self.ctl.lock().unwrap().send(Ctl::Pin(key));
} }
pub fn attach(&self, connector: Arc<NativeClient>) {
let _ = self.ctl.lock().unwrap().send(Ctl::Attach(connector));
}
pub fn detach(&self) {
let _ = self.ctl.lock().unwrap().send(Ctl::Detach);
}
/// What "Automatic" resolves to right now — the virtual pad matching the physical one
/// (Swift parity); no pad connected leaves the host's own default.
pub fn auto_pref(&self) -> GamepadPref {
match self.active() {
Some(p) => p.pref,
None => GamepadPref::Auto,
}
}
} }
fn send(connector: &NativeClient, kind: InputKind, code: u32, x: i32) { fn send(connector: &NativeClient, kind: InputKind, code: u32, x: i32) {
@@ -404,11 +379,7 @@ impl Worker {
} }
#[allow(clippy::too_many_lines)] #[allow(clippy::too_many_lines)]
fn run( fn run(pads_out: &Mutex<Vec<PadInfo>>, ctl: &Receiver<Ctl>) -> Result<(), String> {
pads_out: &Mutex<Vec<PadInfo>>,
active_out: &Mutex<Option<PadInfo>>,
ctl: &Receiver<Ctl>,
) -> Result<(), String> {
// Off-main-thread + no video subsystem: keep SDL away from signals, poll pads on its own // Off-main-thread + no video subsystem: keep SDL away from signals, poll pads on its own
// thread. // thread.
sdl3::hint::set("SDL_NO_SIGNAL_HANDLERS", "1"); sdl3::hint::set("SDL_NO_SIGNAL_HANDLERS", "1");
@@ -437,23 +408,12 @@ fn run(
let mut list: Vec<PadInfo> = w.order.iter().filter_map(|&id| w.pad_info(id)).collect(); let mut list: Vec<PadInfo> = w.order.iter().filter_map(|&id| w.pad_info(id)).collect();
list.reverse(); // most recent first — the Settings list order list.reverse(); // most recent first — the Settings list order
*pads_out.lock().unwrap() = list; *pads_out.lock().unwrap() = list;
*active_out.lock().unwrap() = w.active_id().and_then(|id| w.pad_info(id));
}; };
loop { loop {
// Control plane from the UI thread. // Control plane from the UI thread.
loop { loop {
match ctl.try_recv() { match ctl.try_recv() {
Ok(Ctl::Attach(c)) => {
w.attached = Some(c);
w.last_axis = [i32::MIN; 6];
w.set_sensors(true);
}
Ok(Ctl::Detach) => {
w.flush_held();
w.set_sensors(false);
w.attached = None;
}
Ok(Ctl::Pin(key)) => { Ok(Ctl::Pin(key)) => {
let before = w.active_id(); let before = w.active_id();
w.pinned = key; w.pinned = key;
+7 -230
View File
@@ -1,104 +1,14 @@
//! The single Direct3D 11 device shared by the video decoder (D3D11VA hardware decode) and the //! DXGI adapter enumeration for the Settings "GPU" picker.
//! presenter (the `SwapChainPanel` composition swapchain + the present draw).
//! //!
//! Zero-copy hardware decode requires FFmpeg to decode HEVC into `ID3D11Texture2D`s created by the //! Streaming (decode + present) runs in the spawned `punktfunk-session` binary; the shell only
//! **same** device the presenter binds as shader resources and draws with — a texture from one //! needs the list of real (hardware) adapters to offer on a multi-GPU box (a hybrid laptop or an
//! device can't be sampled by another. So the device is created once, here, and both subsystems //! eGPU). The picked adapter description is persisted (`crate::trust::Settings::adapter`) and read
//! pull it from a process-global `OnceLock` (initialised on whichever thread asks first: the //! by the session child at connect (`PUNKTFUNK_ADAPTER` remains the session binary's env override).
//! session pump when it builds the decoder, or the UI thread when it builds the presenter).
//!
//! **Adapter selection** (matters on hybrid boxes — e.g. an Intel iGPU driving the panel next to
//! an NVIDIA dGPU): `PUNKTFUNK_ADAPTER` (index or case-insensitive name substring, a debugging
//! override) wins; else the persisted Settings GPU pick ([`crate::trust::Settings::adapter`], the
//! Settings-page selector on multi-GPU boxes); else the adapter whose output owns the monitor our
//! window is on — that's the adapter DWM composes that monitor with, so presents are copy-free
//! and decode runs on the near GPU; else the default adapter. Deliberately NOT "the adapter with
//! the best decoder": if the monitor's adapter can't decode the codec we demote to software,
//! which beats a per-frame cross-adapter present copy. The device is cached **keyed by the
//! resolved preference**, so a Settings change takes effect at the next session (the pump and the
//! presenter both resolve at session start and read the same value) without an app restart.
//!
//! `PUNKTFUNK_D3D_DEBUG=1` adds the D3D11 debug layer (validation messages in the debugger /
//! DebugView) — invaluable for present-path bugs, which D3D11 otherwise drops silently.
//!
//! **Thread-safety.** windows-rs COM interfaces are deliberately `!Send`/`!Sync` — thread-safety
//! is per-object, not universal. An `ID3D11Device` and its immediate context become free-threaded
//! once `ID3D11Multithread::SetMultithreadProtected(TRUE)` is set, which FFmpeg's D3D11VA backend
//! does inside `av_hwdevice_ctx_init` (it installs an `ID3D11Multithread`-based default lock when we
//! leave `AVD3D11VADeviceContext.lock` null). The decoder then uses FFmpeg's separate
//! `ID3D11VideoContext` for decode while the presenter uses the immediate context for draw; under
//! multithread protection D3D serialises the two internally, and decode/draw touch disjoint context
//! state. That makes the `unsafe impl Send + Sync` below sound for exactly this usage.
use anyhow::{anyhow, Result};
use std::sync::{Arc, Mutex};
use windows::core::Interface; use windows::core::Interface;
use windows::Win32::Graphics::Direct3D::{
D3D_DRIVER_TYPE, D3D_DRIVER_TYPE_HARDWARE, D3D_DRIVER_TYPE_UNKNOWN, D3D_DRIVER_TYPE_WARP,
D3D_FEATURE_LEVEL_11_0, D3D_FEATURE_LEVEL_11_1,
};
use windows::Win32::Graphics::Direct3D11::{
D3D11CreateDevice, ID3D11Device, ID3D11DeviceContext, ID3D11Multithread,
D3D11_CREATE_DEVICE_BGRA_SUPPORT, D3D11_CREATE_DEVICE_DEBUG, D3D11_CREATE_DEVICE_FLAG,
D3D11_CREATE_DEVICE_VIDEO_SUPPORT, D3D11_SDK_VERSION,
};
use windows::Win32::Graphics::Dxgi::{CreateDXGIFactory1, IDXGIAdapter, IDXGIFactory1}; use windows::Win32::Graphics::Dxgi::{CreateDXGIFactory1, IDXGIAdapter, IDXGIFactory1};
pub struct SharedDevice { /// The adapter's human-readable description.
pub device: ID3D11Device,
pub context: ID3D11DeviceContext,
/// True when this is a real GPU (hardware) adapter — a precondition for D3D11VA decode. WARP
/// (the GPU-less dev box) creates fine for present but cannot hardware-decode HEVC, so the
/// decoder skips straight to the software path there.
pub hardware: bool,
}
// Sound for our usage — see the module docs: the device + immediate context are free-threaded under
// the multithread protection FFmpeg installs, and decode (video context) / present (immediate
// context) never share mutable context state.
unsafe impl Send for SharedDevice {}
unsafe impl Sync for SharedDevice {}
/// The shared device, cached with the GPU preference it was resolved from (empty = automatic).
/// Re-created when the preference changes — in practice only between sessions: within one session
/// the decoder and the presenter both call [`shared`] at session start with the same value.
static SHARED: Mutex<Option<(String, Arc<SharedDevice>)>> = Mutex::new(None);
/// The user's decode/present GPU preference: the `PUNKTFUNK_ADAPTER` env (debugging override)
/// wins, else the persisted Settings pick; empty = automatic.
fn adapter_pref() -> String {
std::env::var("PUNKTFUNK_ADAPTER")
.ok()
.map(|s| s.trim().to_string())
.filter(|s| !s.is_empty())
.unwrap_or_else(|| crate::trust::Settings::load().adapter)
}
/// The process-shared D3D11 device for the current GPU preference, created (or re-created after
/// a preference change) on demand. `None` only if D3D11 device creation fails for both a hardware
/// adapter and WARP (effectively never — WARP is always present).
pub fn shared() -> Option<Arc<SharedDevice>> {
let pref = adapter_pref();
let mut cached = SHARED.lock().unwrap();
if let Some((key, dev)) = cached.as_ref() {
if *key == pref {
return Some(dev.clone());
}
}
match create_device(&pref) {
Ok(d) => {
let d = Arc::new(d);
*cached = Some((pref, d.clone()));
Some(d)
}
Err(e) => {
tracing::error!(error = %e, "shared D3D11 device creation failed — no present/decode");
None
}
}
}
/// The adapter's human-readable description, for the logs.
fn adapter_name(adapter: &IDXGIAdapter) -> String { fn adapter_name(adapter: &IDXGIAdapter) -> String {
unsafe { unsafe {
adapter adapter
@@ -112,7 +22,7 @@ fn adapter_name(adapter: &IDXGIAdapter) -> String {
} }
} }
/// Every DXGI adapter, in enumeration order (`PUNKTFUNK_ADAPTER=<index>` uses these indices). /// Every DXGI adapter, in enumeration order.
fn all_adapters() -> Vec<IDXGIAdapter> { fn all_adapters() -> Vec<IDXGIAdapter> {
let factory: IDXGIFactory1 = match unsafe { CreateDXGIFactory1() } { let factory: IDXGIFactory1 = match unsafe { CreateDXGIFactory1() } {
Ok(f) => f, Ok(f) => f,
@@ -144,136 +54,3 @@ pub fn adapter_names() -> Vec<String> {
.map(adapter_name) .map(adapter_name)
.collect() .collect()
} }
/// Resolve an explicit adapter: a non-empty `pref` (index or case-insensitive name substring, from
/// env or Settings) wins; else the adapter whose output owns the monitor the app window is on (see
/// module docs); else `None` → the default adapter (also the headless-CLI path with no window).
fn resolve_adapter(pref: &str) -> Option<IDXGIAdapter> {
let adapters = all_adapters();
if !pref.is_empty() {
let found = if let Ok(idx) = pref.parse::<usize>() {
adapters.get(idx).cloned()
} else {
let needle = pref.to_lowercase();
adapters
.iter()
.find(|a| adapter_name(a).to_lowercase().contains(&needle))
.cloned()
};
match &found {
Some(a) => tracing::info!(pref, adapter = %adapter_name(a), "GPU preference matched"),
None => tracing::warn!(pref, "GPU preference matched no adapter — using automatic"),
}
if found.is_some() {
return found;
}
}
// The adapter driving the monitor our window sits on: DWM composes that monitor with it, so
// presenting from it is copy-free (a hybrid box's other adapter would pay a cross-adapter
// copy per frame).
let monitor = unsafe {
use windows::Win32::Graphics::Gdi::{MonitorFromWindow, MONITOR_DEFAULTTONULL};
use windows::Win32::UI::WindowsAndMessaging::FindWindowW;
let hwnd = FindWindowW(None, windows::core::w!("Punktfunk")).ok()?;
MonitorFromWindow(hwnd, MONITOR_DEFAULTTONULL)
};
if monitor.is_invalid() {
return None;
}
for adapter in &adapters {
let mut oi = 0u32;
while let Ok(output) = unsafe { adapter.EnumOutputs(oi) } {
oi += 1;
if let Ok(desc) = unsafe { output.GetDesc() } {
if desc.Monitor == monitor {
tracing::info!(adapter = %adapter_name(adapter), "using the window's monitor adapter");
return Some(adapter.clone());
}
}
}
}
None
}
fn create_device(pref: &str) -> Result<SharedDevice> {
// Preference order: the resolved adapter (or the default hardware adapter) with video support
// (enables D3D11VA); the same without the VIDEO flag (a driver that rejects it still presents +
// software-decodes); finally WARP for the GPU-less box. BGRA_SUPPORT is required for the
// composition swapchain in every case. An explicit adapter requires D3D_DRIVER_TYPE_UNKNOWN.
let adapter = resolve_adapter(pref);
let attempts: [(Option<&IDXGIAdapter>, D3D_DRIVER_TYPE, bool, bool); 3] = match &adapter {
Some(a) => [
(Some(a), D3D_DRIVER_TYPE_UNKNOWN, true, true),
(Some(a), D3D_DRIVER_TYPE_UNKNOWN, false, true),
(None, D3D_DRIVER_TYPE_WARP, false, false),
],
None => [
(None, D3D_DRIVER_TYPE_HARDWARE, true, true),
(None, D3D_DRIVER_TYPE_HARDWARE, false, true),
(None, D3D_DRIVER_TYPE_WARP, false, false),
],
};
// The debug layer needs the SDK layers installed (Graphics Tools); when they're missing the
// creation fails, so each attempt retries without the flag rather than failing the ladder.
let debug = std::env::var("PUNKTFUNK_D3D_DEBUG").is_ok_and(|v| v == "1");
for (adapter, driver, video, hardware) in attempts {
let mut flags = D3D11_CREATE_DEVICE_BGRA_SUPPORT;
if video {
flags |= D3D11_CREATE_DEVICE_VIDEO_SUPPORT;
}
let flag_sets: &[D3D11_CREATE_DEVICE_FLAG] = if debug {
&[flags | D3D11_CREATE_DEVICE_DEBUG, flags]
} else {
&[flags]
};
for &flags in flag_sets {
let mut device = None;
let mut context = None;
let r = unsafe {
D3D11CreateDevice(
adapter,
driver,
None,
flags,
Some(&[D3D_FEATURE_LEVEL_11_1, D3D_FEATURE_LEVEL_11_0]),
D3D11_SDK_VERSION,
Some(&mut device),
None,
Some(&mut context),
)
};
if r.is_ok() {
let (device, context) = (device.unwrap(), context.unwrap());
// Make the device + immediate context free-threaded: the decoder (D3D11VA video
// context, pump thread) and the presenter (immediate context, render thread) both
// touch this device. FFmpeg also sets this during hwdevice init, but doing it up
// front keeps the cross-thread `Send`/`Sync` sound from the moment the device exists.
if let Ok(mt) = context.cast::<ID3D11Multithread>() {
unsafe {
let _ = mt.SetMultithreadProtected(true); // returns the prior state; ignore
}
}
tracing::info!(
adapter = %adapter.map(adapter_name).unwrap_or_else(|| if hardware {
"default".into()
} else {
"WARP (software)".into()
}),
video,
debug = (flags & D3D11_CREATE_DEVICE_DEBUG).0 != 0,
"shared D3D11 device created"
);
return Ok(SharedDevice {
device,
context,
hardware,
});
}
}
}
Err(anyhow!(
"D3D11CreateDevice failed for both hardware and WARP"
))
}
-742
View File
@@ -1,742 +0,0 @@
//! Stream input: Win32 low-level keyboard + mouse hooks forwarding to the host while the WinUI
//! window is focused and the pointer is captured.
//!
//! windows-reactor exposes no raw key-down/up or pointer-position/wheel events (only keyboard
//! *accelerators* and pointer button-state), which is insufficient for a game stream. So this
//! drops below XAML to `WH_KEYBOARD_LL` / `WH_MOUSE_LL`, installed on the UI thread when the
//! stream page mounts and removed when it unmounts.
//!
//! **Pointer lock.** While captured the cursor is *locked* the way a game-streaming client locks
//! it (Moonlight/Parsec): the OS cursor is hidden + confined to the window (`ClipCursor`), and
//! every physical move is turned into a **relative** delta (`InputKind::MouseMove`) — we read the
//! offset from the window centre, ship it (scaled screen→host through the Contain-fit factor, with
//! sub-pixel remainder carried so slow drags aren't lost), then warp the cursor back to centre so
//! it never reaches a screen edge. This is why the old absolute path froze: swallowing
//! `WM_MOUSEMOVE` pinned the OS cursor, so `pt` never travelled and the absolute coordinate
//! snapped to one point. Keys carry the **US-positional VK** for the pressed physical key (the
//! punktfunk wire contract shared by every first-party client — see [`scan_to_positional_vk`]):
//! the hook's layout-resolved `vkCode` must NOT go on the wire, or a non-US pair re-maps
//! positions through two layouts (German: y↔z swapped, ü lands on ö).
//!
//! **Capture state machine** (parity with the GTK/Swift clients): capture engages at stream
//! start, **Ctrl+Alt+Shift+Q** releases it (handing the cursor back to the local desktop), and a
//! **click on the stream** re-engages it. Losing foreground also releases the lock so the cursor
//! is never stranded; regaining it while still captured re-locks. When "capture system
//! shortcuts" is off in Settings, Alt+Tab / Alt+Esc / Ctrl+Esc / the Win keys act on the local
//! desktop instead of being forwarded. **Ctrl+Alt+Shift+D disconnects** the session (consumed
//! locally, works captured or released while our window is foreground): it trips the session's
//! stop flag, the pump winds down, and the event loop navigates back to the host list.
//! **Ctrl+Alt+Shift+S** toggles the stats overlay live and **F11** toggles fullscreen — both are
//! client-local shortcuts (consumed, never forwarded), matching the GTK client's stream key set.
use punktfunk_core::client::NativeClient;
use punktfunk_core::config::Mode;
use punktfunk_core::input::{InputEvent, InputKind};
use std::collections::HashSet;
use std::sync::atomic::{AtomicBool, AtomicIsize, Ordering};
use std::sync::{Arc, Mutex};
use windows::core::BOOL;
use windows::Win32::Foundation::{HWND, LPARAM, LRESULT, POINT, RECT, WPARAM};
use windows::Win32::Graphics::Gdi::ClientToScreen;
use windows::Win32::System::LibraryLoader::GetModuleHandleW;
use windows::Win32::UI::Input::KeyboardAndMouse::{VK_D, VK_F11, VK_Q, VK_S};
use windows::Win32::UI::Shell::{DefSubclassProc, RemoveWindowSubclass, SetWindowSubclass};
use windows::Win32::UI::WindowsAndMessaging::{
CallNextHookEx, ClipCursor, EnumChildWindows, GetClientRect, GetForegroundWindow, SetCursor,
SetCursorPos, SetWindowsHookExW, ShowCursor, UnhookWindowsHookEx, HC_ACTION, HHOOK,
KBDLLHOOKSTRUCT, LLKHF_EXTENDED, LLMHF_INJECTED, MSLLHOOKSTRUCT, WH_KEYBOARD_LL, WH_MOUSE_LL,
WM_KEYUP, WM_LBUTTONDOWN, WM_LBUTTONUP, WM_MBUTTONDOWN, WM_MBUTTONUP, WM_MOUSEHWHEEL,
WM_MOUSEMOVE, WM_MOUSEWHEEL, WM_RBUTTONDOWN, WM_RBUTTONUP, WM_SETCURSOR, WM_SYSKEYUP,
WM_XBUTTONDOWN, WM_XBUTTONUP,
};
struct State {
connector: Arc<NativeClient>,
mode: Mode,
/// The session's stop flag (Ctrl+Alt+Shift+D trips it; the pump then ends the session).
stop: Arc<AtomicBool>,
/// Our window handle, stored as the raw `isize` so `State` is `Send` (`HWND` is not).
hwnd: isize,
/// User intent: forward input to the host (toggled by Ctrl+Alt+Shift+Q / click-to-capture).
captured: bool,
/// Forward system shortcuts (Alt+Tab, Win, …) to the host; off = they act locally.
inhibit_shortcuts: bool,
/// The OS pointer is currently locked (hidden + confined + recentering). Tracks the real
/// `ClipCursor`/`ShowCursor` state so we engage/disengage exactly once per transition.
locked: bool,
/// Lock geometry, captured when the lock engages: the confinement rect (screen coordinates,
/// also the click-to-capture hit test), its centre (the cursor is warped here after every
/// move), and the screen→host scale (the Contain-fit display scale's inverse). Stable while
/// locked — the window can't be moved or resized with the cursor confined inside it.
clip: RECT,
center_x: i32,
center_y: i32,
scale: f32,
/// Sub-pixel remainder of the screen→host scale, carried so slow drags aren't truncated away.
acc_x: f32,
acc_y: f32,
/// Modifier state, tracked from the hook's own event stream (see `kbd_proc`).
ctrl: bool,
alt: bool,
shift: bool,
held_keys: HashSet<u8>,
held_buttons: HashSet<u32>,
}
// `State` carries no `!Send` handle (hwnd is an `isize`), so the static is sound. The hook procs
// run on the same UI thread that installs/removes the hooks, so the lock is uncontended.
static STATE: Mutex<Option<State>> = Mutex::new(None);
static KBD_HOOK: AtomicIsize = AtomicIsize::new(0);
static MOUSE_HOOK: AtomicIsize = AtomicIsize::new(0);
/// Mirror of `State::captured` for lock-free reads off the UI thread (the HUD poll).
static CAPTURED: AtomicBool = AtomicBool::new(false);
/// Live stats-overlay visibility. Seeded from `Settings::show_stats` at `install`, then toggled by
/// Ctrl+Alt+Shift+S for the session (parity with the GTK client's live `s` toggle); the HUD poll
/// reads it lock-free to drive the overlay.
static HUD_VISIBLE: AtomicBool = AtomicBool::new(false);
/// Whether the pointer lock currently wants the OS cursor hidden. Read lock-free by
/// [`cursor_subclass_proc`] (which runs on the UI thread inside `WM_SETCURSOR`) so it can override
/// WinUI's per-move arrow re-assertion — a one-shot `ShowCursor(false)` alone loses that race
/// because the content island re-sets the arrow every time the pointer moves.
static CURSOR_HIDDEN: AtomicBool = AtomicBool::new(false);
/// Our `SetWindowSubclass` id on the WinUI window + its content-island children (any stable value;
/// scopes the subclass so install/remove target exactly our proc).
const CURSOR_SUBCLASS_ID: usize = 0x7066_6375; // 'pfcu'
/// Whether stream input is currently captured (drives the HUD's release/capture hint).
pub fn is_captured() -> bool {
CAPTURED.load(Ordering::Relaxed)
}
/// Whether the stats overlay should be shown: the Settings default at stream start, then whatever
/// Ctrl+Alt+Shift+S last set for the session. Read by the HUD poll thread.
pub fn hud_visible() -> bool {
HUD_VISIBLE.load(Ordering::Relaxed)
}
/// Set the capture intent and engage/release the pointer lock to match.
fn set_captured(st: &mut State, on: bool) {
st.captured = on;
CAPTURED.store(on, Ordering::Relaxed);
set_locked(st, on);
if !on {
flush_held(st); // release held keys/buttons so nothing sticks on the host
}
}
/// Install the hooks for a streaming session. Call from the UI thread once the window is shown.
/// `inhibit_shortcuts` forwards system shortcuts (Alt+Tab, Win, …) to the host; off = local.
/// `show_stats` seeds the stats-overlay visibility that Ctrl+Alt+Shift+S then toggles live.
/// `stop` is the session's stop flag, tripped by the disconnect shortcut.
pub fn install(
connector: Arc<NativeClient>,
mode: Mode,
inhibit_shortcuts: bool,
show_stats: bool,
stop: Arc<AtomicBool>,
) {
HUD_VISIBLE.store(show_stats, Ordering::Relaxed);
let hwnd = unsafe { GetForegroundWindow() };
let mut st = State {
connector,
mode,
stop,
hwnd: hwnd.0 as isize,
captured: false,
inhibit_shortcuts,
locked: false,
clip: RECT::default(),
center_x: 0,
center_y: 0,
scale: 1.0,
acc_x: 0.0,
acc_y: 0.0,
ctrl: false,
alt: false,
shift: false,
held_keys: HashSet::new(),
held_buttons: HashSet::new(),
};
// Capture immediately (the window is foreground at mount, like Moonlight grabbing on stream
// start).
set_captured(&mut st, true);
*STATE.lock().unwrap() = Some(st);
unsafe {
let hinst = GetModuleHandleW(None).ok();
if let Ok(h) = SetWindowsHookExW(WH_KEYBOARD_LL, Some(kbd_proc), hinst.map(Into::into), 0) {
KBD_HOOK.store(h.0 as isize, Ordering::SeqCst);
}
if let Ok(h) = SetWindowsHookExW(WH_MOUSE_LL, Some(mouse_proc), hinst.map(Into::into), 0) {
MOUSE_HOOK.store(h.0 as isize, Ordering::SeqCst);
}
}
tracing::info!(
inhibit_shortcuts,
"stream input hooks installed — pointer locked (Ctrl+Alt+Shift+Q toggles capture)"
);
}
/// Remove the hooks, release the pointer lock, and flush any held keys/buttons (so nothing
/// sticks down on the host).
pub fn uninstall() {
unsafe {
let k = KBD_HOOK.swap(0, Ordering::SeqCst);
if k != 0 {
let _ = UnhookWindowsHookEx(HHOOK(k as *mut _));
}
let m = MOUSE_HOOK.swap(0, Ordering::SeqCst);
if m != 0 {
let _ = UnhookWindowsHookEx(HHOOK(m as *mut _));
}
}
if let Some(mut st) = STATE.lock().unwrap().take() {
// Hand the cursor back + flush held state.
set_captured(&mut st, false);
// Drop the WM_SETCURSOR subclass so the long-lived app window (reused for the host list
// once the stream ends) is left pristine — set_captured already cleared CURSOR_HIDDEN.
remove_cursor_subclass(HWND(st.hwnd as *mut _));
// Fullscreen is a streaming-only mode: if F11 put us there, drop back to a normal window
// so the GUI (the host list) is never left borderless-fullscreen after the stream ends.
exit_fullscreen(HWND(st.hwnd as *mut _));
}
}
/// Release every held key/button on the host, so nothing sticks down when capture is dropped
/// (toggled off) or the session ends.
fn flush_held(st: &mut State) {
let c = st.connector.clone();
for vk in st.held_keys.drain() {
send(&c, InputKind::KeyUp, vk as u32, 0, 0, 0);
}
for b in st.held_buttons.drain() {
send(&c, InputKind::MouseButtonUp, b, 0, 0, 0);
}
}
/// Subclass proc on the WinUI window + its content-island children: while the pointer lock wants
/// the cursor hidden ([`CURSOR_HIDDEN`]), answer `WM_SETCURSOR` ourselves with `SetCursor(None)`
/// and return TRUE — halting WinUI's default handling before it re-asserts the arrow. This is what
/// actually keeps the cursor hidden while captured; the sibling `ShowCursor(false)` cannot, because
/// WinUI re-sets the arrow on every pointer move (the content island answers `WM_SETCURSOR` itself,
/// which a low-level mouse hook never sees). When not hidden, we defer to the chain untouched.
unsafe extern "system" fn cursor_subclass_proc(
hwnd: HWND,
msg: u32,
wparam: WPARAM,
lparam: LPARAM,
_id: usize,
_ref: usize,
) -> LRESULT {
if msg == WM_SETCURSOR && CURSOR_HIDDEN.load(Ordering::Relaxed) {
unsafe {
let _ = SetCursor(None);
}
return LRESULT(1); // handled — suppress the framework's arrow re-assertion
}
unsafe { DefSubclassProc(hwnd, msg, wparam, lparam) }
}
unsafe extern "system" fn subclass_install_cb(child: HWND, _l: LPARAM) -> BOOL {
unsafe {
let _ = SetWindowSubclass(child, Some(cursor_subclass_proc), CURSOR_SUBCLASS_ID, 0);
}
BOOL(1) // keep enumerating
}
unsafe extern "system" fn subclass_remove_cb(child: HWND, _l: LPARAM) -> BOOL {
unsafe {
let _ = RemoveWindowSubclass(child, Some(cursor_subclass_proc), CURSOR_SUBCLASS_ID);
}
BOOL(1)
}
/// Install [`cursor_subclass_proc`] on the top-level WinUI window and every descendant — the video
/// is a composition SwapChainPanel, so the pointer actually sits over WinUI's internal content-
/// island child window, which is the window that receives `WM_SETCURSOR`. `EnumChildWindows`
/// recurses into all descendants, so one pass covers it. Idempotent (re-installing the same
/// id+proc just refreshes it), so it's safe to call on every lock engage.
fn install_cursor_subclass(top: HWND) {
unsafe {
let _ = SetWindowSubclass(top, Some(cursor_subclass_proc), CURSOR_SUBCLASS_ID, 0);
let _ = EnumChildWindows(Some(top), Some(subclass_install_cb), LPARAM(0));
}
}
/// Remove our subclass from the top-level window and every descendant. Called on teardown so the
/// long-lived app window (reused for the host list after the stream ends) is left pristine.
fn remove_cursor_subclass(top: HWND) {
unsafe {
let _ = RemoveWindowSubclass(top, Some(cursor_subclass_proc), CURSOR_SUBCLASS_ID);
let _ = EnumChildWindows(Some(top), Some(subclass_remove_cb), LPARAM(0));
}
}
/// Engage or release the pointer lock: confine + hide + recentre on, free + show on off.
/// Guarded so the `ClipCursor`/`ShowCursor` calls stay balanced (one each per transition).
/// Engaging captures the lock geometry (rect, centre, screen→host scale) — see `State::clip`.
fn set_locked(st: &mut State, on: bool) {
if on == st.locked {
return;
}
let hwnd = HWND(st.hwnd as *mut _);
unsafe {
if on {
let mut rc = RECT::default();
if GetClientRect(hwnd, &mut rc).is_ok() {
let mut tl = POINT {
x: rc.left,
y: rc.top,
};
let mut br = POINT {
x: rc.right,
y: rc.bottom,
};
let _ = ClientToScreen(hwnd, &mut tl);
let _ = ClientToScreen(hwnd, &mut br);
st.clip = RECT {
left: tl.x,
top: tl.y,
right: br.x,
bottom: br.y,
};
let _ = ClipCursor(Some(&st.clip as *const RECT));
st.center_x = (tl.x + br.x) / 2;
st.center_y = (tl.y + br.y) / 2;
// Screen px → host px: the Contain-fit display scale's inverse, so the host
// cursor tracks the physical mouse 1:1 on screen at any window size.
let (ww, wh) = ((br.x - tl.x).max(1) as f32, (br.y - tl.y).max(1) as f32);
let (vw, vh) = (st.mode.width.max(1) as f32, st.mode.height.max(1) as f32);
st.scale = (ww / vw).min(wh / vh).max(0.01);
let _ = SetCursorPos(st.center_x, st.center_y);
}
// Hide the OS cursor. ShowCursor(false) is the coarse gate; the subclass is what
// actually holds it hidden against WinUI's per-move arrow re-assertion — see
// cursor_subclass_proc / install_cursor_subclass.
let _ = ShowCursor(false);
CURSOR_HIDDEN.store(true, Ordering::Relaxed);
install_cursor_subclass(hwnd);
st.acc_x = 0.0;
st.acc_y = 0.0;
} else {
CURSOR_HIDDEN.store(false, Ordering::Relaxed);
let _ = ClipCursor(None);
let _ = ShowCursor(true);
}
}
st.locked = on;
}
/// The pre-fullscreen window placement, saved on entering fullscreen and restored on leaving it.
/// Module-level (not a `toggle_fullscreen`-local static) so the F11 toggle and the stream-stop exit
/// ([`uninstall`]) share the one saved placement, and its presence is also the "are we fullscreen?"
/// flag for [`exit_fullscreen`]. Only ever touched on the UI thread (the hook proc / the stream
/// page's unmount), but a Mutex keeps the static sound + `Sync`.
static SAVED_PLACEMENT: Mutex<Option<windows::Win32::UI::WindowsAndMessaging::WINDOWPLACEMENT>> =
Mutex::new(None);
/// Whether our top-level window is currently borderless-fullscreen. Entering strips
/// `WS_OVERLAPPEDWINDOW`, so its absence is the flag — no extra state beyond [`SAVED_PLACEMENT`].
fn is_fullscreen(hwnd: HWND) -> bool {
use windows::Win32::UI::WindowsAndMessaging::{
GetWindowLongPtrW, GWL_STYLE, WS_OVERLAPPEDWINDOW,
};
let overlapped = WS_OVERLAPPEDWINDOW.0 as isize;
unsafe { GetWindowLongPtrW(hwnd, GWL_STYLE) & overlapped == 0 }
}
/// Enter borderless fullscreen: remember the window placement, drop the frame
/// (`WS_OVERLAPPEDWINDOW`), and size the window to cover the whole monitor. windows-reactor owns
/// the WinUI window but exposes no fullscreen API, so we drive the HWND directly (parity with the
/// GTK client's F11). The SwapChainPanel follows the resulting `WM_SIZE` like any window resize.
fn enter_fullscreen(hwnd: HWND) {
use windows::Win32::Graphics::Gdi::{
GetMonitorInfoW, MonitorFromWindow, MONITORINFO, MONITOR_DEFAULTTOPRIMARY,
};
use windows::Win32::UI::WindowsAndMessaging::{
GetWindowLongPtrW, GetWindowPlacement, SetWindowLongPtrW, SetWindowPos, GWL_STYLE,
SWP_FRAMECHANGED, SWP_NOOWNERZORDER, SWP_NOZORDER, WINDOWPLACEMENT, WS_OVERLAPPEDWINDOW,
};
let overlapped = WS_OVERLAPPEDWINDOW.0 as isize;
unsafe {
let style = GetWindowLongPtrW(hwnd, GWL_STYLE);
let mut wp = WINDOWPLACEMENT {
length: std::mem::size_of::<WINDOWPLACEMENT>() as u32,
..Default::default()
};
let mut mi = MONITORINFO {
cbSize: std::mem::size_of::<MONITORINFO>() as u32,
..Default::default()
};
let mon = MonitorFromWindow(hwnd, MONITOR_DEFAULTTOPRIMARY);
if GetWindowPlacement(hwnd, &mut wp).is_ok() && GetMonitorInfoW(mon, &mut mi).as_bool() {
*SAVED_PLACEMENT.lock().unwrap() = Some(wp);
SetWindowLongPtrW(hwnd, GWL_STYLE, style & !overlapped);
let r = mi.rcMonitor;
let _ = SetWindowPos(
hwnd,
None,
r.left,
r.top,
r.right - r.left,
r.bottom - r.top,
SWP_NOZORDER | SWP_NOOWNERZORDER | SWP_FRAMECHANGED,
);
}
}
}
/// Leave borderless fullscreen: restore the frame style and the saved placement. A no-op when we
/// aren't fullscreen (nothing saved), so it's safe to call unconditionally on stream stop.
fn exit_fullscreen(hwnd: HWND) {
use windows::Win32::UI::WindowsAndMessaging::{
GetWindowLongPtrW, SetWindowLongPtrW, SetWindowPlacement, SetWindowPos, GWL_STYLE,
SWP_FRAMECHANGED, SWP_NOMOVE, SWP_NOOWNERZORDER, SWP_NOSIZE, SWP_NOZORDER,
WS_OVERLAPPEDWINDOW,
};
let Some(wp) = SAVED_PLACEMENT.lock().unwrap().take() else {
return; // never went fullscreen — nothing to restore
};
let overlapped = WS_OVERLAPPEDWINDOW.0 as isize;
unsafe {
let style = GetWindowLongPtrW(hwnd, GWL_STYLE);
SetWindowLongPtrW(hwnd, GWL_STYLE, style | overlapped);
let _ = SetWindowPlacement(hwnd, &wp);
let _ = SetWindowPos(
hwnd,
None,
0,
0,
0,
0,
SWP_NOMOVE | SWP_NOSIZE | SWP_NOZORDER | SWP_NOOWNERZORDER | SWP_FRAMECHANGED,
);
}
}
/// Toggle borderless fullscreen for our top-level window (F11), the classic Win32 dance split into
/// [`enter_fullscreen`] / [`exit_fullscreen`] so the stream-stop path can force windowed too.
fn toggle_fullscreen(hwnd: isize) {
let hwnd = HWND(hwnd as *mut _);
if is_fullscreen(hwnd) {
exit_fullscreen(hwnd);
} else {
enter_fullscreen(hwnd);
}
}
fn send(c: &NativeClient, kind: InputKind, code: u32, x: i32, y: i32, flags: u32) {
let _ = c.send_input(&InputEvent {
kind,
_pad: [0; 3],
code,
x,
y,
flags,
});
}
/// System shortcuts that act on the LOCAL desktop when "capture system shortcuts" is off:
/// the Win keys, Alt+Tab, and Alt/Ctrl+Esc.
fn is_system_shortcut(st: &State, vk: u16) -> bool {
match vk {
0x5B | 0x5C => true, // L/R Win
0x09 => st.alt, // Alt+Tab
0x1B => st.alt || st.ctrl, // Alt+Esc / Ctrl+Esc
_ => false,
}
}
unsafe extern "system" fn kbd_proc(code: i32, wparam: WPARAM, lparam: LPARAM) -> LRESULT {
if code == HC_ACTION as i32 {
let kb = unsafe { &*(lparam.0 as *const KBDLLHOOKSTRUCT) };
let msg = wparam.0 as u32;
let up = msg == WM_KEYUP || msg == WM_SYSKEYUP;
let vk = kb.vkCode as u16;
let mut guard = STATE.lock().unwrap();
if let Some(st) = guard.as_mut() {
// Track modifier state from the hook's own event stream — reliable even while we
// swallow these keys (GetAsyncKeyState doesn't reflect keys suppressed by our own LL
// hook, which is why the shortcut never fired). Handles the generic + L/R vk codes.
match kb.vkCode {
0x11 | 0xA2 | 0xA3 => st.ctrl = !up, // (L/R)CONTROL
0x12 | 0xA4 | 0xA5 => st.alt = !up, // (L/R)MENU (Alt)
0x10 | 0xA0 | 0xA1 => st.shift = !up, // (L/R)SHIFT
_ => {}
}
let foreground = unsafe { GetForegroundWindow() }.0 as isize == st.hwnd;
if foreground {
// Capture toggle: Ctrl+Alt+Shift+Q (consumed locally, never forwarded).
if !up && vk == VK_Q.0 && st.ctrl && st.alt && st.shift {
let on = !st.captured;
set_captured(st, on);
tracing::info!(captured = on, "capture toggled (Ctrl+Alt+Shift+Q)");
return LRESULT(1);
}
// Disconnect: Ctrl+Alt+Shift+D (consumed locally). Release capture immediately so
// the cursor is free while the session winds down and the UI navigates home.
if !up && vk == VK_D.0 && st.ctrl && st.alt && st.shift {
set_captured(st, false);
// Deliberate user exit → close with QUIT_CLOSE_CODE so the host tears the session
// down immediately instead of holding the keep-alive linger for a reconnect.
st.connector.disconnect_quit();
st.stop.store(true, Ordering::SeqCst);
tracing::info!("disconnect requested (Ctrl+Alt+Shift+D)");
return LRESULT(1);
}
// Toggle the stats overlay: Ctrl+Alt+Shift+S (consumed locally). Seeded from
// Settings at install; this live toggle overrides it for the session — parity
// with the GTK client, where `s` flips the OSD without leaving the stream.
if !up && vk == VK_S.0 && st.ctrl && st.alt && st.shift {
let on = !HUD_VISIBLE.load(Ordering::Relaxed);
HUD_VISIBLE.store(on, Ordering::Relaxed);
tracing::info!(hud = on, "stats overlay toggled (Ctrl+Alt+Shift+S)");
return LRESULT(1);
}
// Toggle fullscreen: F11 (consumed locally, no modifiers — a client shortcut,
// never a wire key). Works captured or released. The window resize changes the
// client rect, so re-lock to recompute the pointer confinement + recentre.
if !up && vk == VK_F11.0 {
toggle_fullscreen(st.hwnd);
if st.locked {
set_locked(st, false);
set_locked(st, true);
}
tracing::info!("fullscreen toggled (F11)");
return LRESULT(1);
}
if st.captured {
// With shortcut capture off, hand Alt+Tab & co. to the local desktop —
// neither forwarded nor swallowed.
if !st.inhibit_shortcuts && is_system_shortcut(st, vk) {
return unsafe { CallNextHookEx(None, code, wparam, lparam) };
}
// Wire key: the US-positional VK for this physical key (module docs), derived
// from the scancode. `vkCode` is layout-semantic and only passes through for
// keys the table doesn't cover — extended keys and everything outside the
// typing area, where positional == semantic (plus injected events with
// scanCode 0 from remapping tools, best-effort).
let ext = (kb.flags.0 & LLKHF_EXTENDED.0) != 0;
let v = if ext {
vk as u8
} else {
scan_to_positional_vk(kb.scanCode as u16).unwrap_or(vk as u8)
};
if up {
if st.held_keys.remove(&v) {
send(&st.connector, InputKind::KeyUp, v as u32, 0, 0, 0);
}
} else {
st.held_keys.insert(v);
send(&st.connector, InputKind::KeyDown, v as u32, 0, 0, 0);
}
return LRESULT(1); // swallow so it reaches the host, not the local OS
}
}
}
}
unsafe { CallNextHookEx(None, code, wparam, lparam) }
}
/// Whether a screen point lies inside the window's CURRENT client area (the click-to-capture
/// hit test — computed fresh per click, since the window can move/resize while released).
fn in_client_area(hwnd: isize, pt: POINT) -> bool {
let hwnd = HWND(hwnd as *mut _);
let mut rc = RECT::default();
if unsafe { GetClientRect(hwnd, &mut rc) }.is_err() {
return false;
}
let mut tl = POINT {
x: rc.left,
y: rc.top,
};
let mut br = POINT {
x: rc.right,
y: rc.bottom,
};
unsafe {
let _ = ClientToScreen(hwnd, &mut tl);
let _ = ClientToScreen(hwnd, &mut br);
}
pt.x >= tl.x && pt.x < br.x && pt.y >= tl.y && pt.y < br.y
}
unsafe extern "system" fn mouse_proc(code: i32, wparam: WPARAM, lparam: LPARAM) -> LRESULT {
if code == HC_ACTION as i32 {
let ms = unsafe { &*(lparam.0 as *const MSLLHOOKSTRUCT) };
let msg = wparam.0 as u32;
let injected = (ms.flags & LLMHF_INJECTED) != 0;
let mut guard = STATE.lock().unwrap();
if let Some(st) = guard.as_mut() {
let foreground = unsafe { GetForegroundWindow() }.0 as isize == st.hwnd;
let want_lock = st.captured && foreground;
if want_lock != st.locked {
set_locked(st, want_lock); // sync to focus changes (e.g. lost foreground)
}
// Click-to-capture: after a Ctrl+Alt+Shift+Q release, a primary click on the stream
// re-engages capture. The click is consumed — it starts the grab, it isn't gameplay.
if !st.captured
&& foreground
&& msg == WM_LBUTTONDOWN
&& !injected
&& in_client_area(st.hwnd, ms.pt)
{
set_captured(st, true);
tracing::info!("capture re-engaged (click on stream)");
return LRESULT(1);
}
if st.locked {
// Skip the synthetic move our own SetCursorPos recentre generates.
if injected {
return unsafe { CallNextHookEx(None, code, wparam, lparam) };
}
let c = st.connector.clone();
match msg {
WM_MOUSEMOVE => {
let dx = (ms.pt.x - st.center_x) as f32;
let dy = (ms.pt.y - st.center_y) as f32;
if dx != 0.0 || dy != 0.0 {
st.acc_x += dx / st.scale;
st.acc_y += dy / st.scale;
let (hx, hy) = (st.acc_x.trunc() as i32, st.acc_y.trunc() as i32);
st.acc_x -= hx as f32;
st.acc_y -= hy as f32;
if hx != 0 || hy != 0 {
send(&c, InputKind::MouseMove, 0, hx, hy, 0);
}
}
let _ = unsafe { SetCursorPos(st.center_x, st.center_y) };
}
WM_LBUTTONDOWN => button(st, 1, true),
WM_LBUTTONUP => button(st, 1, false),
WM_RBUTTONDOWN => button(st, 3, true),
WM_RBUTTONUP => button(st, 3, false),
WM_MBUTTONDOWN => button(st, 2, true),
WM_MBUTTONUP => button(st, 2, false),
WM_XBUTTONDOWN => button(st, 3 + ((ms.mouseData >> 16) as u16 as u32), true),
WM_XBUTTONUP => button(st, 3 + ((ms.mouseData >> 16) as u16 as u32), false),
WM_MOUSEWHEEL => send(
&c,
InputKind::MouseScroll,
0,
(ms.mouseData >> 16) as i16 as i32,
0,
0,
),
WM_MOUSEHWHEEL => send(
&c,
InputKind::MouseScroll,
1,
(ms.mouseData >> 16) as i16 as i32,
0,
0,
),
_ => {}
}
return LRESULT(1); // swallow inside the locked window
}
}
}
unsafe { CallNextHookEx(None, code, wparam, lparam) }
}
fn button(st: &mut State, id: u32, down: bool) {
let c = st.connector.clone();
if down {
st.held_buttons.insert(id);
send(&c, InputKind::MouseButtonDown, id, 0, 0, 0);
} else if st.held_buttons.remove(&id) {
send(&c, InputKind::MouseButtonUp, id, 0, 0, 0);
}
}
/// Set-1 make scancode → US-positional VK for the layout-**variant** typing area (letters, digit
/// row, OEM punctuation, the ISO 102nd key) — the exact inverse of the host injector's positional
/// table and the Windows analogue of the Linux client's `evdev_to_vk`. Keys not listed (F-row,
/// nav cluster, numpad, modifiers — plus every E0-extended key, which the caller filters out)
/// have layout-invariant VKs, so the hook's `vkCode` is already correct for them.
fn scan_to_positional_vk(scan: u16) -> Option<u8> {
Some(match scan {
0x02..=0x0A => (scan - 0x02) as u8 + 0x31, // 1..9
0x0B => 0x30, // 0
0x0C => 0xBD, // -_ VK_OEM_MINUS (DE: ß)
0x0D => 0xBB, // =+ VK_OEM_PLUS
0x10 => 0x51, // Q
0x11 => 0x57, // W
0x12 => 0x45, // E
0x13 => 0x52, // R
0x14 => 0x54, // T
0x15 => 0x59, // Y position (QWERTZ: the Z key)
0x16 => 0x55, // U
0x17 => 0x49, // I
0x18 => 0x4F, // O
0x19 => 0x50, // P
0x1A => 0xDB, // [{ VK_OEM_4 (DE: ü)
0x1B => 0xDD, // ]} VK_OEM_6
0x1E => 0x41, // A
0x1F => 0x53, // S
0x20 => 0x44, // D
0x21 => 0x46, // F
0x22 => 0x47, // G
0x23 => 0x48, // H
0x24 => 0x4A, // J
0x25 => 0x4B, // K
0x26 => 0x4C, // L
0x27 => 0xBA, // ;: VK_OEM_1 (DE: ö)
0x28 => 0xDE, // '" VK_OEM_7 (DE: ä)
0x29 => 0xC0, // `~ VK_OEM_3 (DE: ^)
0x2B => 0xDC, // \| VK_OEM_5
0x2C => 0x5A, // Z position (QWERTZ: the Y key)
0x2D => 0x58, // X
0x2E => 0x43, // C
0x2F => 0x56, // V
0x30 => 0x42, // B
0x31 => 0x4E, // N
0x32 => 0x4D, // M
0x33 => 0xBC, // ,< VK_OEM_COMMA
0x34 => 0xBE, // .> VK_OEM_PERIOD
0x35 => 0xBF, // /? VK_OEM_2
0x56 => 0xE2, // <>| VK_OEM_102 (ISO)
_ => return None,
})
}
#[cfg(test)]
mod tests {
use super::*;
/// The German-scramble regression pins: the physical keys a QWERTZ board labels Z/Y/ö/ü must
/// leave this client as their US-position VKs, regardless of the local layout's vkCode.
#[test]
fn positional_pins_for_the_qwertz_scramble() {
assert_eq!(scan_to_positional_vk(0x15), Some(0x59)); // QWERTZ Z key → VK_Y (US position)
assert_eq!(scan_to_positional_vk(0x2C), Some(0x5A)); // QWERTZ Y key → VK_Z (US position)
assert_eq!(scan_to_positional_vk(0x27), Some(0xBA)); // ö key → VK_OEM_1 (US ;: position)
assert_eq!(scan_to_positional_vk(0x1A), Some(0xDB)); // ü key → VK_OEM_4 (US [{ position)
assert_eq!(scan_to_positional_vk(0x28), Some(0xDE)); // ä key → VK_OEM_7 (US '" position)
assert_eq!(scan_to_positional_vk(0x0C), Some(0xBD)); // ß key → VK_OEM_MINUS (US -_ position)
}
/// Keys outside the layout-variant typing area stay un-mapped (vkCode passes through).
#[test]
fn invariant_keys_fall_through() {
for scan in [
0x01u16, 0x0E, 0x0F, 0x1C, 0x1D, 0x2A, 0x36, 0x38, 0x39, 0x3B, 0x45, 0x57,
] {
assert_eq!(scan_to_positional_vk(scan), None, "scan 0x{scan:02X}");
}
}
/// Exactly the 48 typing-area keys are covered (10 digits + 26 letters + 12 OEM), and every
/// mapping is unique — two physical keys must never collapse onto one wire VK.
#[test]
fn table_covers_the_typing_area_bijectively() {
let mapped: Vec<(u16, u8)> = (0u16..=0xFF)
.filter_map(|sc| scan_to_positional_vk(sc).map(|vk| (sc, vk)))
.collect();
assert_eq!(mapped.len(), 48);
let mut vks: Vec<u8> = mapped.iter().map(|&(_, vk)| vk).collect();
vks.sort_unstable();
vks.dedup();
assert_eq!(vks.len(), 48, "duplicate wire VK in the positional table");
}
}
+15 -162
View File
@@ -1,17 +1,16 @@
//! `punktfunk-client` — the native Windows punktfunk/1 client. //! `punktfunk-client` — the native Windows punktfunk/1 client.
//! //!
//! Pure Rust: `NativeClient` linked as a crate (no C ABI, like the GTK Linux client) · FFmpeg //! Pure Rust: `NativeClient` linked as a crate (no C ABI, like the GTK Linux client) · SDL3
//! decode · WASAPI audio · SDL3 gamepads · a **WinUI 3** shell (windows-reactor) with the video //! gamepads · a **WinUI 3** shell (windows-reactor). Streaming (decode + present + audio) runs in
//! on a `SwapChainPanel` bound to a D3D11 composition swapchain. The trust surface mirrors the //! the spawned `punktfunk-session` Vulkan binary; the shell owns host selection, trust and
//! pairing. The trust surface mirrors the
//! other native clients: persistent identity, trust-on-first-use, SPAKE2 PIN pairing — all in-app //! other native clients: persistent identity, trust-on-first-use, SPAKE2 PIN pairing — all in-app
//! (host list, settings, pairing). `--headless` keeps a CLI connect path for tests/measurement. //! (host list, settings, pairing). Streaming runs in the spawned `punktfunk-session` binary;
//! `--headless --speed-test` keeps a decode-less CLI measurement path.
//! //!
//! Usage: //! Usage:
//! punktfunk-client (open the WinUI 3 window: host list, settings, pairing) //! punktfunk-client (open the WinUI 3 window: host list, settings, pairing)
//! punktfunk-client --discover (list punktfunk hosts on the LAN) //! punktfunk-client --discover (list punktfunk hosts on the LAN)
//! punktfunk-client --headless --connect host[:port] [--pin HEX] [--pair PIN] [--mode WxHxHz]
//! [--bitrate MBPS] [--mic] [--decoder auto|hardware|software] [--no-hdr]
//! (no window; count frames + print stats)
//! punktfunk-client --headless --speed-test --connect host[:port] //! punktfunk-client --headless --speed-test --connect host[:port]
//! (measure the path: probe burst → goodput / loss / recommended bitrate) //! (measure the path: probe burst → goodput / loss / recommended bitrate)
@@ -23,29 +22,19 @@
#[cfg(windows)] #[cfg(windows)]
mod app; mod app;
#[cfg(windows)] #[cfg(windows)]
mod audio;
#[cfg(windows)]
mod discovery; mod discovery;
#[cfg(windows)] #[cfg(windows)]
mod gamepad; mod gamepad;
#[cfg(windows)] #[cfg(windows)]
mod gpu; mod gpu;
#[cfg(windows)] #[cfg(windows)]
mod input; mod probe;
#[cfg(windows)]
mod present;
#[cfg(windows)]
mod render;
#[cfg(windows)]
mod session;
#[cfg(windows)] #[cfg(windows)]
mod shell_window; mod shell_window;
#[cfg(windows)] #[cfg(windows)]
mod spawn; mod spawn;
#[cfg(windows)] #[cfg(windows)]
mod trust; mod trust;
#[cfg(windows)]
mod video;
#[cfg(windows)] #[cfg(windows)]
mod wol; mod wol;
@@ -124,13 +113,11 @@ fn set_app_user_model_id() {
} }
} }
/// `--headless --connect host[:port]`: connect from the CLI, count frames, print stats — the /// `--headless --speed-test --connect host[:port]`: measure the path over the real data plane and
/// Windows analogue of `punktfunk-probe`. /// print the outcome — the Windows analogue of `punktfunk-probe`. The former in-process
/// frame-count connect path went with the legacy builtin stream; real streaming is windowed-only.
#[cfg(windows)] #[cfg(windows)]
fn run_headless_cli(args: &[String], identity: (String, String)) { fn run_headless_cli(args: &[String], identity: (String, String)) {
use punktfunk_core::config::{CompositorPref, GamepadPref, Mode};
use std::time::{Duration, Instant};
let arg = |name: &str| -> Option<String> { let arg = |name: &str| -> Option<String> {
args.iter() args.iter()
.position(|a| a == name) .position(|a| a == name)
@@ -154,7 +141,7 @@ fn run_headless_cli(args: &[String], identity: (String, String)) {
let fp = trust::KnownHosts::load() let fp = trust::KnownHosts::load()
.find_by_addr(&host, port) .find_by_addr(&host, port)
.map(|k| k.fp_hex.clone()); .map(|k| k.fp_hex.clone());
match session::run_speed_probe(&host, port, fp.as_deref(), identity) { match probe::run_speed_probe(&host, port, fp.as_deref(), identity) {
Ok(r) => { Ok(r) => {
let mbps = f64::from(r.throughput_kbps) / 1000.0; let mbps = f64::from(r.throughput_kbps) / 1000.0;
let recommended = f64::from(r.throughput_kbps / 10 * 7) / 1000.0; let recommended = f64::from(r.throughput_kbps / 10 * 7) / 1000.0;
@@ -171,144 +158,10 @@ fn run_headless_cli(args: &[String], identity: (String, String)) {
} }
return; return;
} }
let mode = arg("--mode") // Only --speed-test remains headless: real streaming runs in the windowed app's spawned
.and_then(|m| { // punktfunk-session binary, which the deleted in-process frame-count path was replaced by.
let mut it = m.split(['x', 'X']); eprintln!("--headless supports only --speed-test now \u{2014} run the windowed app to stream");
Some(Mode { std::process::exit(2);
width: it.next()?.parse().ok()?,
height: it.next()?.parse().ok()?,
refresh_hz: it.next()?.parse().ok()?,
})
})
.unwrap_or(Mode {
width: 1280,
height: 720,
refresh_hz: 60,
});
let bitrate_kbps = arg("--bitrate")
.and_then(|b| b.parse::<u32>().ok())
.map(|m| m * 1000)
.unwrap_or(0);
let known = trust::KnownHosts::load();
let mut pin = arg("--pin")
.and_then(|h| trust::parse_hex32(&h))
.or_else(|| {
known
.find_by_addr(&host, port)
.and_then(|k| trust::parse_hex32(&k.fp_hex))
});
if let Some(code) = arg("--pair") {
let name = std::env::var("COMPUTERNAME").unwrap_or_else(|_| "windows-client".into());
match punktfunk_core::client::NativeClient::pair(
&host,
port,
(&identity.0, &identity.1),
code.trim(),
&name,
Duration::from_secs(90),
) {
Ok(fp) => {
let mut k = trust::KnownHosts::load();
k.upsert(trust::KnownHost {
name: host.clone(),
addr: host.clone(),
port,
fp_hex: trust::hex(&fp),
paired: true,
last_used: None,
mac: Vec::new(),
});
let _ = k.save();
tracing::info!(fp = %trust::hex(&fp), "paired");
pin = Some(fp);
}
Err(e) => {
eprintln!("Pairing failed: {e:?}");
std::process::exit(1);
}
}
}
let decoder = arg("--decoder")
.map(|d| crate::video::DecoderPref::from_name(&d))
.unwrap_or_default();
tracing::info!(%host, port, ?mode, tofu = pin.is_none(), ?decoder, "connecting (headless)");
let handle = session::start(session::SessionParams {
host,
port,
mode,
compositor: CompositorPref::Auto,
gamepad: GamepadPref::Auto,
bitrate_kbps,
// Headless CLI path (test/scripting) — stereo baseline; the GUI sources this from settings.
audio_channels: 2,
mic_enabled: flag("--mic"),
hdr_enabled: !flag("--no-hdr"),
decoder,
// `--codec h264|hevc|av1` sets the soft preference; default auto (host decides).
preferred_codec: match arg("--codec").as_deref() {
Some("h264") | Some("avc") => punktfunk_core::quic::CODEC_H264,
Some("hevc") | Some("h265") => punktfunk_core::quic::CODEC_HEVC,
Some("av1") => punktfunk_core::quic::CODEC_AV1,
_ => 0,
},
pin,
identity,
// Headless CLI uses the normal (short) handshake budget; the long request-access wait is a
// GUI-only flow.
connect_timeout: Duration::from_secs(15),
});
let deadline = Instant::now() + Duration::from_secs(60);
let mut frames_seen = 0u64;
loop {
while let Ok(ev) = handle.events.try_recv() {
match ev {
session::SessionEvent::Connected {
mode, fingerprint, ..
} => tracing::info!(?mode, fp = %trust::hex(&fingerprint), "connected"),
// With per-AU 0xCF host timings the combined host+network stage splits into
// host (capture→sent on the host) + net; an old host emits none → combined only.
session::SessionEvent::Stats(s) if s.split => tracing::info!(
fps = format!("{:.0}", s.fps),
mbps = format!("{:.1}", s.mbps),
decode_p50_ms = format!("{:.2}", s.decode_ms),
hostnet_p50_ms = format!("{:.2}", s.hostnet_ms),
host_p50_ms = format!("{:.2}", s.host_ms),
net_p50_ms = format!("{:.2}", s.net_ms),
frames_seen,
"stats"
),
session::SessionEvent::Stats(s) => tracing::info!(
fps = format!("{:.0}", s.fps),
mbps = format!("{:.1}", s.mbps),
decode_p50_ms = format!("{:.2}", s.decode_ms),
hostnet_p50_ms = format!("{:.2}", s.hostnet_ms),
frames_seen,
"stats"
),
session::SessionEvent::Failed { msg, .. } => {
tracing::error!(%msg, "connect failed");
return;
}
session::SessionEvent::Ended(err) => {
tracing::info!(reason = err.as_deref().unwrap_or("done"), "session ended");
return;
}
}
}
while handle.frames.try_recv().is_ok() {
frames_seen += 1;
}
if Instant::now() > deadline {
tracing::info!(frames_seen, "harness deadline — stopping");
handle.stop.store(true, std::sync::atomic::Ordering::SeqCst);
return;
}
std::thread::sleep(Duration::from_millis(2));
}
} }
/// `--discover`: browse the LAN for punktfunk hosts (mDNS) and print them, then exit. /// `--discover`: browse the LAN for punktfunk hosts (mDNS) and print them, then exit.
-915
View File
@@ -1,915 +0,0 @@
//! Direct3D11 presenter for a WinUI 3 `SwapChainPanel`. It draws a decoded frame Contain-fit into a
//! **composition** flip-model swapchain, which the reactor stream page binds to the panel via
//! `SwapChainPanelHandle::set_swap_chain`. After that one UI-thread bind, the presenter lives on
//! the dedicated render thread ([`crate::render`]) — presenting never touches (or is stalled by)
//! the XAML thread.
//!
//! Two frame sources, ONE YCbCr→RGB shader whose conversion rows arrive per frame in a constant
//! buffer (`pf_client_core::video::csc_rows` from the frame's CICP signaling — identical colour
//! math for both sources, and the stream's signaled matrix/range is honored, not assumed):
//!
//! * **GPU (D3D11VA)** — [`crate::video::GpuFrame`] is a slice of the decoder-only NV12/P010
//! texture array. One `CopySubresourceRegion` with a display-size box moves the slice — **both
//! planes; in D3D11 a planar slice is a single subresource** (unlike D3D12) — into our
//! sampleable texture, which per-plane SRVs (R8/R8G8, R16/R16G16) expose to the shaders. The
//! source box is mandatory: the decode array is coded-size (e.g. 1920×1088), the target
//! display-size (1920×1080), and D3D11 silently drops size-mismatched full-resource copies.
//! * **CPU upload** — [`crate::video::CpuFrame`] carries NV12/P010 planes from the software
//! decoder; they upload into two dynamic plane textures feeding the same SRV slots/shaders.
//!
//! **Pacing**: the swapchain is created with `DXGI_SWAP_CHAIN_FLAG_FRAME_LATENCY_WAITABLE_OBJECT`
//! and `SetMaximumFrameLatency(1)` (flagless fallback for odd drivers). The render thread waits
//! on the latency waitable before drawing, so at most one present is ever queued (minimum compose
//! latency) and a stream faster than the display drops frames *before* any GPU work. Every
//! `ResizeBuffers` must re-pass the creation flags — that's `swap_flags`.
//!
//! **HiDPI**: buffers are sized in physical pixels and `IDXGISwapChain2::SetMatrixTransform`
//! (scale 96/DPI) maps them to the panel's DIP coordinate space — without it XAML samples a
//! DIP-sized buffer up and the video is blurry at 125/150 % scaling.
//!
//! **HDR10**: when a frame is BT.2020 PQ the swapchain flips to `R10G10B10A2` +
//! `DXGI_COLOR_SPACE_RGB_FULL_G2084_NONE_P2020` (+ HDR10 metadata) via `ResizeBuffers`/
//! `SetColorSpace1`; the shader output is already PQ-encoded so the compositor maps PQ→display. SDR
//! stays 8-bit B8G8R8A8.
//!
//! All `windows` types here come from the same windows-rs commit as `windows-reactor`, so the
//! `IDXGISwapChain1` handed to `set_swap_chain` satisfies reactor's `windows_core::Interface`.
use crate::video::{CpuFrame, DecodedFrame, GpuFrame};
use anyhow::{anyhow, Context, Result};
use windows::core::{Interface, PCSTR};
use windows::Win32::Foundation::{CloseHandle, HANDLE, WAIT_OBJECT_0};
use windows::Win32::Graphics::Direct3D::Fxc::{D3DCompile, D3DCOMPILE_OPTIMIZATION_LEVEL3};
use windows::Win32::Graphics::Direct3D::{
ID3DBlob, D3D_PRIMITIVE_TOPOLOGY_TRIANGLELIST, D3D_SRV_DIMENSION_TEXTURE2D,
};
use windows::Win32::Graphics::Direct3D11::*;
use windows::Win32::Graphics::Dxgi::Common::*;
use windows::Win32::Graphics::Dxgi::*;
use windows::Win32::System::Threading::WaitForSingleObject;
// One vertex shader (fullscreen triangle) + ONE pixel shader for every colour combination:
// tex0 is the luma plane, tex1 the chroma plane, and the YCbCr→RGB conversion arrives as three
// constant-buffer rows precomputed on the CPU per frame (`pf_client_core::video::csc_rows` —
// bit-depth exact, range expansion + the P010 ×65535/65472 high-bit repack folded in). One shader
// honors whatever the stream signals (BT.601/709/2020, full/limited, 8/10-bit) instead of the old
// two hardcoded matrices — a BT.601-signaled stream (a Linux host's RGB-input NVENC) used to
// render with BT.709 coefficients, a constant hue error. A PQ stream's rows yield PQ-encoded
// RGB passed through as-is to the HDR10 swapchain, exactly as before.
const SHADER_HLSL: &str = r#"
struct VSOut { float4 pos : SV_Position; float2 uv : TEXCOORD0; };
VSOut vs_main(uint vid : SV_VertexID) {
float2 uv = float2((vid << 1) & 2, vid & 2);
VSOut o;
o.pos = float4(uv * float2(2, -2) + float2(-1, 1), 0, 1);
o.uv = uv;
return o;
}
Texture2D tex0 : register(t0);
Texture2D tex1 : register(t1);
SamplerState smp : register(s0);
cbuffer Csc : register(b0) {
float4 r0; // rgb[i] = dot(ri.xyz, yuv) + ri.w
float4 r1;
float4 r2;
};
float4 ps_yuv(VSOut i) : SV_Target {
// 4:2:0 chroma is left-cosited (H.273 type 0 — the default inference when unsignaled, and
// what the hosts produce), but sampling the half-res plane at the luma UV assumes CENTER
// siting — a ~0.5-luma-px rightward chroma shift on hard colored edges. Offset +0.25 chroma
// texels to re-align (the same correction the Apple client applies). Self-disables when the
// plane widths match (a full-size 4:4:4 chroma plane has no subsampling to correct).
float lw, lh, cw, ch;
tex0.GetDimensions(lw, lh);
tex1.GetDimensions(cw, ch);
float2 cuv = i.uv;
if (cw < lw) { cuv.x += 0.25 / cw; }
float3 yuv = float3(tex0.Sample(smp, i.uv).r, tex1.Sample(smp, cuv).rg);
float3 rgb = float3(dot(r0.xyz, yuv) + r0.w,
dot(r1.xyz, yuv) + r1.w,
dot(r2.xyz, yuv) + r2.w);
return float4(saturate(rgb), 1.0);
}
"#;
/// The currently bound frame: per-plane SRVs (over the GPU sample texture or the CPU plane
/// textures). Redraws (resize, letterbox) re-present it — the CSC constant buffer still holds
/// this frame's rows, and the swapchain mode was latched by `set_hdr` when the frame arrived.
struct Bound {
y: ID3D11ShaderResourceView,
c: ID3D11ShaderResourceView,
}
pub struct Presenter {
device: ID3D11Device,
context: ID3D11DeviceContext,
vs: ID3D11VertexShader,
ps_yuv: ID3D11PixelShader,
/// Dynamic constant buffer holding the bound frame's three CSC rows (`csc_rows`), rewritten
/// on every bind (colour signaling can flip in-band, e.g. the host's SDR→HDR re-init).
csc_buf: ID3D11Buffer,
sampler: ID3D11SamplerState,
swap: IDXGISwapChain1,
/// Creation flags — MUST be re-passed to every `ResizeBuffers` or it fails.
swap_flags: u32,
/// The frame-latency waitable (owned; closed in `Drop`), `None` on the flagless fallback.
waitable: Option<HANDLE>,
rtv: Option<ID3D11RenderTargetView>,
/// GPU path: sampleable copy target for the decoded slice — `(tex, w, h, ten_bit)`, recreated
/// when the decoded size/bit depth changes. Format must equal the decode array's (NV12/P010).
sample_tex: Option<(ID3D11Texture2D, u32, u32, bool)>,
/// The last GPU frame, held until the NEXT bind so its decode surface stays out of the reuse
/// pool at least until this frame's copy has been queued ahead of any later decoder write.
gpu_frame: Option<GpuFrame>,
/// CPU path: dynamic luma + chroma plane textures + their SRVs — `(y, uv, y_srv, uv_srv, w, h,
/// ten_bit)`, recreated when the decoded size/bit depth changes.
#[allow(clippy::type_complexity)]
plane_tex: Option<(
ID3D11Texture2D,
ID3D11Texture2D,
ID3D11ShaderResourceView,
ID3D11ShaderResourceView,
u32,
u32,
bool,
)>,
bound: Option<Bound>,
/// Source frame dimensions, for the Contain-fit letterbox.
src_w: u32,
src_h: u32,
/// Panel (swapchain) size in physical pixels + the window DPI, updated on resize.
panel_w: u32,
panel_h: u32,
dpi: u32,
/// Whether the swapchain is currently in 10-bit HDR10 (R10G10B10A2 + ST.2084) mode.
hdr: bool,
/// The source's static HDR mastering metadata received over the protocol (`0xCE`), applied via
/// `SetHDRMetaData` so the display tone-maps from the real grade instead of a generic 1000-nit
/// guess. `None` until the first update arrives (then the generic baseline is used).
hdr_meta: Option<punktfunk_core::quic::HdrMeta>,
}
/// Latest source HDR mastering metadata, written by the session pump (`session.rs`, the sole
/// `next_hdr_meta` consumer) and read by the render thread before each present — decoupled so the
/// presenter doesn't need the connector. One session at a time on the client, so a single slot.
pub static LATEST_HDR_META: std::sync::Mutex<Option<punktfunk_core::quic::HdrMeta>> =
std::sync::Mutex::new(None);
impl Presenter {
/// Create the presenter on the process-wide shared D3D11 device (the one the decoder uses), plus
/// the composition swapchain + shaders, sized to the panel in physical pixels at `dpi`.
pub fn new(width: u32, height: u32, dpi: u32) -> Result<Presenter> {
let shared = crate::gpu::shared().ok_or_else(|| anyhow!("no shared D3D11 device"))?;
let device = shared.device.clone();
let context = shared.context.clone();
let (vs, ps_yuv, sampler) = build_pipeline(&device)?;
// The per-frame CSC rows (three float4s). Dynamic: rewritten with Map-discard on bind.
let csc_desc = D3D11_BUFFER_DESC {
ByteWidth: 48,
Usage: D3D11_USAGE_DYNAMIC,
BindFlags: D3D11_BIND_CONSTANT_BUFFER.0 as u32,
CPUAccessFlags: D3D11_CPU_ACCESS_WRITE.0 as u32,
..Default::default()
};
let csc_buf = unsafe {
let mut b = None;
device
.CreateBuffer(&csc_desc, None, Some(&mut b))
.context("CreateBuffer (CSC rows)")?;
b.ok_or_else(|| anyhow!("null CSC constant buffer"))?
};
let (swap, swap_flags) =
create_composition_swapchain(&device, width.max(1), height.max(1))?;
// ≤1 queued present: the render thread blocks on the waitable, so a frame is only drawn
// when the compositor is ready to take it — the newest-wins drain happens after the wait.
let waitable = (swap_flags & DXGI_SWAP_CHAIN_FLAG_FRAME_LATENCY_WAITABLE_OBJECT.0 as u32
!= 0)
.then(|| unsafe {
let sc2: IDXGISwapChain2 = swap.cast().ok()?;
sc2.SetMaximumFrameLatency(1).ok()?;
let h = sc2.GetFrameLatencyWaitableObject();
(!h.is_invalid()).then_some(h)
})
.flatten();
let p = Presenter {
device,
context,
vs,
ps_yuv,
csc_buf,
sampler,
swap,
swap_flags,
waitable,
rtv: None,
sample_tex: None,
gpu_frame: None,
plane_tex: None,
bound: None,
src_w: 1,
src_h: 1,
panel_w: width.max(1),
panel_h: height.max(1),
dpi: dpi.max(96),
hdr: false,
hdr_meta: None,
};
p.apply_dpi_matrix();
Ok(p)
}
/// Block until the swapchain can take another present (≤ `timeout_ms`). True when a present
/// slot is free; also true on the flagless fallback (no throttle available, just present).
pub fn wait_present_slot(&self, timeout_ms: u32) -> bool {
match self.waitable {
Some(h) => unsafe { WaitForSingleObject(h, timeout_ms) == WAIT_OBJECT_0 },
None => true,
}
}
/// Update the source HDR mastering metadata (from the `0xCE` plane). Stored for the next HDR
/// swapchain switch, and applied immediately if already presenting HDR. A no-op when unchanged
/// (so it's cheap to call every frame from the render loop).
pub fn set_hdr_metadata(&mut self, meta: punktfunk_core::quic::HdrMeta) {
if self.hdr_meta == Some(meta) {
return;
}
self.hdr_meta = Some(meta);
if self.hdr {
unsafe { self.apply_hdr_metadata() };
}
}
/// The DXGI swapchain to hand to `SwapChainPanelHandle::set_swap_chain`.
pub fn swap_chain(&self) -> &IDXGISwapChain1 {
&self.swap
}
/// Resize the back buffers to the panel's new size in physical pixels at `dpi` (drops the
/// stale RTV, re-applies the DIP↔pixel matrix).
pub fn resize(&mut self, width: u32, height: u32, dpi: u32) {
let dpi = dpi.max(96);
if width == 0
|| height == 0
|| (width == self.panel_w && height == self.panel_h && dpi == self.dpi)
{
return;
}
self.rtv = None; // release all back-buffer refs before ResizeBuffers
unsafe {
if let Err(e) = self.swap.ResizeBuffers(
0,
width,
height,
DXGI_FORMAT_UNKNOWN,
DXGI_SWAP_CHAIN_FLAG(self.swap_flags as i32),
) {
tracing::warn!(error = %e, "ResizeBuffers failed");
return;
}
}
self.panel_w = width;
self.panel_h = height;
self.dpi = dpi;
self.apply_dpi_matrix();
}
/// Map the pixel-sized buffers into the panel's DIP coordinate space (scale 96/DPI) — XAML
/// otherwise stretches whatever size the buffers are to the panel's DIP bounds (blurry).
fn apply_dpi_matrix(&self) {
let s = 96.0 / self.dpi as f32;
if let Ok(sc2) = self.swap.cast::<IDXGISwapChain2>() {
let m = DXGI_MATRIX_3X2_F {
_11: s,
_22: s,
..Default::default()
};
if let Err(e) = unsafe { sc2.SetMatrixTransform(&m) } {
tracing::warn!(error = %e, "SetMatrixTransform failed");
}
}
}
/// Present one decoded frame (Contain-fit) — or, when `frame` is `None`, re-present the last
/// one (or black). Called from the render thread. Takes the frame by value: the GPU path
/// retains the decoder surface until the next bind.
pub fn present(&mut self, frame: Option<DecodedFrame>) {
match frame {
Some(DecodedFrame::Cpu(c)) => {
if c.hdr != self.hdr {
self.set_hdr(c.hdr);
}
if let Err(e) = self.upload(&c) {
tracing::warn!(error = %e, "frame upload failed");
}
}
Some(DecodedFrame::Gpu(g)) => {
if g.hdr != self.hdr {
self.set_hdr(g.hdr);
}
if let Err(e) = self.bind_gpu(g) {
tracing::warn!(error = %e, "GPU frame bind failed");
}
}
None => {}
}
self.draw();
}
/// Copy the decoded slice into our sampleable texture and build per-plane SRVs over it. The
/// decode array is decoder-only (NVIDIA won't bind a decoder array as a shader resource), so
/// it can't be sampled directly — one GPU-to-GPU copy makes the frame sampleable on every
/// vendor. D3D11 planar semantics: the slice is ONE subresource (both planes copy together),
/// and the source box is display-size (the array is coded-size; a full-resource copy would
/// size-mismatch and be silently dropped).
fn bind_gpu(&mut self, g: GpuFrame) -> Result<()> {
let src: ID3D11Texture2D = unsafe {
let raw = g.texture_ptr();
ID3D11Texture2D::from_raw_borrowed(&raw)
.ok_or_else(|| anyhow!("null D3D11 texture"))?
.clone()
};
self.ensure_sample_tex(g.width, g.height, g.ten_bit)?;
let dst = self.sample_tex.as_ref().unwrap().0.clone();
// Even-aligned luma coordinates (NV12/P010 chroma is 2×2 subsampled).
let src_box = D3D11_BOX {
left: 0,
top: 0,
front: 0,
right: g.width & !1,
bottom: g.height & !1,
back: 1,
};
unsafe {
self.context
.CopySubresourceRegion(&dst, 0, 0, 0, 0, &src, g.index, Some(&src_box));
}
let (fy, fc) = plane_formats(g.ten_bit);
let y = self.plane_srv(&dst, fy)?;
let c = self.plane_srv(&dst, fc)?;
self.write_csc_rows(g.color, g.ten_bit)?;
self.src_w = g.width;
self.src_h = g.height;
self.bound = Some(Bound { y, c });
// Hold the frame until the next bind: its decode surface stays out of the reuse pool
// until this copy is queued ahead of any later decoder write (previous frame drops here).
self.gpu_frame = Some(g);
Ok(())
}
/// Ensure the sampleable copy texture matches the decoded frame's size + bit depth (NV12 for
/// 8-bit, P010 for 10-bit — the same format as the decode array, a `CopySubresourceRegion`
/// requirement), recreating it on a change.
fn ensure_sample_tex(&mut self, w: u32, h: u32, ten_bit: bool) -> Result<()> {
if matches!(&self.sample_tex, Some((_, tw, th, tb)) if *tw == w && *th == h && *tb == ten_bit)
{
return Ok(());
}
let desc = D3D11_TEXTURE2D_DESC {
Width: w,
Height: h,
MipLevels: 1,
ArraySize: 1,
Format: if ten_bit {
DXGI_FORMAT_P010
} else {
DXGI_FORMAT_NV12
},
SampleDesc: DXGI_SAMPLE_DESC {
Count: 1,
Quality: 0,
},
Usage: D3D11_USAGE_DEFAULT,
BindFlags: D3D11_BIND_SHADER_RESOURCE.0 as u32,
CPUAccessFlags: 0,
MiscFlags: 0,
};
let tex = unsafe {
let mut t = None;
self.device
.CreateTexture2D(&desc, None, Some(&mut t))
.context("CreateTexture2D (sample target)")?;
t.ok_or_else(|| anyhow!("null sample texture"))?
};
self.sample_tex = Some((tex, w, h, ten_bit));
Ok(())
}
/// A shader-resource view over one plane of a single (non-array) NV12/P010 texture — the
/// R8/R8G8 (or R16/R16G16) format selects the luma vs. chroma plane (the D3D11 video
/// sub-format trick).
fn plane_srv(
&self,
tex: &ID3D11Texture2D,
format: DXGI_FORMAT,
) -> Result<ID3D11ShaderResourceView> {
let desc = D3D11_SHADER_RESOURCE_VIEW_DESC {
Format: format,
ViewDimension: D3D_SRV_DIMENSION_TEXTURE2D,
Anonymous: D3D11_SHADER_RESOURCE_VIEW_DESC_0 {
Texture2D: D3D11_TEX2D_SRV {
MostDetailedMip: 0,
MipLevels: 1,
},
},
};
unsafe {
let mut srv = None;
self.device
.CreateShaderResourceView(tex, Some(&desc), Some(&mut srv))
.context("CreateShaderResourceView (plane)")?;
srv.ok_or_else(|| anyhow!("null SRV"))
}
}
/// Upload a software-decoded frame's two planes into the dynamic plane textures (created to
/// match size/bit depth), feeding the same SRV slots + shaders as the GPU path.
fn upload(&mut self, frame: &CpuFrame) -> Result<()> {
let (w, h) = (frame.width, frame.height);
let rebuild = !matches!(&self.plane_tex,
Some((.., tw, th, tb)) if *tw == w && *th == h && *tb == frame.ten_bit);
if rebuild {
let (fy, fc) = plane_formats(frame.ten_bit);
let y = self.dynamic_tex(w, h, fy)?;
let uv = self.dynamic_tex(w.div_ceil(2), h.div_ceil(2), fc)?;
let y_srv = self.plane_srv(&y, fy)?;
let uv_srv = self.plane_srv(&uv, fc)?;
self.plane_tex = Some((y, uv, y_srv, uv_srv, w, h, frame.ten_bit));
}
let (y, uv, y_srv, uv_srv, ..) = self.plane_tex.as_ref().unwrap();
let bytes = if frame.ten_bit { 2 } else { 1 };
self.map_rows(y, &frame.y, frame.y_stride, w as usize * bytes, h as usize)?;
self.map_rows(
uv,
&frame.uv,
frame.uv_stride,
w.div_ceil(2) as usize * 2 * bytes,
h.div_ceil(2) as usize,
)?;
let (y_srv, uv_srv) = (y_srv.clone(), uv_srv.clone());
self.write_csc_rows(frame.color, frame.ten_bit)?;
self.src_w = w;
self.src_h = h;
self.bound = Some(Bound {
y: y_srv,
c: uv_srv,
});
self.gpu_frame = None; // drop any held GPU frame
Ok(())
}
fn dynamic_tex(&self, w: u32, h: u32, format: DXGI_FORMAT) -> Result<ID3D11Texture2D> {
let desc = D3D11_TEXTURE2D_DESC {
Width: w,
Height: h,
MipLevels: 1,
ArraySize: 1,
Format: format,
SampleDesc: DXGI_SAMPLE_DESC {
Count: 1,
Quality: 0,
},
Usage: D3D11_USAGE_DYNAMIC,
BindFlags: D3D11_BIND_SHADER_RESOURCE.0 as u32,
CPUAccessFlags: D3D11_CPU_ACCESS_WRITE.0 as u32,
MiscFlags: 0,
};
unsafe {
let mut t = None;
self.device
.CreateTexture2D(&desc, None, Some(&mut t))
.context("CreateTexture2D (plane)")?;
t.ok_or_else(|| anyhow!("null plane texture"))
}
}
/// Recompute the bound frame's YCbCr→RGB rows from its CICP signaling and Map-discard them
/// into the CSC constant buffer. `ten_bit` selects the 10-bit code points AND the P010
/// high-bit repack (the plane SRVs are R16/R16G16 UNORM for 10-bit).
fn write_csc_rows(&self, color: pf_client_core::video::ColorDesc, ten_bit: bool) -> Result<()> {
let rows = pf_client_core::video::csc_rows(color, if ten_bit { 10 } else { 8 }, ten_bit);
unsafe {
let mut mapped = D3D11_MAPPED_SUBRESOURCE::default();
self.context
.Map(
&self.csc_buf,
0,
D3D11_MAP_WRITE_DISCARD,
0,
Some(&mut mapped),
)
.context("Map CSC constant buffer")?;
std::ptr::copy_nonoverlapping(
rows.as_ptr() as *const u8,
mapped.pData as *mut u8,
48, // [[f32; 4]; 3]
);
self.context.Unmap(&self.csc_buf, 0);
}
Ok(())
}
/// Map-discard `tex` and copy `rows` rows of `row_bytes` from `src` (stride `src_pitch`).
fn map_rows(
&self,
tex: &ID3D11Texture2D,
src: &[u8],
src_pitch: usize,
row_bytes: usize,
rows: usize,
) -> Result<()> {
unsafe {
let mut mapped = D3D11_MAPPED_SUBRESOURCE::default();
self.context
.Map(tex, 0, D3D11_MAP_WRITE_DISCARD, 0, Some(&mut mapped))
.context("Map plane texture")?;
let dst = mapped.pData as *mut u8;
let dst_pitch = mapped.RowPitch as usize;
let n = row_bytes.min(src_pitch);
for r in 0..rows {
std::ptr::copy_nonoverlapping(
src.as_ptr().add(r * src_pitch),
dst.add(r * dst_pitch),
n,
);
}
self.context.Unmap(tex, 0);
}
Ok(())
}
fn draw(&mut self) {
let Ok(rtv) = self.rtv() else {
return;
};
let (pw, ph) = (self.panel_w, self.panel_h);
unsafe {
let c = &self.context;
c.ClearRenderTargetView(&rtv, &[0.0, 0.0, 0.0, 1.0]);
if let Some(bound) = &self.bound {
// Contain-fit viewport: scale to the smaller axis, centre, letterbox the rest.
let (ww, wh, vfw, vfh) = (
pw as f32,
ph as f32,
self.src_w.max(1) as f32,
self.src_h.max(1) as f32,
);
let scale = (ww / vfw).min(wh / vfh);
let (dw, dh) = (vfw * scale, vfh * scale);
let (ox, oy) = ((ww - dw) / 2.0, (wh - dh) / 2.0);
c.OMSetRenderTargets(Some(&[Some(rtv.clone())]), None);
let vp = D3D11_VIEWPORT {
TopLeftX: ox,
TopLeftY: oy,
Width: dw,
Height: dh,
MinDepth: 0.0,
MaxDepth: 1.0,
};
c.RSSetViewports(Some(&[vp]));
c.IASetInputLayout(None);
c.IASetPrimitiveTopology(D3D_PRIMITIVE_TOPOLOGY_TRIANGLELIST);
c.VSSetShader(&self.vs, None);
c.PSSetShader(&self.ps_yuv, None);
c.PSSetConstantBuffers(0, Some(&[Some(self.csc_buf.clone())]));
c.PSSetShaderResources(0, Some(&[Some(bound.y.clone()), Some(bound.c.clone())]));
c.PSSetSamplers(0, Some(&[Some(self.sampler.clone())]));
c.Draw(3, 0);
}
let _ = self.swap.Present(1, DXGI_PRESENT(0));
}
}
/// Switch the swapchain between 8-bit SDR (B8G8R8A8, BT.709) and 10-bit HDR10 (R10G10B10A2,
/// ST.2084 PQ BT.2020). `ResizeBuffers` changes the back-buffer format in place, so the panel
/// binding (`set_swap_chain`) stays valid — no rebind. Both frame sources already produce
/// PQ-encoded BT.2020 for HDR, so the colour space is all the compositor needs.
fn set_hdr(&mut self, on: bool) {
self.rtv = None; // release back-buffer refs before ResizeBuffers
let format = if on {
DXGI_FORMAT_R10G10B10A2_UNORM
} else {
DXGI_FORMAT_B8G8R8A8_UNORM
};
unsafe {
if let Err(e) = self.swap.ResizeBuffers(
0,
self.panel_w,
self.panel_h,
format,
DXGI_SWAP_CHAIN_FLAG(self.swap_flags as i32),
) {
tracing::warn!(error = %e, "ResizeBuffers for HDR switch failed");
return;
}
let colorspace = if on {
DXGI_COLOR_SPACE_RGB_FULL_G2084_NONE_P2020
} else {
DXGI_COLOR_SPACE_RGB_FULL_G22_NONE_P709
};
if let Ok(sc3) = self.swap.cast::<IDXGISwapChain3>() {
// Only set a colour space the swapchain accepts for present (on an SDR desktop the
// DWM still tone-maps HDR10 → SDR, so leaving the default there is fine).
if let Ok(support) = sc3.CheckColorSpaceSupport(colorspace) {
if support & DXGI_SWAP_CHAIN_COLOR_SPACE_SUPPORT_FLAG_PRESENT.0 as u32 != 0 {
if let Err(e) = sc3.SetColorSpace1(colorspace) {
// A silent failure here presents PQ content as SDR gamma (crushed/dark) —
// surface it instead of swallowing it.
tracing::warn!(error = %e, ?colorspace, "SetColorSpace1 failed");
}
} else if on {
tracing::warn!("swapchain rejects BT.2020 PQ present colour space (SDR display?) — DWM tone-maps");
}
}
}
self.hdr = on;
if on {
self.apply_hdr_metadata();
}
}
self.apply_dpi_matrix(); // belt-and-braces: keep the DIP mapping across the format switch
tracing::info!(hdr = on, "swapchain colour mode switched");
}
/// Push the current `DXGI_HDR_METADATA_HDR10` to the swapchain. Uses the source's received
/// mastering metadata when known, else a generic HDR10 baseline. Caller ensures HDR mode.
unsafe fn apply_hdr_metadata(&self) {
if let Ok(sc4) = self.swap.cast::<IDXGISwapChain4>() {
let md = self
.hdr_meta
.map(hdr_meta_to_dxgi)
.unwrap_or_else(generic_hdr10_metadata);
let bytes = std::slice::from_raw_parts(
&md as *const DXGI_HDR_METADATA_HDR10 as *const u8,
std::mem::size_of::<DXGI_HDR_METADATA_HDR10>(),
);
if let Err(e) = sc4.SetHDRMetaData(DXGI_HDR_METADATA_TYPE_HDR10, Some(bytes)) {
tracing::warn!(error = %e, "SetHDRMetaData failed");
}
}
}
fn rtv(&mut self) -> Result<ID3D11RenderTargetView> {
if self.rtv.is_none() {
let back: ID3D11Texture2D = unsafe { self.swap.GetBuffer(0).context("GetBuffer")? };
let rtv = unsafe {
let mut v = None;
self.device
.CreateRenderTargetView(&back, None, Some(&mut v))
.context("CreateRenderTargetView")?;
v.unwrap()
};
self.rtv = Some(rtv);
}
Ok(self.rtv.clone().unwrap())
}
}
impl Drop for Presenter {
fn drop(&mut self) {
if let Some(h) = self.waitable.take() {
unsafe {
let _ = CloseHandle(h);
}
}
}
}
/// Luma + chroma plane view formats for NV12 (8-bit) vs P010 (10-in-16-bit).
fn plane_formats(ten_bit: bool) -> (DXGI_FORMAT, DXGI_FORMAT) {
if ten_bit {
(DXGI_FORMAT_R16_UNORM, DXGI_FORMAT_R16G16_UNORM)
} else {
(DXGI_FORMAT_R8_UNORM, DXGI_FORMAT_R8G8_UNORM)
}
}
/// A composition flip-model swapchain (no HWND) for binding to a XAML `SwapChainPanel`, with the
/// frame-latency waitable when the driver allows it. Returns the swapchain + the flags it was
/// created with (every `ResizeBuffers` must re-pass them).
fn create_composition_swapchain(
device: &ID3D11Device,
width: u32,
height: u32,
) -> Result<(IDXGISwapChain1, u32)> {
let dxdev: IDXGIDevice = device.cast().context("IDXGIDevice cast")?;
let factory: IDXGIFactory2 = unsafe {
let adapter = dxdev.GetAdapter().context("GetAdapter")?;
adapter.GetParent().context("GetParent (IDXGIFactory2)")?
};
let mut desc = DXGI_SWAP_CHAIN_DESC1 {
Width: width,
Height: height,
Format: DXGI_FORMAT_B8G8R8A8_UNORM,
Stereo: false.into(),
SampleDesc: DXGI_SAMPLE_DESC {
Count: 1,
Quality: 0,
},
BufferUsage: DXGI_USAGE_RENDER_TARGET_OUTPUT,
BufferCount: 2,
Scaling: DXGI_SCALING_STRETCH,
SwapEffect: DXGI_SWAP_EFFECT_FLIP_SEQUENTIAL,
// IGNORE (opaque), not PREMULTIPLIED: the video fills the panel with opaque RGB either way.
AlphaMode: DXGI_ALPHA_MODE_IGNORE,
Flags: DXGI_SWAP_CHAIN_FLAG_FRAME_LATENCY_WAITABLE_OBJECT.0 as u32,
};
unsafe {
match factory.CreateSwapChainForComposition(device, &desc, None) {
Ok(sc) => Ok((sc, desc.Flags)),
Err(e) => {
// Odd driver/WARP combinations can reject the waitable — fall back to plain
// Present(1) pacing rather than failing the stream page.
tracing::warn!(error = %e, "waitable swapchain rejected — creating without");
desc.Flags = 0;
let sc = factory
.CreateSwapChainForComposition(device, &desc, None)
.context("CreateSwapChainForComposition")?;
Ok((sc, 0))
}
}
}
}
fn build_pipeline(
device: &ID3D11Device,
) -> Result<(ID3D11VertexShader, ID3D11PixelShader, ID3D11SamplerState)> {
let vs_blob = compile(SHADER_HLSL, "vs_main", "vs_5_0")?;
let yuv_blob = compile(SHADER_HLSL, "ps_yuv", "ps_5_0")?;
unsafe {
let mut vs = None;
device
.CreateVertexShader(blob_bytes(&vs_blob), None, Some(&mut vs))
.context("CreateVertexShader")?;
let mut ps_yuv = None;
device
.CreatePixelShader(blob_bytes(&yuv_blob), None, Some(&mut ps_yuv))
.context("CreatePixelShader (yuv)")?;
let sdesc = D3D11_SAMPLER_DESC {
Filter: D3D11_FILTER_MIN_MAG_MIP_LINEAR,
AddressU: D3D11_TEXTURE_ADDRESS_CLAMP,
AddressV: D3D11_TEXTURE_ADDRESS_CLAMP,
AddressW: D3D11_TEXTURE_ADDRESS_CLAMP,
MaxLOD: D3D11_FLOAT32_MAX,
..Default::default()
};
let mut sampler = None;
device
.CreateSamplerState(&sdesc, Some(&mut sampler))
.context("CreateSamplerState")?;
Ok((vs.unwrap(), ps_yuv.unwrap(), sampler.unwrap()))
}
}
fn compile(src: &str, entry: &str, target: &str) -> Result<ID3DBlob> {
let entry_c = std::ffi::CString::new(entry).unwrap();
let target_c = std::ffi::CString::new(target).unwrap();
let mut code = None;
let mut errors = None;
let r = unsafe {
D3DCompile(
src.as_ptr() as *const _,
src.len(),
PCSTR::null(),
None,
None,
PCSTR(entry_c.as_ptr() as *const u8),
PCSTR(target_c.as_ptr() as *const u8),
D3DCOMPILE_OPTIMIZATION_LEVEL3,
0,
&mut code,
Some(&mut errors),
)
};
if r.is_err() {
let msg = errors
.as_ref()
.map(|b| unsafe {
let p = b.GetBufferPointer() as *const u8;
let n = b.GetBufferSize();
String::from_utf8_lossy(std::slice::from_raw_parts(p, n)).to_string()
})
.unwrap_or_default();
return Err(anyhow!("D3DCompile {entry}: {msg}"));
}
code.ok_or_else(|| anyhow!("D3DCompile produced no bytecode"))
}
fn blob_bytes(blob: &ID3DBlob) -> &[u8] {
unsafe {
let p = blob.GetBufferPointer() as *const u8;
let n = blob.GetBufferSize();
std::slice::from_raw_parts(p, n)
}
}
/// True if any attached display is currently in HDR (BT.2020 PQ) mode. The client advertises HDR
/// caps only when this holds, so an SDR display gets a proper 8-bit BT.709 stream instead of PQ it
/// would mis-tone-map (the washed-out/dark failure); an HDR display self-tone-maps from the
/// mastering metadata. Coarse — checks ANY output, not the app's specific monitor; a mid-session
/// monitor move to/from HDR is a follow-up (the `Reconfigure` downgrade).
pub fn display_supports_hdr() -> bool {
unsafe {
let factory: IDXGIFactory1 = match CreateDXGIFactory1() {
Ok(f) => f,
Err(_) => return false,
};
let mut ai = 0u32;
while let Ok(adapter) = factory.EnumAdapters1(ai) {
ai += 1;
let mut oi = 0u32;
while let Ok(output) = adapter.EnumOutputs(oi) {
oi += 1;
if let Ok(o6) = output.cast::<IDXGIOutput6>() {
if let Ok(desc) = o6.GetDesc1() {
if desc.ColorSpace == DXGI_COLOR_SPACE_RGB_FULL_G2084_NONE_P2020 {
return true;
}
}
}
}
}
}
false
}
/// The HDR display's colour volume from `IDXGIOutput6::GetDesc1` — the first output currently in
/// HDR (BT.2020 PQ) mode, as [`HdrMeta`](punktfunk_core::quic::HdrMeta) for `Hello::display_hdr`.
/// The host writes this volume into its virtual display's EDID, so host apps tone-map to THIS
/// panel and the PQ stream needs no client-side rescue. Chromaticities come as CIE xy floats
/// (×50000 → ST.2086 units, G/B/R order); luminances as nits floats (max ×10000 → 0.0001-cd/m²
/// units); `MaxFullFrameLuminance` → MaxFALL (whole nits); MaxCLL stays 0 (a display has no
/// content light level). Same ANY-output coarseness as [`display_supports_hdr`] — the session
/// gates on that check first, so both look at the same panel in the single-HDR-display case.
pub fn display_hdr_volume() -> Option<punktfunk_core::quic::HdrMeta> {
let to_2086 = |v: f32| (v * 50000.0).round().clamp(0.0, 65535.0) as u16;
unsafe {
let factory: IDXGIFactory1 = CreateDXGIFactory1().ok()?;
let mut ai = 0u32;
while let Ok(adapter) = factory.EnumAdapters1(ai) {
ai += 1;
let mut oi = 0u32;
while let Ok(output) = adapter.EnumOutputs(oi) {
oi += 1;
let Ok(o6) = output.cast::<IDXGIOutput6>() else {
continue;
};
let Ok(desc) = o6.GetDesc1() else { continue };
if desc.ColorSpace != DXGI_COLOR_SPACE_RGB_FULL_G2084_NONE_P2020 {
continue;
}
return Some(punktfunk_core::quic::HdrMeta {
// ST.2086 order is G, B, R.
display_primaries: [
[to_2086(desc.GreenPrimary[0]), to_2086(desc.GreenPrimary[1])],
[to_2086(desc.BluePrimary[0]), to_2086(desc.BluePrimary[1])],
[to_2086(desc.RedPrimary[0]), to_2086(desc.RedPrimary[1])],
],
white_point: [to_2086(desc.WhitePoint[0]), to_2086(desc.WhitePoint[1])],
max_display_mastering_luminance: (desc.MaxLuminance.max(0.0) * 10_000.0).round()
as u32,
min_display_mastering_luminance: (desc.MinLuminance.max(0.0) * 10_000.0).round()
as u32,
max_cll: 0,
max_fall: desc.MaxFullFrameLuminance.max(0.0).round() as u16,
});
}
}
}
None
}
/// Generic HDR10 mastering metadata: BT.2020 primaries + D65 white, a 1000-nit mastering display,
/// MaxCLL 1000 / MaxFALL 400. The fallback used only until the host's real `0xCE` metadata arrives.
fn generic_hdr10_metadata() -> DXGI_HDR_METADATA_HDR10 {
DXGI_HDR_METADATA_HDR10 {
RedPrimary: [35400, 14600],
GreenPrimary: [8500, 39850],
BluePrimary: [6550, 2300],
WhitePoint: [15635, 16450],
MaxMasteringLuminance: 1000,
MinMasteringLuminance: 1, // 0.0001-nit units → 0.0001 nits
MaxContentLightLevel: 1000,
MaxFrameAverageLightLevel: 400,
}
}
/// Map the protocol's [`HdrMeta`](punktfunk_core::quic::HdrMeta) to `DXGI_HDR_METADATA_HDR10`.
/// Two careful conversions: HdrMeta stores primaries in **ST.2086 G,B,R order**, DXGI wants
/// **R,G,B**; and HdrMeta mastering luminance is in **0.0001-cd/m² units** while DXGI's
/// `MaxMasteringLuminance` is in **whole nits** (MinMasteringLuminance stays 0.0001-nit). Chromaticity
/// units (1/50000) and MaxCLL/MaxFALL (nits) match 1:1.
fn hdr_meta_to_dxgi(m: punktfunk_core::quic::HdrMeta) -> DXGI_HDR_METADATA_HDR10 {
let [g, b, r] = m.display_primaries; // ST.2086 order
DXGI_HDR_METADATA_HDR10 {
RedPrimary: r,
GreenPrimary: g,
BluePrimary: b,
WhitePoint: m.white_point,
MaxMasteringLuminance: m.max_display_mastering_luminance / 10_000, // 0.0001-nit → nit
MinMasteringLuminance: m.min_display_mastering_luminance, // already 0.0001-nit
MaxContentLightLevel: m.max_cll,
MaxFrameAverageLightLevel: m.max_fall,
}
}
+79
View File
@@ -0,0 +1,79 @@
//! Network speed-test probe — the GUI's per-host "Test Network Speed…" ([`crate::app`]'s
//! speed page) and the `--headless --speed-test` CLI.
//!
//! Split out of the former in-process session module: the shared spawned-`punktfunk-session`
//! binary owns real streaming now, but the speed test is a shell-side, decode-less measurement
//! over the real data plane, so it stays here. [`decodable_codecs`] rode along for the same
//! reason — the probe connect still advertises which codecs this client can decode.
use ffmpeg_next as ffmpeg;
use punktfunk_core::client::NativeClient;
use punktfunk_core::config::{CompositorPref, GamepadPref, Mode};
use std::time::{Duration, Instant};
/// The `quic` codec bitfield this client can decode — whatever FFmpeg has a decoder for (HEVC/H.264
/// always; AV1 when built in). Advertised to the host so it never emits a codec we can't decode.
pub fn decodable_codecs() -> u8 {
let _ = ffmpeg::init();
let mut bits = 0u8;
for (id, bit) in [
(ffmpeg::codec::Id::HEVC, punktfunk_core::quic::CODEC_HEVC),
(ffmpeg::codec::Id::H264, punktfunk_core::quic::CODEC_H264),
(ffmpeg::codec::Id::AV1, punktfunk_core::quic::CODEC_AV1),
] {
if ffmpeg::decoder::find(id).is_some() {
bits |= bit;
}
}
bits
}
/// Blocking speed-test probe (the GUI's per-host "Test" and the `--headless --speed-test` CLI):
/// a minimal identified connect (720p60 — the host builds a virtual output, but nothing is
/// decoded), then `request_probe` (a 2 s burst up to the host's 3 Gbps ceiling) polled to
/// completion. Run on a worker thread.
pub fn run_speed_probe(
addr: &str,
port: u16,
fp_hex: Option<&str>,
identity: (String, String),
) -> Result<punktfunk_core::client::ProbeOutcome, String> {
// Pin the saved/advertised fingerprint when we have one; a manual host measures over TOFU.
let pin = fp_hex.and_then(crate::trust::parse_hex32);
let c = NativeClient::connect(
addr,
port,
Mode {
width: 1280,
height: 720,
refresh_hz: 60,
},
CompositorPref::Auto,
GamepadPref::Auto,
0, // bitrate_kbps: host default
0, // video_caps: probe connect, nothing is decoded
2, // audio_channels: stereo baseline
decodable_codecs(),
0, // preferred_codec: no preference
None, // display_hdr: probe connect, nothing presents
None, // launch: no game
pin,
Some(identity),
Duration::from_secs(15),
)
.map_err(|e| format!("connect: {e:?}"))?;
c.request_probe(3_000_000, 2_000)
.map_err(|e| format!("probe: {e:?}"))?;
let deadline = Instant::now() + Duration::from_secs(10);
loop {
std::thread::sleep(Duration::from_millis(250));
if c.probe_result().done {
// Let the last UDP shards land before tearing down.
std::thread::sleep(Duration::from_millis(400));
return Ok(c.probe_result());
}
if Instant::now() > deadline {
return Err("probe timed out".to_string());
}
}
}
-285
View File
@@ -1,285 +0,0 @@
//! The dedicated video render thread: decoded frames flow session pump → bounded channel → here →
//! `Presenter::present`. Presenting off the XAML thread means UI jank (layout, input, dialogs)
//! never stalls video, and a filled present queue never blocks the UI thread — the two failure
//! modes of the old present-from-`on_rendering` design.
//!
//! Pacing: block on the channel (the host paces the stream), then on the swapchain's
//! frame-latency waitable (≤1 queued present — see `present.rs`), then drain to the NEWEST frame
//! so a stream faster than the display drops backlog before any GPU work. The UI thread only
//! writes panel size/DPI into [`RenderShared`] atomics; the loop applies them before the next
//! draw (and redraws the held frame after a resize — fresh back buffers are blank).
use crate::present::Presenter;
use crate::session::{FrameRx, FrameTimes};
use crossbeam_channel::RecvTimeoutError;
use std::sync::atomic::{AtomicBool, AtomicI64, AtomicU32, AtomicU64, Ordering};
use std::sync::Arc;
use std::time::{Duration, Instant};
/// The last 1-second render window, published for the HUD (one render thread at a time):
/// presents/s, frames skipped by the newest-wins drain, the end-to-end (capture→on-glass)
/// p50/p95 and the `display` stage (decoded→displayed) p50, all stamped post-`Present()`, in µs.
/// Zeroed when a render thread starts so a new session never shows the previous one's numbers.
static PRESENT_FPS: AtomicU32 = AtomicU32::new(0);
static PRESENT_SKIPPED: AtomicU32 = AtomicU32::new(0);
static E2E_P50_US: AtomicU64 = AtomicU64::new(0);
static E2E_P95_US: AtomicU64 = AtomicU64::new(0);
static DISPLAY_P50_US: AtomicU64 = AtomicU64::new(0);
/// The last render window's glass-side numbers (see the statics above) — the HUD's headline
/// (end-to-end) and trailing stage (display) come from here.
#[derive(Clone, Copy, Default, PartialEq)]
pub struct PresentStats {
/// Presents per second (includes resize redraws of a held frame).
pub fps: u32,
/// Frames dropped by the newest-wins drain this window (client-side pacing skips).
pub skipped: u32,
/// End-to-end capture→displayed p50, ms (host-clock corrected, measured directly).
pub e2e_p50_ms: f32,
/// End-to-end capture→displayed p95, ms.
pub e2e_p95_ms: f32,
/// `display` stage p50, ms: decoded → displayed, single-clock client-local.
pub display_p50_ms: f32,
}
pub fn present_stats() -> PresentStats {
PresentStats {
fps: PRESENT_FPS.load(Ordering::Relaxed),
skipped: PRESENT_SKIPPED.load(Ordering::Relaxed),
e2e_p50_ms: E2E_P50_US.load(Ordering::Relaxed) as f32 / 1000.0,
e2e_p95_ms: E2E_P95_US.load(Ordering::Relaxed) as f32 / 1000.0,
display_p50_ms: DISPLAY_P50_US.load(Ordering::Relaxed) as f32 / 1000.0,
}
}
/// UI-thread → render-thread state. Size is packed into ONE atomic (w<<32|h) so a resize never
/// tears into a (new-width, old-height) pair.
pub struct RenderShared {
size_px: AtomicU64,
dpi: AtomicU32,
stop: AtomicBool,
}
impl RenderShared {
pub fn new(width: u32, height: u32, dpi: u32) -> Arc<RenderShared> {
Arc::new(RenderShared {
size_px: AtomicU64::new(pack(width, height)),
dpi: AtomicU32::new(dpi),
stop: AtomicBool::new(false),
})
}
pub fn set_size(&self, width: u32, height: u32) {
self.size_px.store(pack(width, height), Ordering::Relaxed);
}
pub fn set_dpi(&self, dpi: u32) {
self.dpi.store(dpi, Ordering::Relaxed);
}
fn snapshot(&self) -> (u32, u32, u32) {
let s = self.size_px.load(Ordering::Relaxed);
((s >> 32) as u32, s as u32, self.dpi.load(Ordering::Relaxed))
}
}
fn pack(w: u32, h: u32) -> u64 {
((w as u64) << 32) | h as u64
}
/// Handle owned by the stream page; stops + joins the thread on unmount (and on drop, so a
/// navigation away can't leak a presenting thread).
pub struct RenderThread {
shared: Arc<RenderShared>,
join: Option<std::thread::JoinHandle<()>>,
}
impl RenderThread {
pub fn shared(&self) -> &Arc<RenderShared> {
&self.shared
}
pub fn stop_and_join(&mut self) {
self.shared.stop.store(true, Ordering::SeqCst);
if let Some(j) = self.join.take() {
let _ = j.join();
}
}
}
impl Drop for RenderThread {
fn drop(&mut self) {
self.stop_and_join();
}
}
/// Moves the presenter (COM interfaces, `!Send` by default) onto the render thread. Sound here:
/// the shared device + immediate context are multithread-protected (see `crate::gpu`), D3D/DXGI
/// objects are apartment-agile, and after this one handoff the swapchain/RTV/context calls happen
/// on exactly the render thread — the same single-owner discipline as `SharedDevice`.
struct SendPresenter(Presenter);
unsafe impl Send for SendPresenter {}
/// Spawn the render thread. `frames` carries `(frame, FrameTimes)`; `clock_offset_ns` maps our
/// wall clock onto the host's so the end-to-end (capture→on-glass) number is cross-machine valid
/// (same math as the pump's host+network stage). A live handle (loaded per present) so
/// mid-stream clock re-syncs keep the number honest after an NTP step / drift.
pub fn spawn(
presenter: Presenter,
frames: FrameRx,
shared: Arc<RenderShared>,
clock_offset_ns: Arc<AtomicI64>,
) -> RenderThread {
let boxed = SendPresenter(presenter);
let shared_w = shared.clone();
let join = std::thread::Builder::new()
.name("pf-render".into())
.spawn(move || run(boxed, frames, shared_w, clock_offset_ns))
.expect("spawn render thread");
RenderThread {
shared,
join: Some(join),
}
}
fn now_ns() -> u64 {
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_nanos() as u64)
.unwrap_or(0)
}
/// The window DPI, polled ~1 Hz as belt-and-braces for a monitor move that changes DPI without a
/// `SizeChanged` (same DIP size on both screens). `None` when the window isn't up (headless).
fn poll_window_dpi() -> Option<u32> {
use windows::Win32::UI::HiDpi::GetDpiForWindow;
use windows::Win32::UI::WindowsAndMessaging::FindWindowW;
unsafe {
let hwnd = FindWindowW(None, windows::core::w!("Punktfunk")).ok()?;
match GetDpiForWindow(hwnd) {
0 => None,
d => Some(d),
}
}
}
fn run(
presenter: SendPresenter,
frames: FrameRx,
shared: Arc<RenderShared>,
clock_offset_ns: Arc<AtomicI64>,
) {
let mut p = presenter.0;
let mut applied = (0u32, 0u32, 0u32); // last (w, h, dpi) handed to the presenter
let mut presented = 0u32;
let mut dropped = 0u32;
// 1 s tumbling windows: end-to-end (capture→displayed) and the display stage
// (decoded→displayed), sampled post-Present. Percentiles only (spec: stats-unification.md).
let mut e2e_us: Vec<u64> = Vec::with_capacity(256);
let mut display_us: Vec<u64> = Vec::with_capacity(256);
let mut window_start = Instant::now();
let mut last_dpi_poll = Instant::now();
PRESENT_FPS.store(0, Ordering::Relaxed);
PRESENT_SKIPPED.store(0, Ordering::Relaxed);
E2E_P50_US.store(0, Ordering::Relaxed);
E2E_P95_US.store(0, Ordering::Relaxed);
DISPLAY_P50_US.store(0, Ordering::Relaxed);
loop {
if shared.stop.load(Ordering::SeqCst) {
break;
}
let first = match frames.recv_timeout(Duration::from_millis(50)) {
Ok(f) => Some(f),
Err(RecvTimeoutError::Timeout) => None,
Err(RecvTimeoutError::Disconnected) => break,
};
if last_dpi_poll.elapsed() >= Duration::from_secs(1) {
last_dpi_poll = Instant::now();
if let Some(dpi) = poll_window_dpi() {
shared.set_dpi(dpi);
}
}
let snap = shared.snapshot();
let resized = snap != applied && snap.0 > 0 && snap.1 > 0;
if resized {
p.resize(snap.0, snap.1, snap.2);
applied = snap;
}
if first.is_none() && !resized {
continue; // nothing new to show — don't burn GPU re-presenting a static frame
}
// Throttle to the compositor: with ≤1 present outstanding this returns as DWM frees a
// slot, and frames decoded meanwhile are drained below so the newest is what's drawn.
if !p.wait_present_slot(1000) {
tracing::debug!("frame-latency waitable timed out — presenting anyway");
}
let mut newest = first;
while let Ok(f) = frames.try_recv() {
if newest.is_some() {
dropped += 1;
}
newest = Some(f);
}
// The session pump is the sole 0xCE consumer and stashes the latest here (rare updates).
if let Some(meta) = *crate::present::LATEST_HDR_META.lock().unwrap() {
p.set_hdr_metadata(meta);
}
let times: Option<FrameTimes> = newest.as_ref().map(|(_, t)| *t);
p.present(newest.map(|(f, _)| f));
presented += 1;
if let Some(t) = times {
// The `displayed` point: post-Present() on this thread (the honest best-effort
// presentation instant on Windows — endpoint label `capture→on-glass`).
let displayed_ns = now_ns();
// End-to-end = capture → displayed, host-clock corrected, measured directly
// (never the sum of stage percentiles). Clamped (0, 10 s).
let e2e = (displayed_ns as i128 + clock_offset_ns.load(Ordering::Relaxed) as i128
- t.pts_ns as i128)
.max(0) as u64;
if e2e > 0 && e2e < 10_000_000_000 {
e2e_us.push(e2e / 1000);
}
// `display` stage = decoded → displayed, single-clock client-local.
let disp = displayed_ns.saturating_sub(t.decoded_ns);
if disp < 10_000_000_000 {
display_us.push(disp / 1000);
}
}
if window_start.elapsed() >= Duration::from_secs(1) {
e2e_us.sort_unstable();
display_us.sort_unstable();
let p50 = |v: &[u64]| v.get(v.len() / 2).copied().unwrap_or(0);
// p95 = sorted[min(len*95/100, len-1)] — the empty-window case falls to 0 via `get`.
let p95 = |v: &[u64]| {
v.get((v.len() * 95 / 100).min(v.len().saturating_sub(1)))
.copied()
.unwrap_or(0)
};
tracing::debug!(
presented,
dropped,
e2e_p50_us = p50(&e2e_us),
e2e_p95_us = p95(&e2e_us),
display_p50_us = p50(&display_us),
"render window"
);
PRESENT_FPS.store(presented, Ordering::Relaxed);
PRESENT_SKIPPED.store(dropped, Ordering::Relaxed);
E2E_P50_US.store(p50(&e2e_us), Ordering::Relaxed);
E2E_P95_US.store(p95(&e2e_us), Ordering::Relaxed);
DISPLAY_P50_US.store(p50(&display_us), Ordering::Relaxed);
window_start = Instant::now();
presented = 0;
dropped = 0;
e2e_us.clear();
display_us.clear();
}
}
tracing::info!("render thread exiting");
}
-550
View File
@@ -1,550 +0,0 @@
//! Session controller: one worker thread runs connect → pump (video pull + decode, audio
//! pull + Opus decode, stats), feeding the UI over channels. The UI keeps the
//! `Arc<NativeClient>` from the `Connected` event for direct input sends (no extra hop on
//! the input path) — `NativeClient` is `Sync`, planes stay one-consumer-per-thread:
//! video+audio here, rumble+hidout on the gamepad thread.
//!
//! Ported from the GTK Linux client; the platform-specific pieces are the video decoder
//! (software-only here) and the audio backend (WASAPI). The pump body is identical.
use crate::audio;
use crate::video::{DecodedFrame, Decoder, DecoderPref};
use punktfunk_core::client::NativeClient;
use punktfunk_core::config::{CompositorPref, GamepadPref, Mode};
use punktfunk_core::PunktfunkError;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc;
use std::time::{Duration, Instant};
pub struct SessionParams {
pub host: String,
pub port: u16,
pub mode: Mode,
pub compositor: CompositorPref,
pub gamepad: GamepadPref,
pub bitrate_kbps: u32,
/// Requested audio channel count (2/6/8); the host echoes the resolved value.
pub audio_channels: u8,
/// Stream the default microphone to the host's virtual mic source.
pub mic_enabled: bool,
/// Advertise 10-bit + HDR10 so the host may upgrade HDR content to a Main10/PQ stream.
pub hdr_enabled: bool,
/// Which video decode backend to use (auto/hardware/software).
pub decoder: DecoderPref,
/// The user's preferred video codec (a `quic::CODEC_*` bit, `0` = auto). Soft — the host honors
/// it when it can emit it, else falls back; the resolved codec drives the decoder.
pub preferred_codec: u8,
/// Pinned host fingerprint; `None` = trust on first use (caller persists the observed one).
pub pin: Option<[u8; 32]>,
pub identity: (String, String),
/// How long to wait for the handshake. The normal path uses a short budget; the
/// "request access" (delegated-approval) path uses a long one, because the host PARKS the
/// connection until the operator clicks Approve in its console (so this must exceed the
/// host's approval window — see `PENDING_APPROVAL_WAIT`).
pub connect_timeout: Duration,
}
#[derive(Clone, Copy, Default, PartialEq)]
pub struct Stats {
/// AUs received (reassembled) per second — actual-elapsed-time denominator.
pub fps: f32,
/// Received payload goodput (excludes FEC overhead).
pub mbps: f32,
/// `decode` stage p50 over the last 1 s window: received → decoded, client-local clock.
pub decode_ms: f32,
/// `host+network` stage p50 over the last 1 s window: capture (`pts_ns`) → received,
/// host-clock corrected via `clock_offset_ns`.
pub hostnet_ms: f32,
/// `host` stage p50 (host capture→sent, from the per-AU 0xCF host-timing plane). Valid only
/// when `split` — an old host emits no 0xCF and the HUD keeps the combined stage.
pub host_ms: f32,
/// `network` stage p50 (`hostnet host`, tiled per frame before taking the percentile).
/// Valid only when `split`.
pub net_ms: f32,
/// True when any 0xCF host timings matched received AUs this window — the HUD then renders
/// `host + network` instead of the combined `host+network` term.
pub split: bool,
/// True when `clock_offset_ns == 0` (host didn't answer the skew handshake / same host) —
/// the HUD appends `(same-host clock)` to the end-to-end line.
pub same_host: bool,
/// True when decoding on the GPU (D3D11VA) vs. CPU (software).
pub hardware: bool,
/// True when the stream is BT.2020 PQ HDR10 (last decoded frame).
pub hdr: bool,
/// The negotiated wire codec (a `quic::CODEC_*` bit) — the HUD's codec chip.
pub codec: u8,
/// Frames lost to unrecoverable network drops since session start (reassembler count; each
/// triggers a keyframe re-request).
pub dropped: u64,
/// Seconds since the stream started.
pub uptime_secs: u32,
}
pub enum SessionEvent {
Connected {
connector: Arc<NativeClient>,
mode: Mode,
fingerprint: [u8; 32],
},
/// `trust_rejected` is set when the connect failed the TLS trust check (a `Crypto`
/// error): for a pinned connect this is the fingerprint-changed signal, so the UI can
/// offer a re-pair (PIN) path rather than a dead-end error.
Failed {
msg: String,
trust_rejected: bool,
},
Ended(Option<String>),
Stats(Stats),
}
/// Per-frame measurement points carried with a decoded frame to the render thread: the host
/// capture clock (`pts_ns`) and our local `decoded` stamp (wall-clock ns). Post-`Present()` the
/// render thread derives the `display` stage (displayed decoded, single-clock) and the
/// end-to-end headline (displayed + clock_offset pts) from them.
#[derive(Clone, Copy)]
pub struct FrameTimes {
pub pts_ns: u64,
pub decoded_ns: u64,
}
/// Decoded frames + their measurement points, session pump → render thread (crossbeam so that
/// thread can block with a timeout — async-channel has no `recv_timeout`).
pub type FrameRx = crossbeam_channel::Receiver<(DecodedFrame, FrameTimes)>;
pub struct SessionHandle {
pub events: async_channel::Receiver<SessionEvent>,
pub frames: FrameRx,
pub stop: Arc<AtomicBool>,
}
/// Blocking speed-test probe (the GUI's per-host "Test" and the `--headless --speed-test` CLI):
/// a minimal identified connect (720p60 — the host builds a virtual output, but nothing is
/// decoded), then `request_probe` (a 2 s burst up to the host's 3 Gbps ceiling) polled to
/// completion. Run on a worker thread.
pub fn run_speed_probe(
addr: &str,
port: u16,
fp_hex: Option<&str>,
identity: (String, String),
) -> Result<punktfunk_core::client::ProbeOutcome, String> {
// Pin the saved/advertised fingerprint when we have one; a manual host measures over TOFU.
let pin = fp_hex.and_then(crate::trust::parse_hex32);
let c = NativeClient::connect(
addr,
port,
Mode {
width: 1280,
height: 720,
refresh_hz: 60,
},
CompositorPref::Auto,
GamepadPref::Auto,
0, // bitrate_kbps: host default
0, // video_caps: probe connect, nothing is decoded
2, // audio_channels: stereo baseline
crate::video::decodable_codecs(),
0, // preferred_codec: no preference
None, // display_hdr: probe connect, nothing presents
None, // launch: no game
pin,
Some(identity),
Duration::from_secs(15),
)
.map_err(|e| format!("connect: {e:?}"))?;
c.request_probe(3_000_000, 2_000)
.map_err(|e| format!("probe: {e:?}"))?;
let deadline = Instant::now() + Duration::from_secs(10);
loop {
std::thread::sleep(Duration::from_millis(250));
if c.probe_result().done {
// Let the last UDP shards land before tearing down.
std::thread::sleep(Duration::from_millis(400));
return Ok(c.probe_result());
}
if Instant::now() > deadline {
return Err("probe timed out".to_string());
}
}
}
pub fn start(params: SessionParams) -> SessionHandle {
let (ev_tx, ev_rx) = async_channel::unbounded();
// Tiny frame queue, newest wins: the pump displaces the oldest when the renderer lags (it
// keeps a Receiver clone for exactly that).
let (frame_tx, frame_rx) = crossbeam_channel::bounded(2);
let stop = Arc::new(AtomicBool::new(false));
let stop_w = stop.clone();
let frame_rx_pump = frame_rx.clone();
std::thread::Builder::new()
.name("punktfunk-session".into())
.spawn(move || pump(params, ev_tx, frame_tx, frame_rx_pump, stop_w))
.expect("spawn session thread");
SessionHandle {
events: ev_rx,
frames: frame_rx,
stop,
}
}
fn now_ns() -> u64 {
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_nanos() as u64)
.unwrap_or(0)
}
/// Opus decoder for the audio plane: a plain stereo decoder (the validated path) or a multistream
/// decoder for 5.1/7.1, both behind one `decode_float`. Built from the host-RESOLVED channel count
/// via the shared layout table.
enum AudioDec {
Stereo(opus::Decoder),
Surround(opus::MSDecoder),
}
impl AudioDec {
fn new(channels: u8) -> Result<AudioDec, opus::Error> {
if channels == 2 {
Ok(AudioDec::Stereo(opus::Decoder::new(
48_000,
opus::Channels::Stereo,
)?))
} else {
let l = punktfunk_core::audio::layout_for(channels, false);
Ok(AudioDec::Surround(opus::MSDecoder::new(
48_000, l.streams, l.coupled, l.mapping,
)?))
}
}
fn decode_float(
&mut self,
input: &[u8],
out: &mut [f32],
fec: bool,
) -> Result<usize, opus::Error> {
match self {
AudioDec::Stereo(d) => d.decode_float(input, out, fec),
AudioDec::Surround(d) => d.decode_float(input, out, fec),
}
}
}
fn pump(
params: SessionParams,
ev_tx: async_channel::Sender<SessionEvent>,
frame_tx: crossbeam_channel::Sender<(DecodedFrame, FrameTimes)>,
frame_rx: FrameRx,
stop: Arc<AtomicBool>,
) {
// Advertise 10-bit + HDR10 only when the user enabled HDR AND a display is actually in HDR
// mode: the host then upgrades HDR content to a Main10/PQ stream (its own 10-bit gate still
// applies). On an SDR display we advertise `0` so the host sends a proper 8-bit BT.709 stream
// rather than PQ the panel would mis-tone-map (washed-out/dark). The presenter handles BT.2020
// PQ frames (P010 / X2BGR10).
let hdr_active = params.hdr_enabled && crate::present::display_supports_hdr();
if params.hdr_enabled && !hdr_active {
tracing::info!("HDR enabled in settings but no HDR display detected — requesting SDR");
}
// With HDR active, also report the panel's real colour volume (GetDesc1): the host writes it
// into its virtual display's EDID, so host apps tone-map to THIS panel and the PQ stream
// arrives already inside its volume — the client presents it untouched.
// PUNKTFUNK_CLIENT_PEAK_NITS pins a synthetic volume for A/B runs.
let display_hdr = if hdr_active {
let vol = punktfunk_core::client::display_hdr_env_override()
.or_else(crate::present::display_hdr_volume);
if let Some(m) = vol {
tracing::info!(
max_nits = m.max_display_mastering_luminance / 10_000,
min_millinits = m.min_display_mastering_luminance / 10,
max_fall = m.max_fall,
"advertising this display's HDR volume to the host"
);
}
vol
} else {
None
};
let connector = match NativeClient::connect(
&params.host,
params.port,
params.mode,
params.compositor,
params.gamepad,
params.bitrate_kbps,
if hdr_active {
punktfunk_core::quic::VIDEO_CAP_10BIT | punktfunk_core::quic::VIDEO_CAP_HDR
} else {
0
},
params.audio_channels,
crate::video::decodable_codecs(), // codecs FFmpeg can decode (HEVC/H.264/AV1)
params.preferred_codec, // the user's soft codec preference (0 = auto)
display_hdr,
None, // launch: the Windows client has no library picker yet
params.pin,
Some(params.identity),
params.connect_timeout,
) {
Ok(c) => Arc::new(c),
Err(e) => {
let trust_rejected = matches!(e, PunktfunkError::Crypto);
let msg = match e {
PunktfunkError::Crypto => {
"Host identity rejected — wrong fingerprint, or the host requires pairing"
.to_string()
}
PunktfunkError::Timeout => "Connection timed out".to_string(),
other => format!("Connect failed: {other:?}"),
};
let _ = ev_tx.send_blocking(SessionEvent::Failed {
msg,
trust_rejected,
});
return;
}
};
let _ = ev_tx.send_blocking(SessionEvent::Connected {
connector: connector.clone(),
mode: connector.mode(),
fingerprint: connector.host_fingerprint,
});
// Build the decoder for the codec the host resolved (never assume HEVC).
let codec_id = crate::video::ffmpeg_codec_id(connector.codec);
tracing::info!(
?codec_id,
welcome_codec = connector.codec,
"negotiated video codec"
);
let mut decoder = match Decoder::new(params.decoder, codec_id) {
Ok(d) => d,
Err(e) => {
let _ = ev_tx.send_blocking(SessionEvent::Ended(Some(format!("video decoder: {e}"))));
return;
}
};
let mut hardware = decoder.is_hardware();
let mut hdr = false;
// Audio is best-effort: a session without it still streams. Gamepads are the
// app-lifetime service's job (the UI attaches it on Connected). Build the decoder + playback
// from the host-RESOLVED channel count (never the request), so an older/clamping host that
// resolves stereo is decoded as stereo.
let channels = connector.audio_channels;
let player = audio::AudioPlayer::spawn(channels)
.map_err(|e| tracing::warn!(error = %e, "audio disabled"))
.ok();
let mut opus_dec = AudioDec::new(channels)
.map_err(|e| tracing::warn!(error = %e, "opus decoder failed — audio disabled"))
.ok();
let _mic = params
.mic_enabled
.then(|| {
audio::MicStreamer::spawn(connector.clone())
.map_err(|e| tracing::warn!(error = %e, "mic uplink disabled"))
.ok()
})
.flatten();
// Force an immediate IDR (with in-band parameter sets) rather than waiting for the host's own
// first keyframe — under infinite GOP a late/missed IDR means the decoder sits on
// "PPS id out of range" (a black screen) until one arrives.
let _ = connector.request_keyframe();
// Live host↔client clock offset: loaded per use (Relaxed) so mid-stream re-syncs (an NTP
// step, drift) keep the capture-clock latency stats honest — never cached at session start.
let clock_offset_live = connector.clock_offset_shared();
let mut total_frames = 0u64;
let session_start = Instant::now();
let mut window_start = Instant::now();
let mut frames_n = 0u32;
let mut bytes_n = 0u64;
// 1 s tumbling stage windows (spec: design/stats-unification.md — percentiles, never means).
let mut hostnet_us: Vec<u64> = Vec::with_capacity(256);
let mut decode_us: Vec<u64> = Vec::with_capacity(256);
// Host/network split (Phase 2): received AUs awaiting their 0xCF host timing, `(pts_ns,
// hostnet_us)`, matched as the datagrams arrive. Bounded — an old host never sends any.
let mut pending_split: std::collections::VecDeque<(u64, u64)> =
std::collections::VecDeque::with_capacity(256);
let mut host_us_w: Vec<u64> = Vec::with_capacity(256);
let mut net_us_w: Vec<u64> = Vec::with_capacity(256);
let mut pcm = vec![0f32; 5760 * channels as usize]; // scratch: max Opus frame (120 ms) × channels
// Loss recovery: watch the host→client unrecoverable-drop count and ask for an IDR when it climbs.
let mut last_dropped = connector.frames_dropped();
let mut last_kf_req: Option<Instant> = None;
let end: Option<String> = loop {
if stop.load(Ordering::SeqCst) {
break None;
}
match connector.next_frame(Duration::from_millis(4)) {
Ok(frame) => {
// The `received` point: AU fully reassembled, handed to us, before decode.
let received_ns = now_ns();
// fps = AUs received per second, Mb/s = received goodput (spec: counted at the
// received point, not the decoded one).
frames_n += 1;
bytes_n += frame.data.len() as u64;
// `host+network` stage: capture → received, host-clock corrected. Clamped (0, 10 s).
let clock_offset = clock_offset_live.load(Ordering::Relaxed);
let hostnet = (received_ns as i128 + clock_offset as i128 - frame.pts_ns as i128)
.max(0) as u64;
if hostnet > 0 && hostnet < 10_000_000_000 {
hostnet_us.push(hostnet / 1000);
// Remember this AU for the 0xCF match below (host/network split).
pending_split.push_back((frame.pts_ns, hostnet / 1000));
if pending_split.len() > 256 {
pending_split.pop_front();
}
}
// A D3D11VA→software demotion (see `Decoder::decode`) starts a FRESH decoder that
// has none of the stream's parameter sets; under infinite GOP it would sit on
// "PPS id out of range" forever. Detect the transition and force a new IDR so the
// rebuilt decoder resynchronizes immediately.
let was_hw = decoder.is_hardware();
let decoded = decoder.decode(&frame.data);
if was_hw && !decoder.is_hardware() {
tracing::info!("decoder demoted to software — requesting keyframe to resync");
let _ = connector.request_keyframe();
}
match decoded {
Ok(Some(decoded)) => {
// The `decoded` point: decoder output frame available.
let decoded_ns = now_ns();
total_frames += 1;
hdr = decoded.hdr();
// The backend can demote D3D11VA → software mid-session on a hardware error.
hardware = decoder.is_hardware();
if total_frames == 1 {
let (w, h) = decoded.dims();
tracing::info!(
width = w,
height = h,
path = if hardware { "d3d11va" } else { "software" },
hdr,
"first frame decoded"
);
}
// `decode` stage: received → decoded, single-clock client-local.
decode_us.push(decoded_ns.saturating_sub(received_ns) / 1000);
// Newest wins: displace the oldest queued frame when the renderer lags.
if let Err(crossbeam_channel::TrySendError::Full(item)) =
frame_tx.try_send((
decoded,
FrameTimes {
pts_ns: frame.pts_ns,
decoded_ns,
},
))
{
let _ = frame_rx.try_recv();
let _ = frame_tx.try_send(item);
}
}
Ok(None) => {}
// Survivable (loss until the next IDR/RFI recovery) — keep feeding.
Err(e) => tracing::debug!(error = %e, "decode error (recovering)"),
}
}
Err(PunktfunkError::NoFrame) => {}
Err(PunktfunkError::Closed) => break Some("Host ended the session".to_string()),
Err(e) => break Some(format!("session: {e:?}")),
}
// Loss recovery: under infinite GOP the only recovery keyframe is one we request. The
// reassembler drops unrecoverable AUs (frames_dropped); the decoder conceals the
// reference-missing delta frames that follow and returns Ok, so keying off a decode error
// rarely fires. Request an IDR when the drop count climbs, throttled.
let dropped = connector.frames_dropped();
if dropped > last_dropped {
last_dropped = dropped;
let now = Instant::now();
if last_kf_req.is_none_or(|t| now.duration_since(t) >= Duration::from_millis(100)) {
last_kf_req = Some(now);
let _ = connector.request_keyframe();
tracing::debug!(dropped, "requested keyframe (loss recovery)");
}
}
// Drain audio between frames (packets land every 5 ms; the queue holds 320 ms).
while let Ok(pkt) = connector.next_audio(Duration::ZERO) {
if let (Some(player), Some(dec)) = (&player, opus_dec.as_mut()) {
match dec.decode_float(&pkt.data, &mut pcm, false) {
// `samples` is per-channel; the interleaved frame is `samples * channels`.
Ok(samples) => player.push(pcm[..samples * channels as usize].to_vec()),
Err(e) => tracing::debug!(error = %e, "opus decode"),
}
}
}
// Drain the HDR static-metadata plane (0xCE): the source's real mastering display + content
// light level. Stash the latest for the UI-thread presenter to apply via SetHDRMetaData —
// this pump is the sole consumer of the plane. Rare (start + on change/keyframe).
while let Ok(meta) = connector.next_hdr_meta(Duration::ZERO) {
*crate::present::LATEST_HDR_META.lock().unwrap() = Some(meta);
}
// Drain the per-AU host-timing plane (0xCF) and match by pts: `host` = the host's own
// capture→sent, `network` = our capture→received minus it — the two tile per frame
// (design/stats-unification.md Phase 2). An old host never emits any; `split` stays false
// and the HUD keeps the combined `host+network` stage.
while let Ok(t) = connector.next_host_timing(Duration::ZERO) {
if let Some(i) = pending_split.iter().position(|(p, _)| *p == t.pts_ns) {
let (_, hn_us) = pending_split.remove(i).unwrap();
host_us_w.push(t.host_us as u64);
net_us_w.push(hn_us.saturating_sub(t.host_us as u64));
}
}
if window_start.elapsed() >= Duration::from_secs(1) {
let secs = window_start.elapsed().as_secs_f32();
hostnet_us.sort_unstable();
decode_us.sort_unstable();
host_us_w.sort_unstable();
net_us_w.sort_unstable();
let p50 = |v: &[u64]| v.get(v.len() / 2).copied().unwrap_or(0);
let (hostnet_p50, decode_p50) = (p50(&hostnet_us), p50(&decode_us));
let (host_p50, net_p50) = (p50(&host_us_w), p50(&net_us_w));
let split = !host_us_w.is_empty();
tracing::debug!(
fps = frames_n,
hostnet_p50_us = hostnet_p50,
host_p50_us = host_p50,
net_p50_us = net_p50,
split,
decode_p50_us = decode_p50,
total_frames,
"stream window"
);
let _ = ev_tx.try_send(SessionEvent::Stats(Stats {
fps: frames_n as f32 / secs,
mbps: bytes_n as f32 * 8.0 / 1e6 / secs,
decode_ms: decode_p50 as f32 / 1000.0,
hostnet_ms: hostnet_p50 as f32 / 1000.0,
host_ms: host_p50 as f32 / 1000.0,
net_ms: net_p50 as f32 / 1000.0,
split,
same_host: clock_offset_live.load(Ordering::Relaxed) == 0,
hardware,
hdr,
codec: connector.codec,
dropped: last_dropped,
uptime_secs: session_start.elapsed().as_secs() as u32,
}));
window_start = Instant::now();
frames_n = 0;
bytes_n = 0;
hostnet_us.clear();
decode_us.clear();
host_us_w.clear();
net_us_w.clear();
}
};
tracing::info!(
total_frames,
reason = end.as_deref().unwrap_or("user"),
"session ended"
);
stop.store(true, Ordering::SeqCst);
let _ = ev_tx.send_blocking(SessionEvent::Ended(end));
}
-4
View File
@@ -6,10 +6,6 @@
//! [`SpawnEvent`]s a reader thread hands to the app's navigation closure: spinner until //! [`SpawnEvent`]s a reader thread hands to the app's navigation closure: spinner until
//! `{"ready":true}`, banner from the `{"error"|"ended": …}` line, `trust_rejected` //! `{"ready":true}`, banner from the `{"error"|"ended": …}` line, `trust_rejected`
//! routed to the re-pair PIN ceremony, `stats:` lines to the session status page. //! routed to the re-pair PIN ceremony, `stats:` lines to the session status page.
//!
//! The legacy in-process D3D11VA presenter remains reachable via the
//! `PUNKTFUNK_BUILTIN_STREAM=1` env override (`app::use_builtin_stream`) — the
//! developer A/B baseline until its deletion.
use std::io::BufRead as _; use std::io::BufRead as _;
use std::process::{Child, Command, Stdio}; use std::process::{Child, Command, Stdio};
+2 -4
View File
@@ -5,10 +5,8 @@
//! //!
//! The shell is the settings file's only writer; the session only reads it. The shell's //! The shell is the settings file's only writer; the session only reads it. The shell's
//! former private `Settings` copy (≤ 0.8.4: `show_hud`, `engine`) is gone — old files //! former private `Settings` copy (≤ 0.8.4: `show_hud`, `engine`) is gone — old files
//! still load via a serde alias in core, and the legacy in-process presenter is now //! still load via a serde alias in core.
//! reachable only through `PUNKTFUNK_BUILTIN_STREAM=1` (see `app::use_builtin_stream`).
pub use pf_client_core::trust::{ pub use pf_client_core::trust::{
hex, learn_mac, load_or_create_identity, parse_hex32, touch_last_used, KnownHost, KnownHosts, hex, learn_mac, load_or_create_identity, parse_hex32, KnownHost, KnownHosts, Settings,
Settings,
}; };
-653
View File
@@ -1,653 +0,0 @@
//! Video decode: reassembled HEVC access units → frames for the D3D11 presenter.
//!
//! Two backends, picked at session start (override via [`DecoderPref`] / the Settings UI):
//!
//! * **D3D11VA** (any GPU — the vendor-agnostic DXVA path on NVIDIA/AMD/Intel): libavcodec decodes
//! on the GPU into an `ID3D11Texture2D` decode array (decoder-only bind — NVIDIA rejects a
//! decoder array that is also a shader resource). The presenter copies each decoded slice into
//! its own sampleable NV12/P010 texture and converts YUV→RGB in a shader — one cheap GPU-to-GPU
//! copy per frame (no swscale, no CPU readback). The decode array is created by the process-wide
//! shared device ([`crate::gpu`]) the presenter also draws with, so the copy stays on-GPU. This
//! is the big latency/throughput win over software.
//! * **Software**: libavcodec on the CPU + swscale to the same planar layout the hardware path
//! produces (NV12, or P010 for 10-bit) — the presenter uploads the two planes and runs the SAME
//! YUV→RGB shaders, so hw/sw color math is identical. The fallback on a GPU-less box (WARP),
//! when D3D11VA init fails, or when a mid-session hardware error demotes us — the host's
//! IDR/RFI recovery resynchronizes on the next keyframe either way.
//!
//! D3D11VA viability is settled **before the session's first frame** by two probes: the adapter
//! must expose the negotiated codec's DXVA decode profile ([`decode_profile_supported`] — hwaccel
//! init otherwise only fails at the first AU, burning the IDR), and it must be able to create the
//! decode surface pool ([`d3d11va_decode_supported`]). Either failing commits to software decode
//! from frame one (a clean, gap-free stream) instead of dying mid-stream.
//!
//! Both run `AV_CODEC_FLAG_LOW_DELAY`; the host encodes zero-reorder streams (no B-frames, in-band
//! parameter sets on every IDR), so decode is strictly one-in/one-out.
//!
//! HDR is detected in-band from the decoded frame's transfer characteristic (`SMPTE2084` / PQ in the
//! HEVC VUI) — the same signal every other punktfunk client keys off — not from a protocol field.
use anyhow::{anyhow, bail, Context as _, Result};
use ffmpeg::format::Pixel;
use ffmpeg::software::scaling;
use ffmpeg::util::frame::Video as AvFrame;
use ffmpeg_next as ffmpeg;
use pf_client_core::video::ColorDesc;
use std::ffi::c_void;
use std::ptr;
use windows::core::{Interface, GUID};
use windows::Win32::Graphics::Direct3D11::{ID3D11Device, ID3D11VideoDevice};
use windows::Win32::Graphics::Dxgi::Common::{DXGI_FORMAT, DXGI_FORMAT_NV12, DXGI_FORMAT_P010};
/// Which decode backend to use; the Settings UI persists this as a string.
#[derive(Clone, Copy, PartialEq, Eq, Debug, Default)]
pub enum DecoderPref {
/// Try D3D11VA, fall back to software.
#[default]
Auto,
/// Force D3D11VA (error out if unavailable, for debugging).
Hardware,
/// Force software decode.
Software,
}
impl DecoderPref {
pub fn from_name(s: &str) -> DecoderPref {
match s {
"hardware" => DecoderPref::Hardware,
"software" => DecoderPref::Software,
_ => DecoderPref::Auto,
}
}
}
pub enum DecodedFrame {
Cpu(CpuFrame),
Gpu(GpuFrame),
}
impl DecodedFrame {
pub fn dims(&self) -> (u32, u32) {
match self {
DecodedFrame::Cpu(c) => (c.width, c.height),
DecodedFrame::Gpu(g) => (g.width, g.height),
}
}
pub fn hdr(&self) -> bool {
match self {
DecodedFrame::Cpu(c) => c.hdr,
DecodedFrame::Gpu(g) => g.hdr,
}
}
}
/// A software-decoded frame in the same planar layout the hardware path produces: an NV12 (or
/// P010 for 10-bit) luma plane + interleaved chroma plane, each with its swscale row stride
/// (≥ the row bytes — swscale pads rows for SIMD). The presenter uploads them into two dynamic
/// plane textures sampled by the same shaders as the D3D11VA path.
pub struct CpuFrame {
pub width: u32,
pub height: u32,
/// Luma plane (`W×H` samples, 1 byte each; 2 for 10-bit) + its row stride in bytes.
pub y: Vec<u8>,
pub y_stride: usize,
/// Interleaved chroma plane (`⌈W/2⌉×⌈H/2⌉` UV pairs) + its row stride in bytes.
pub uv: Vec<u8>,
pub uv_stride: usize,
/// P010 sample layout (10 bits in the high bits of 16) vs NV12. Selects texture/SRV formats.
pub ten_bit: bool,
/// BT.2020 PQ HDR10 vs ordinary BT.709 SDR. Selects the swapchain colour space.
pub hdr: bool,
/// The frame's CICP signaling (HEVC VUI → `AVFrame`), read per-frame — the presenter derives
/// its YCbCr→RGB constant buffer from it (`csc_rows`), so a BT.601-signaled stream (a Linux
/// host's RGB-input NVENC) no longer renders with BT.709 coefficients.
pub color: ColorDesc,
}
/// A decoded frame still on the GPU: a D3D11 texture **array** plus the slice index the decoder
/// wrote this frame into. The presenter copies the slice into its own sampleable texture and
/// converts YUV→RGB in a pixel shader. The underlying surface stays alive — and out of the decoder's
/// reuse pool — for exactly as long as `guard` (an `av_frame_clone` of the decoded frame) lives.
pub struct GpuFrame {
pub width: u32,
pub height: u32,
/// Texture-array slice this frame occupies (`AVFrame::data[1]`).
pub index: u32,
/// The decode pool is P010 (10 bits in the high bits) vs NV12 — from the frames context's
/// `sw_format`. The presenter keys its copy-texture/SRV formats off this: they must match the
/// source array exactly for `CopySubresourceRegion`.
pub ten_bit: bool,
/// BT.2020 PQ HDR10 (ST.2084 transfer) vs ordinary BT.709 SDR. Selects the swapchain colour
/// space only (the host couples 10-bit ⟺ HDR today, but formats key off `ten_bit`).
pub hdr: bool,
/// Per-frame CICP signaling — see [`CpuFrame::color`].
pub color: ColorDesc,
guard: D3d11FrameGuard,
}
impl GpuFrame {
/// The decoder's D3D11 texture array holding this frame's slice, borrowed from the live cloned
/// `AVFrame`. Construct the windows-rs interface on the thread that will use it (the render
/// thread): COM interfaces are `!Send`, but the raw pointer is fine to carry across threads.
pub fn texture_ptr(&self) -> *mut c_void {
unsafe { (*self.guard.0).data[0] as *mut c_void }
}
}
/// Owns a cloned decoded `AVFrame` (which refs the D3D11 surface in the decoder pool). Dropping it
/// releases the surface back for reuse. The clone is plain refcounted data; freeing it from the
/// render thread is fine.
pub struct D3d11FrameGuard(*mut ffmpeg::ffi::AVFrame);
unsafe impl Send for D3d11FrameGuard {}
impl Drop for D3d11FrameGuard {
fn drop(&mut self) {
unsafe { ffmpeg::ffi::av_frame_free(&mut self.0) };
}
}
enum Backend {
D3d11va(D3d11vaDecoder),
Software(SoftwareDecoder),
}
pub struct Decoder {
backend: Backend,
/// The negotiated codec, so a mid-session D3D11VA→software demotion rebuilds for the same codec.
codec_id: ffmpeg::codec::Id,
}
/// Map a negotiated `quic` codec bit to the FFmpeg decoder id the client opens.
pub fn ffmpeg_codec_id(wire: u8) -> ffmpeg::codec::Id {
match wire {
punktfunk_core::quic::CODEC_H264 => ffmpeg::codec::Id::H264,
punktfunk_core::quic::CODEC_AV1 => ffmpeg::codec::Id::AV1,
_ => ffmpeg::codec::Id::HEVC,
}
}
/// The `quic` codec bitfield this client can decode — whatever FFmpeg has a decoder for (HEVC/H.264
/// always; AV1 when built in). Advertised to the host so it never emits a codec we can't decode.
/// Deliberately NOT gated on the DXVA profiles: software decode covers anything FFmpeg can.
pub fn decodable_codecs() -> u8 {
let _ = ffmpeg::init();
let mut bits = 0u8;
for (id, bit) in [
(ffmpeg::codec::Id::HEVC, punktfunk_core::quic::CODEC_HEVC),
(ffmpeg::codec::Id::H264, punktfunk_core::quic::CODEC_H264),
(ffmpeg::codec::Id::AV1, punktfunk_core::quic::CODEC_AV1),
] {
if ffmpeg::decoder::find(id).is_some() {
bits |= bit;
}
}
bits
}
impl Decoder {
pub fn new(pref: DecoderPref, codec_id: ffmpeg::codec::Id) -> Result<Decoder> {
ffmpeg::init().context("ffmpeg init")?;
if pref != DecoderPref::Software {
match D3d11vaDecoder::new(codec_id) {
Ok(d) => {
tracing::info!(?codec_id, "D3D11VA hardware decode active");
return Ok(Decoder {
backend: Backend::D3d11va(d),
codec_id,
});
}
Err(e) => {
if pref == DecoderPref::Hardware {
return Err(e.context("decoder=hardware but D3D11VA failed"));
}
tracing::info!(reason = %e, "D3D11VA unavailable — software decode");
}
}
}
Ok(Decoder {
backend: Backend::Software(SoftwareDecoder::new(codec_id)?),
codec_id,
})
}
/// True for the GPU hardware backend (shown in the stream HUD).
pub fn is_hardware(&self) -> bool {
matches!(self.backend, Backend::D3d11va(_))
}
/// Feed one access unit; returns the decoded frame (the host's streams are one-in/one-out). A
/// software decode error after packet loss is survivable — keep feeding. A D3D11VA error demotes
/// to software for the rest of the session (the next IDR resynchronizes).
pub fn decode(&mut self, au: &[u8]) -> Result<Option<DecodedFrame>> {
match &mut self.backend {
Backend::D3d11va(d) => match d.decode(au) {
Ok(f) => Ok(f.map(DecodedFrame::Gpu)),
Err(e) => {
tracing::warn!(error = %e, "D3D11VA decode failed — falling back to software");
self.backend = Backend::Software(SoftwareDecoder::new(self.codec_id)?);
Ok(None)
}
},
Backend::Software(s) => Ok(s.decode(au)?.map(DecodedFrame::Cpu)),
}
}
}
// --- DXVA decode-profile probe --------------------------------------------------------
/// DXVA decode-profile GUIDs (`dxva.h`), defined locally so no extra windows-rs feature or
/// metadata surface is pulled in for four constants.
const PROFILE_H264_VLD_NOFGT: GUID = GUID::from_u128(0x1b81be68_a0c7_11d3_b984_00c04f2e73c5);
const PROFILE_HEVC_VLD_MAIN: GUID = GUID::from_u128(0x5b11d51b_2f4c_4452_bcc3_09f2a1160cc0);
const PROFILE_HEVC_VLD_MAIN10: GUID = GUID::from_u128(0x107af0e0_ef1a_4d19_aba8_67a163073d13);
const PROFILE_AV1_VLD_PROFILE0: GUID = GUID::from_u128(0xb8be4ccb_cf53_46ba_8d59_d6b8a6da5d2a);
/// Does the shared device's adapter expose a DXVA decode profile for `codec_id`? Checked before
/// building the FFmpeg hwdevice because hwaccel selection (`get_format`) only runs on the FIRST
/// access unit — an unsupported profile would otherwise burn the opening IDR and recover through
/// the mid-stream demotion path instead of committing to software up front. Also logs (once) the
/// adapter's full profile list plus Main10 availability — the forensics for a new GPU/driver.
fn decode_profile_supported(device: &ID3D11Device, codec_id: ffmpeg::codec::Id) -> Result<()> {
let video: ID3D11VideoDevice = device
.cast()
.context("device lacks ID3D11VideoDevice (created without VIDEO_SUPPORT)")?;
let profiles: Vec<GUID> = unsafe {
let n = video.GetVideoDecoderProfileCount();
(0..n)
.filter_map(|i| video.GetVideoDecoderProfile(i).ok())
.collect()
};
log_profiles_once(&profiles);
let (wanted, format, name): (GUID, DXGI_FORMAT, &str) = match codec_id {
ffmpeg::codec::Id::H264 => (PROFILE_H264_VLD_NOFGT, DXGI_FORMAT_NV12, "H.264 VLD NoFGT"),
ffmpeg::codec::Id::HEVC => (PROFILE_HEVC_VLD_MAIN, DXGI_FORMAT_NV12, "HEVC Main"),
ffmpeg::codec::Id::AV1 => (PROFILE_AV1_VLD_PROFILE0, DXGI_FORMAT_NV12, "AV1 Profile 0"),
other => bail!("no DXVA profile known for {other:?}"),
};
let ok = profiles.contains(&wanted)
&& unsafe { video.CheckVideoDecoderFormat(&wanted, format) }
.map(|b| b.as_bool())
.unwrap_or(false);
if !ok {
bail!("adapter exposes no {name} decode profile");
}
// 10-bit (a mid-session HDR upgrade needs Main10): informational — if it's missing the
// decode error → software demotion + keyframe re-request path covers the switch.
if codec_id == ffmpeg::codec::Id::HEVC {
let main10 = profiles.contains(&PROFILE_HEVC_VLD_MAIN10)
&& unsafe { video.CheckVideoDecoderFormat(&PROFILE_HEVC_VLD_MAIN10, DXGI_FORMAT_P010) }
.map(|b| b.as_bool())
.unwrap_or(false);
tracing::info!(main10, "HEVC Main10 (10-bit/HDR) decode profile");
}
Ok(())
}
/// One-time dump of the adapter's DXVA decode profiles.
fn log_profiles_once(profiles: &[GUID]) {
use std::sync::atomic::{AtomicBool, Ordering};
static ONCE: AtomicBool = AtomicBool::new(true);
if ONCE.swap(false, Ordering::Relaxed) {
let list: Vec<String> = profiles.iter().map(|g| format!("{g:?}")).collect();
tracing::info!(count = profiles.len(), profiles = ?list, "adapter DXVA decode profiles");
}
}
// --- software backend ---------------------------------------------------------------
struct SoftwareDecoder {
decoder: ffmpeg::decoder::Video,
/// Rebuilt whenever the decoded format/size **or output format** changes (mid-stream
/// `Reconfigure`, or an 8↔10-bit flip): `(ctx, src_fmt, w, h, dst_fmt)`.
sws: Option<(scaling::Context, Pixel, u32, u32, Pixel)>,
}
impl SoftwareDecoder {
fn new(codec_id: ffmpeg::codec::Id) -> Result<SoftwareDecoder> {
let codec = ffmpeg::decoder::find(codec_id)
.ok_or_else(|| anyhow!("no {codec_id:?} decoder in libavcodec"))?;
let mut ctx = ffmpeg::codec::Context::new_with_codec(codec);
unsafe {
let raw = ctx.as_mut_ptr();
(*raw).flags |= ffmpeg::ffi::AV_CODEC_FLAG_LOW_DELAY as i32;
// Slice threading adds no frame delay (frame threading adds thread_count-1).
(*raw).thread_type = ffmpeg::ffi::FF_THREAD_SLICE;
(*raw).thread_count = 0; // auto
}
let decoder = ctx.decoder().video().context("open video decoder")?;
Ok(SoftwareDecoder { decoder, sws: None })
}
fn decode(&mut self, au: &[u8]) -> Result<Option<CpuFrame>> {
let packet = ffmpeg::Packet::copy(au);
self.decoder
.send_packet(&packet)
.map_err(|e| anyhow!("send_packet: {e}"))?;
let mut frame = AvFrame::empty();
let mut out = None;
while self.decoder.receive_frame(&mut frame).is_ok() {
out = Some(self.convert(&frame)?);
}
Ok(out)
}
/// Convert the decoded planar YUV to the hardware path's layout: NV12 for 8-bit, P010 for
/// 10-bit — a chroma interleave (and 10→16-high-bits shift), NOT a colour conversion. The
/// matrix/range/transfer handling all lives in the presenter's shaders, shared with the
/// D3D11VA path, so software frames are bit-comparable with hardware ones.
fn convert(&mut self, frame: &AvFrame) -> Result<CpuFrame> {
let (fmt, w, h) = (frame.format(), frame.width(), frame.height());
// SAFETY: `frame` wraps a live decoded AVFrame for the duration of this call.
let color = unsafe { ColorDesc::from_raw(frame.as_ptr()) };
let hdr = color.is_pq();
// Source bit depth from the pix-fmt descriptor (stable FFmpeg public API).
let ten_bit = unsafe {
let desc = ffmpeg::ffi::av_pix_fmt_desc_get(fmt.into());
!desc.is_null() && (*desc).comp[0].depth > 8
};
let dst = if ten_bit { Pixel::P010LE } else { Pixel::NV12 };
let rebuild = !matches!(&self.sws, Some((_, f, sw, sh, d)) if *f == fmt && *sw == w && *sh == h && *d == dst);
if rebuild {
let ctx = scaling::Context::get(fmt, w, h, dst, w, h, scaling::Flags::POINT)
.context("swscale context")?;
self.sws = Some((ctx, fmt, w, h, dst));
}
let (sws, ..) = self.sws.as_mut().unwrap();
let mut conv = AvFrame::empty();
sws.run(frame, &mut conv).map_err(|e| anyhow!("sws: {e}"))?;
Ok(CpuFrame {
width: w,
height: h,
y: conv.data(0).to_vec(),
y_stride: conv.stride(0),
uv: conv.data(1).to_vec(),
uv_stride: conv.stride(1),
ten_bit,
hdr,
color,
})
}
}
// --- D3D11VA backend ------------------------------------------------------------------
//
// Raw FFI: ffmpeg-next has no hwaccel wrappers. The COM-typed hwcontext structs are declared here
// (stable FFmpeg public ABI) rather than relied on from ffmpeg-sys bindgen — the generic
// AVHWDeviceContext / AVHWFramesContext (whose payload is an opaque `void *hwctx`) come from
// ffmpeg-sys, and we cast `hwctx` to the structs below. All owned pointers are freed in Drop;
// decoded surfaces transfer out through D3d11FrameGuard.
const AVERROR_EAGAIN: i32 = -11; // -EAGAIN
/// D3D11VA decode surface pool depth: the zero-reorder DPB (12 refs) + the bounded decoded channel
/// (2) + the frame the presenter currently holds (until its copy flushes) + one in-flight decode —
/// 12 is comfortable. A GPU that can't create the pool at all is gated out by
/// `d3d11va_decode_supported` and the session uses software decode.
const DECODE_POOL_SIZE: i32 = 12;
/// `hwcontext_d3d11va.h` — `AVHWDeviceContext::hwctx`. Leaving `lock` null makes FFmpeg install an
/// `ID3D11Multithread` default lock + set multithread protection on `device_context` during init,
/// which is what lets the presenter share this device's immediate context from the render thread.
#[repr(C)]
struct AVD3D11VADeviceContext {
device: *mut c_void, // ID3D11Device*
device_context: *mut c_void, // ID3D11DeviceContext*
video_device: *mut c_void, // ID3D11VideoDevice*
video_context: *mut c_void, // ID3D11VideoContext*
lock: *mut c_void, // void (*)(void*)
unlock: *mut c_void, // void (*)(void*)
lock_ctx: *mut c_void,
}
/// `hwcontext_d3d11va.h` — `AVHWFramesContext::hwctx`. The header is explicit: "The user must at
/// least set D3D11_BIND_DECODER if the frames context is to be used for video decoding" — a
/// user-built frames context gets NO default (BindFlags 0 → `CreateTexture2D` E_INVALIDARG); the
/// automatic OR-in lives only in libavcodec's own frames-param path, which we bypass.
#[repr(C)]
struct AVD3D11VAFramesContext {
texture: *mut c_void, // ID3D11Texture2D* (null → FFmpeg allocates the pool)
bind_flags: u32, // UINT BindFlags
misc_flags: u32, // UINT MiscFlags
texture_infos: *mut c_void, // AVD3D11FrameDescriptor* (FFmpeg-managed)
}
/// `D3D11_BIND_DECODER` — the decode pool's ONLY bind flag. Adding `D3D11_BIND_SHADER_RESOURCE`
/// is what NVIDIA rejects on a decoder texture ARRAY; the presenter samples via its own copy.
const BIND_DECODER: u32 = 0x200;
fn averr(what: &str, code: i32) -> anyhow::Error {
anyhow!("{what}: {}", ffmpeg::Error::from(code))
}
/// libavcodec's `get_format` callback: pick the D3D11 hw surface format and nothing else.
/// Deliberately does NOT build a frames context — with `hw_device_ctx` set and `hw_frames_ctx`
/// left null, libavcodec derives the decode pool itself (`ff_decode_get_hw_frames_ctx`), applying
/// every vendor quirk: DXVA surface alignment (128 for HEVC/AV1), DPB-based pool sizing, and the
/// decoder-only `D3D11_BIND_DECODER` flags. A hand-built context validated on NVIDIA was rejected
/// by Intel at the first `SubmitDecoderBuffers` (E_INVALIDARG) — the vendor-proof path is the one
/// the ffmpeg CLI/mpv ship. Returning anything but `AV_PIX_FMT_D3D11` aborts hardware decode →
/// the session demotes to software.
unsafe extern "C" fn get_format_d3d11(
avctx: *mut ffmpeg::ffi::AVCodecContext,
mut list: *const ffmpeg::ffi::AVPixelFormat,
) -> ffmpeg::ffi::AVPixelFormat {
use ffmpeg::ffi::*;
unsafe {
if (*avctx).hw_device_ctx.is_null() {
return AVPixelFormat::AV_PIX_FMT_NONE;
}
while *list != AVPixelFormat::AV_PIX_FMT_NONE {
if *list == AVPixelFormat::AV_PIX_FMT_D3D11 {
return AVPixelFormat::AV_PIX_FMT_D3D11;
}
list = list.add(1);
}
AVPixelFormat::AV_PIX_FMT_NONE
}
}
/// Predict whether D3D11VA decode will work by doing EXACTLY what the decoder's `get_format` does —
/// allocate an `AVHWFramesContext` (decoder-only pool, no shader-resource bind) and initialize it,
/// which creates the real NV12 decode surface array. On a GPU/driver that can't create the pool this
/// fails here, up front, so the session commits to software decode from the first frame (a clean,
/// gap-free stream) rather than decoding the IDR then dying mid-stream on a texture error that a
/// software demotion can't reliably recover from (the host's infinite GOP won't re-send an IDR).
unsafe fn d3d11va_decode_supported(hw_device: *mut ffmpeg::ffi::AVBufferRef) -> bool {
use ffmpeg::ffi::*;
unsafe {
let frames_ref = av_hwframe_ctx_alloc(hw_device);
if frames_ref.is_null() {
return false;
}
let frames = (*frames_ref).data as *mut AVHWFramesContext;
(*frames).format = AVPixelFormat::AV_PIX_FMT_D3D11;
(*frames).sw_format = AVPixelFormat::AV_PIX_FMT_NV12;
(*frames).width = 1920;
(*frames).height = 1152; // 128-aligned 1080p surface (the HEVC DXVA alignment, see get_format)
(*frames).initial_pool_size = DECODE_POOL_SIZE;
// Decoder-only — matches get_format exactly.
let fhw = (*frames).hwctx as *mut AVD3D11VAFramesContext;
(*fhw).bind_flags = BIND_DECODER;
let r = av_hwframe_ctx_init(frames_ref);
let mut fr = frames_ref;
av_buffer_unref(&mut fr);
r >= 0
}
}
struct D3d11vaDecoder {
ctx: *mut ffmpeg::ffi::AVCodecContext,
hw_device: *mut ffmpeg::ffi::AVBufferRef,
packet: *mut ffmpeg::ffi::AVPacket,
frame: *mut ffmpeg::ffi::AVFrame,
}
// Single-owner pointers, only touched from the session pump thread.
unsafe impl Send for D3d11vaDecoder {}
impl D3d11vaDecoder {
fn new(codec_id: ffmpeg::codec::Id) -> Result<D3d11vaDecoder> {
use ffmpeg::ffi;
let shared = crate::gpu::shared().ok_or_else(|| anyhow!("no shared D3D11 device"))?;
if !shared.hardware {
bail!("shared device is WARP (no hardware video decode)");
}
// The adapter must expose the codec's DXVA profile — checked here, not at the first AU.
decode_profile_supported(&shared.device, codec_id)?;
unsafe {
// Build a D3D11VA hwdevice context around the *shared* device, so decoded textures live
// on the same device the presenter samples + draws with.
let hw_device =
ffi::av_hwdevice_ctx_alloc(ffi::AVHWDeviceType::AV_HWDEVICE_TYPE_D3D11VA);
if hw_device.is_null() {
bail!("av_hwdevice_ctx_alloc(D3D11VA) failed");
}
let devctx = (*hw_device).data as *mut ffi::AVHWDeviceContext;
let d3dctx = (*devctx).hwctx as *mut AVD3D11VADeviceContext;
// Hand FFmpeg an owned ref to the device + immediate context (it Releases them when the
// hwdevice ctx is freed). `into_raw()` transfers a +1 ref without releasing.
(*d3dctx).device = shared.device.clone().into_raw();
(*d3dctx).device_context = shared.context.clone().into_raw();
// lock left null → FFmpeg installs the ID3D11Multithread default lock in init.
let r = ffi::av_hwdevice_ctx_init(hw_device);
if r < 0 {
let mut hw = hw_device;
ffi::av_buffer_unref(&mut hw);
bail!("av_hwdevice_ctx_init: {}", ffmpeg::Error::from(r));
}
// Up-front viability probe (see `d3d11va_decode_supported`): a GPU/driver that can't
// create the decode surface pool commits to software NOW, so it decodes cleanly from the
// first frame instead of failing mid-stream (which a demotion can't reliably recover).
if !d3d11va_decode_supported(hw_device) {
let mut hw = hw_device;
ffi::av_buffer_unref(&mut hw);
bail!("GPU can't create the D3D11VA decode surface pool — using software decode");
}
let codec = ffi::avcodec_find_decoder(codec_id.into());
if codec.is_null() {
let mut hw = hw_device;
ffi::av_buffer_unref(&mut hw);
bail!("no {codec_id:?} decoder");
}
let ctx = ffi::avcodec_alloc_context3(codec);
(*ctx).hw_device_ctx = ffi::av_buffer_ref(hw_device);
(*ctx).get_format = Some(get_format_d3d11);
(*ctx).flags |= ffi::AV_CODEC_FLAG_LOW_DELAY as i32;
// hwaccel: threads only add latency.
(*ctx).thread_count = 1;
// On top of the DPB-based pool libavcodec sizes for us: the bounded decoded channel
// (2) + the frame the presenter holds until its copy flushes + margin.
(*ctx).extra_hw_frames = 4;
let r = ffi::avcodec_open2(ctx, codec, ptr::null_mut());
if r < 0 {
let mut ctx = ctx;
ffi::avcodec_free_context(&mut ctx);
let mut hw = hw_device;
ffi::av_buffer_unref(&mut hw);
bail!("avcodec_open2 (D3D11VA): {}", ffmpeg::Error::from(r));
}
Ok(D3d11vaDecoder {
ctx,
hw_device,
packet: ffi::av_packet_alloc(),
frame: ffi::av_frame_alloc(),
})
}
}
fn decode(&mut self, au: &[u8]) -> Result<Option<GpuFrame>> {
use ffmpeg::ffi;
unsafe {
let r = ffi::av_new_packet(self.packet, au.len() as i32);
if r < 0 {
return Err(averr("av_new_packet", r));
}
ptr::copy_nonoverlapping(au.as_ptr(), (*self.packet).data, au.len());
let r = ffi::avcodec_send_packet(self.ctx, self.packet);
ffi::av_packet_unref(self.packet);
if r < 0 {
return Err(averr("send_packet", r));
}
let mut out = None;
loop {
let r = ffi::avcodec_receive_frame(self.ctx, self.frame);
if r == AVERROR_EAGAIN {
break;
}
if r < 0 {
return Err(averr("receive_frame", r));
}
out = Some(self.lift()?); // newest wins; older guards drop here
ffi::av_frame_unref(self.frame);
}
Ok(out)
}
}
/// Lift the decoded D3D11 surface into a `GpuFrame`. `data[0]` is the texture array, `data[1]`
/// the slice index. We `av_frame_clone` so the surface stays referenced (kept out of the reuse
/// pool) until the presenter drops the guard.
unsafe fn lift(&mut self) -> Result<GpuFrame> {
use ffmpeg::ffi;
unsafe {
if (*self.frame).format != ffi::AVPixelFormat::AV_PIX_FMT_D3D11 as i32 {
bail!("decoder returned a software frame (no D3D11 surface)");
}
// SAFETY: `self.frame` is the live decoded AVFrame for the duration of this call.
let color = ColorDesc::from_raw(self.frame);
let hdr = color.is_pq();
let ten_bit = {
let hwfc = (*self.frame).hw_frames_ctx;
!hwfc.is_null()
&& (*((*hwfc).data as *const ffi::AVHWFramesContext)).sw_format
== ffi::AVPixelFormat::AV_PIX_FMT_P010LE
};
let cloned = ffi::av_frame_clone(self.frame);
if cloned.is_null() {
bail!("av_frame_clone failed");
}
let frame = GpuFrame {
width: (*self.frame).width as u32,
height: (*self.frame).height as u32,
index: (*self.frame).data[1] as usize as u32,
ten_bit,
hdr,
color,
guard: D3d11FrameGuard(cloned),
};
log_layout_once(frame.width, frame.height, frame.index, hdr, ten_bit);
Ok(frame)
}
}
}
impl Drop for D3d11vaDecoder {
fn drop(&mut self) {
use ffmpeg::ffi;
unsafe {
ffi::av_packet_free(&mut self.packet);
ffi::av_frame_free(&mut self.frame);
ffi::avcodec_free_context(&mut self.ctx);
ffi::av_buffer_unref(&mut self.hw_device);
}
}
}
/// One-time dump of the first decoded surface's layout — so a new GPU/driver combination's real
/// format (slice index range, HDR/bit-depth) is visible in the logs without a debugger.
fn log_layout_once(width: u32, height: u32, index: u32, hdr: bool, ten_bit: bool) {
use std::sync::atomic::{AtomicBool, Ordering};
static ONCE: AtomicBool = AtomicBool::new(true);
if ONCE.swap(false, Ordering::Relaxed) {
tracing::info!(
width,
height,
slice = index,
hdr,
ten_bit,
"D3D11VA first frame"
);
}
}
File diff suppressed because it is too large Load Diff
+134 -45
View File
@@ -10,6 +10,7 @@ use crate::audio;
use crate::video::{DecodedFrame, DecodedImage, Decoder}; use crate::video::{DecodedFrame, DecodedImage, Decoder};
use punktfunk_core::client::NativeClient; use punktfunk_core::client::NativeClient;
use punktfunk_core::config::{CompositorPref, GamepadPref, Mode}; use punktfunk_core::config::{CompositorPref, GamepadPref, Mode};
use punktfunk_core::reanchor::{index_gap, GateVerdict, ReanchorGate};
use punktfunk_core::PunktfunkError; use punktfunk_core::PunktfunkError;
use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc; use std::sync::Arc;
@@ -99,11 +100,6 @@ pub struct Stats {
pub decoder: &'static str, pub decoder: &'static str,
} }
/// Consecutive no-output AUs that force a keyframe request. ~50 ms at 60 Hz — long
/// enough not to fire on a one-frame decoder hiccup, short enough that a lost initial
/// IDR (or a mid-GOP join) unfreezes almost immediately instead of never.
const NO_OUTPUT_KEYFRAME_STREAK: u32 = 3;
/// Frames the pump keeps waiting for their 0xCF host timing (pts → capture→received µs). /// Frames the pump keeps waiting for their 0xCF host timing (pts → capture→received µs).
/// ~2 s at 120 Hz — a timing arrives within a frame or two of its AU, and against an old /// ~2 s at 120 Hz — a timing arrives within a frame or two of its AU, and against an old
/// host (no 0xCF at all) this just caps the dead-weight ring. /// host (no 0xCF at all) this just caps the dead-weight ring.
@@ -307,18 +303,22 @@ fn pump(
// What actually decoded the last frame — a VAAPI failure demotes mid-session, so // What actually decoded the last frame — a VAAPI failure demotes mid-session, so
// this is read off each frame's image variant rather than fixed at startup. // this is read off each frame's image variant rather than fixed at startup.
let mut dec_path: &'static str = ""; let mut dec_path: &'static str = "";
// Loss recovery: watch the host→client unrecoverable-drop count and ask for an IDR when it climbs.
let mut last_dropped = connector.frames_dropped();
// The stats window keeps its own drop cursor — the OSD shows the per-window delta. // The stats window keeps its own drop cursor — the OSD shows the per-window delta.
let mut window_dropped = last_dropped; let mut window_dropped = connector.frames_dropped();
let mut last_kf_req: Option<Instant> = None; let mut last_kf_req: Option<Instant> = None;
// Consecutive received AUs that produced NO decoded frame (decode error, or the // Freeze-until-reanchor: the shared post-loss gate ([`punktfunk_core::reanchor::ReanchorGate`]).
// decoder swallowed a reference-missing delta and returned nothing). Distinct from // Armed on any loss signal (frame-index gap, dropped-count climb, decoder wedge/demotion), it
// `frames_dropped`, which counts reassembler drops: when the initial IDR is lost (or // withholds the decoder's concealed frames from the presenter — which then redraws the last good
// we join mid-GOP) the reassembler delivers complete-but-undecodable deltas — it // picture — until a proven clean re-anchor (IDR / RFI anchor / second recovery mark) lifts it. It
// never drops, so the drop-count trigger below stays silent and the stream freezes // also owns the no-output streak and the overdue-freeze backstop; the client keeps its own
// on the last good frame. A short streak forces a fresh IDR to re-anchor. // `last_kf_req` request throttle and routes the gate's keyframe intents through it. Seeded with the
let mut no_output_streak = 0u32; // current drop count so the first `poll` doesn't read the baseline as a loss.
let mut gate = ReanchorGate::new(connector.frames_dropped());
// The frame_index we expect next (the host numbers frames consecutively). A jump means a frame
// went missing — the earliest, most reliable signal that the decoder is about to conceal, ~120 ms
// ahead of `frames_dropped` (the reassembler only declares a straggler lost once it ages out of
// the loss window, by which point the concealment already reached the screen).
let mut next_expected_index: Option<u32> = None;
let end: Option<String> = loop { let end: Option<String> = loop {
if stop.load(Ordering::SeqCst) { if stop.load(Ordering::SeqCst) {
@@ -334,9 +334,77 @@ fn pump(
// fps / goodput count every received AU (spec), decoded or not. // fps / goodput count every received AU (spec), decoded or not.
frames_n += 1; frames_n += 1;
bytes_n += frame.data.len() as u64; bytes_n += frame.data.len() as u64;
// Reference-continuity gate: the host numbers frames consecutively, so a jump in
// frame_index means a frame is missing (lost, or an out-of-order straggler the
// reassembler emitted a newer frame ahead of) and this AU references a picture we
// never decoded. On RADV the decoder conceals that as a gray plate with the new
// motion on top — the reported artifact, and it shows most on high-motion frames (a
// full-screen pan bursts far more packets than a static desktop or a UFO-test's small
// moving sprite, so it is the frame that loses shards). Arm the freeze at the FIRST
// such frame — ~120 ms before `frames_dropped` would — so the gray never reaches the
// screen; recovery IDRs stay on the existing throttled path (see the arm below).
match next_expected_index {
Some(exp) if frame.frame_index == exp => {
next_expected_index = Some(exp.wrapping_add(1)); // contiguous
}
// A forward gap: hold the last good frame — but DO NOT ask for a keyframe here.
// Hiding the concealment is free (the presenter redraws the last picture); an IDR
// is not — at 4K120 it is a multi-megabyte frame and a visible stutter, and it can
// re-trigger the very burst loss that caused this. The existing loss recovery below
// (`frames_dropped`, host-coalesced + throttled) still requests it at exactly the
// cadence it did before this change, so we add zero IDR pressure per pan. A
// straggler behind us (`index_gap` → None) leaves the expectation put so the real
// gap still trips.
Some(exp) => {
if let Some(gap) = index_gap(exp, frame.frame_index) {
let now = Instant::now();
gate.arm(now);
next_expected_index = Some(frame.frame_index.wrapping_add(1));
// The gap carries the PRECISE lost range — [first missing, newest
// received - 1] — so this is the one recovery signal that can drive true
// reference-frame invalidation. Prefer an RFI request over a keyframe: an
// RFI-capable host (AMD LTR / NVENC) re-references a known-good picture and
// emits a clean P-frame tagged USER_FLAG_RECOVERY_ANCHOR (the freeze lifts
// on ONE frame, no 20-40× IDR spike); an incapable/old host forces a
// host-coalesced IDR instead, or ignores it (then the frames_dropped /
// overdue keyframe paths below are the backstop). Throttled with those
// paths (one recovery ask per 100 ms) so a burst of gaps — a full-screen
// pan shedding shards — can't storm the control stream. This fires ~120 ms
// before frames_dropped would, so recovery also starts sooner.
//
// A gap wider than RFI_MAX_RANGE is beyond any encoder's reference
// history (a seconds-long outage — or a phantom index jump, e.g. the
// first real AU after an old host's speed-test burst consumed video
// indexes): RFI is hopeless there, so ask for the IDR resync directly.
if last_kf_req
.is_none_or(|t| now.duration_since(t) >= Duration::from_millis(100))
{
last_kf_req = Some(now);
if gap > punktfunk_core::packet::RFI_MAX_RANGE {
let _ = connector.request_keyframe();
} else {
let _ = connector
.request_rfi(exp, frame.frame_index.wrapping_sub(1));
}
}
tracing::trace!(
gap,
"frame gap — RFI recovery, holding last frame until re-anchor"
);
}
}
None => next_expected_index = Some(frame.frame_index.wrapping_add(1)),
}
match decoder.decode(&frame.data) { match decoder.decode(&frame.data) {
Ok(Some(image)) => { Ok(Some(image)) => {
no_output_streak = 0; // a decoded frame — the anchor holds // Fold this decoded frame through the shared freeze gate: it reads the AU's
// re-anchor wire flags (FLAG_SOF IDR marker / RECOVERY_ANCHOR / RECOVERY_POINT),
// takes `image.is_keyframe()` as the ffmpeg keyframe belt, applies the two-mark
// rule + the mark-patience backstop, clears the no-output streak, and returns
// whether to present this frame or withhold it as a post-loss concealment.
let present =
gate.on_decoded(frame.flags, image.is_keyframe(), Instant::now())
== GateVerdict::Present;
total_frames += 1; total_frames += 1;
dec_path = match &image { dec_path = match &image {
DecodedImage::Cpu(_) => "software", DecodedImage::Cpu(_) => "software",
@@ -391,11 +459,20 @@ fn pump(
DecodedImage::VkFrame(v) => Some((v.timeline_sem, v.decode_done_value)), DecodedImage::VkFrame(v) => Some((v.timeline_sem, v.decode_done_value)),
_ => None, _ => None,
}; };
if present {
let _ = frame_tx.force_send(DecodedFrame { let _ = frame_tx.force_send(DecodedFrame {
pts_ns: frame.pts_ns, pts_ns: frame.pts_ns,
decoded_ns, decoded_ns,
image, image,
}); });
} else {
// Post-loss concealment: withhold this frame (it references a lost/gray
// reference) so the presenter keeps redrawing the last good picture rather
// than flashing the decoder's gray plate. Dropped here — the hw-decode stat
// below still samples via `hw_fence` (raw handle + value, valid past the
// guard). The gate lifts the freeze on the next clean re-anchor / backstop.
tracing::trace!("holding last frame — awaiting post-loss re-anchor");
}
// `decode` stage: received→decode COMPLETE, single clock. // `decode` stage: received→decode COMPLETE, single clock.
match hw_fence { match hw_fence {
Some((sem, value)) => { Some((sem, value)) => {
@@ -410,30 +487,35 @@ fn pump(
} }
} }
} }
Ok(None) => no_output_streak += 1, // The decoder produced nothing — under zero-reorder LOW_DELAY (one-in/one-out) that
// Survivable (loss until the next IDR/RFI recovery) — keep feeding. // means it's wedged on missing references with no reassembler drop to trigger
Err(e) => { // recovery. The gate counts the streak and, once it trips, arms the freeze and tells
no_output_streak += 1; // us to (throttled) request a fresh IDR to re-anchor. Both the empty-output and the
tracing::debug!(error = %e, "decode error (recovering)"); // survivable-decode-error arms feed it; a decoded frame resets the streak in
} // `on_decoded`.
} Ok(None) => {
// The decoder has produced nothing for a short run — under zero-reorder
// LOW_DELAY (one-in/one-out) that means it's wedged on missing references
// with no reassembler drop to trigger recovery below. Ask for a fresh IDR
// (throttled), then re-arm the streak so we wait out the request→IDR round
// trip before asking again instead of flooding.
if no_output_streak >= NO_OUTPUT_KEYFRAME_STREAK {
let now = Instant::now(); let now = Instant::now();
if last_kf_req if gate.on_no_output(now)
&& last_kf_req
.is_none_or(|t| now.duration_since(t) >= Duration::from_millis(100)) .is_none_or(|t| now.duration_since(t) >= Duration::from_millis(100))
{ {
last_kf_req = Some(now); last_kf_req = Some(now);
let _ = connector.request_keyframe(); let _ = connector.request_keyframe();
tracing::debug!( tracing::debug!("requested keyframe (decoder produced no output)");
streak = no_output_streak, }
"requested keyframe (decoder produced no output)" }
); // Survivable (loss until the next IDR/RFI recovery) — keep feeding.
no_output_streak = 0; Err(e) => {
tracing::debug!(error = %e, "decode error (recovering)");
let now = Instant::now();
if gate.on_no_output(now)
&& last_kf_req
.is_none_or(|t| now.duration_since(t) >= Duration::from_millis(100))
{
last_kf_req = Some(now);
let _ = connector.request_keyframe();
tracing::debug!("requested keyframe (decode error recovery)");
}
} }
} }
// The presenter's verdict: hardware frames can't be displayed (GL converter // The presenter's verdict: hardware frames can't be displayed (GL converter
@@ -451,6 +533,7 @@ fn pump(
// through the same throttle as loss recovery below. // through the same throttle as loss recovery below.
if decoder.take_keyframe_request() { if decoder.take_keyframe_request() {
let now = Instant::now(); let now = Instant::now();
gate.arm(now);
if last_kf_req if last_kf_req
.is_none_or(|t| now.duration_since(t) >= Duration::from_millis(100)) .is_none_or(|t| now.duration_since(t) >= Duration::from_millis(100))
{ {
@@ -478,20 +561,26 @@ fn pump(
} }
} }
// Loss recovery: under infinite GOP the only recovery keyframe is one we request. The // Loss recovery + overdue backstop, folded through the shared gate. A climb in the
// reassembler drops unrecoverable AUs (frames_dropped); the decoder then conceals the // reassembler's unrecoverable-drop count (`frames_dropped`) means the AUs after the lost one
// reference-missing delta frames that follow and returns Ok, so keying off a decode error // reference a picture we never decoded — the decoder conceals them (gray on RADV) and returns
// rarely fires. Request an IDR when the drop count climbs, throttled — the decode stays // Ok, so a decode-error trigger rarely fires; the gate arms the freeze on the climb instead. An
// wedged for several frames until the IDR lands, so requesting every frame would flood. // overdue freeze (held a full REANCHOR_FREEZE_MAX with no clean re-anchor — a lost recovery IDR,
// or a benign reorder that produced no `frames_dropped`) re-asks while it keeps holding: NEVER
// resume to gray — a genuinely dead stream is the QUIC idle-timeout watchdog's job. Both route
// the gate's keyframe intent through the shared 100 ms throttle; under infinite GOP the only
// recovery keyframe is one we request.
let dropped = connector.frames_dropped(); let dropped = connector.frames_dropped();
if dropped > last_dropped {
last_dropped = dropped;
let now = Instant::now(); let now = Instant::now();
if last_kf_req.is_none_or(|t| now.duration_since(t) >= Duration::from_millis(100)) { if gate.poll(dropped, now)
&& last_kf_req.is_none_or(|t| now.duration_since(t) >= Duration::from_millis(100))
{
last_kf_req = Some(now); last_kf_req = Some(now);
let _ = connector.request_keyframe(); let _ = connector.request_keyframe();
tracing::debug!(dropped, "requested keyframe (loss recovery)"); tracing::debug!(
} dropped,
"requested keyframe (loss recovery / overdue re-anchor)"
);
} }
if window_start.elapsed() >= Duration::from_secs(1) { if window_start.elapsed() >= Duration::from_secs(1) {
+16
View File
@@ -406,6 +406,19 @@ pub struct Settings {
/// Experimental: the game-library browser ("Browse library…" on saved cards) — /// Experimental: the game-library browser ("Browse library…" on saved cards) —
/// mirrors the Apple client's "Show game library" toggle, default off. /// mirrors the Apple client's "Show game library" toggle, default off.
pub library_enabled: bool, pub library_enabled: bool,
/// Match-window resolution policy (design/midstream-resolution-resize.md D1): the
/// stream mode follows the session window — the connect asks for the window's pixel
/// size and a mid-session resize renegotiates the host's virtual display + encoder
/// (`Reconfigure`), so windowed sessions stream native-resolution pixels instead of
/// scaling. Overrides `width`/`height` while on; on fullscreen it degenerates to the
/// display's native mode. Default off (Auto-native stays the shipped default until
/// the per-backend validation matrix is green).
pub match_window: bool,
/// The session window's last logical size under `match_window`: the next launch
/// opens its window at this size, so the first connect's mode already matches what
/// the user will be looking at. `0` = never stored → the 1280×720 default.
pub last_window_w: u32,
pub last_window_h: u32,
} }
fn default_codec() -> String { fn default_codec() -> String {
@@ -466,6 +479,9 @@ impl Default for Settings {
stats_verbosity: None, stats_verbosity: None,
fullscreen_on_stream: true, fullscreen_on_stream: true,
library_enabled: false, library_enabled: false,
match_window: false,
last_window_w: 0,
last_window_h: 0,
} }
} }
} }
+65
View File
@@ -98,6 +98,10 @@ pub struct VkVideoFrame {
pub width: u32, pub width: u32,
pub height: u32, pub height: u32,
pub color: ColorDesc, pub color: ColorDesc,
/// Intra keyframe (IDR/I): the stream's re-anchor point. The pump resumes display on
/// one after suppressing the concealed frames a reference loss leaves in its wake (on
/// RADV a lost reference decodes to a gray plate with the new motion painted on top).
pub keyframe: bool,
/// Keeps the cloned AVFrame (and through it the VkImage + frames context) alive /// Keeps the cloned AVFrame (and through it the VkImage + frames context) alive
/// until the presenter's fence proves the GPU reads done — same mechanism as the /// until the presenter's fence proves the GPU reads done — same mechanism as the
/// VAAPI path's DRM guard. /// VAAPI path's DRM guard.
@@ -143,6 +147,58 @@ impl ColorDesc {
} }
} }
/// True if the decoder tagged this frame as a full IDR keyframe — a guaranteed clean re-anchor
/// after which the picture is loss-free, so the pump can lift a post-loss display freeze here.
///
/// Keys off `AV_FRAME_FLAG_KEY` (with `pict_type == I` as a belt for decoders that fill pict_type
/// but not the flag). NOTE: FFmpeg's H.264/HEVC decode layer sets this flag **only for true IDR
/// frames**, never for an *intra-refresh recovery point*. H.264 flags key only when a picture's
/// `recovery_frame_cnt == 0` (a moving band uses `> 0`); HEVC clears the flag on every non-IRAP
/// frame regardless of the recovery-point SEI. So an intra-refresh host (NVENC/AMF/QSV) heals the
/// picture over N P-frames with no decoded frame ever flagged key — this function cannot detect
/// that clean point, and the pump would freeze until the `REANCHOR_FREEZE_MAX` backstop (in
/// `session.rs`) forces a real IDR. Detecting an intra-refresh re-anchor requires an out-of-band
/// host wire signal on the AU that completes the wave; that is not yet plumbed.
///
/// # Safety
/// `frame` must point to a valid `AVFrame` alive for the duration of the call.
pub unsafe fn frame_is_keyframe(frame: *const ffmpeg::ffi::AVFrame) -> bool {
// SAFETY: caller guarantees a live AVFrame; plain field reads.
unsafe {
((*frame).flags & ffmpeg::ffi::AV_FRAME_FLAG_KEY) != 0
|| (*frame).pict_type == ffmpeg::ffi::AVPictureType::AV_PICTURE_TYPE_I
}
}
impl DecodedImage {
/// Whether the frame is an intra keyframe — see [`frame_is_keyframe`]. The pump uses
/// this as the stream's re-anchor signal after a loss.
pub fn is_keyframe(&self) -> bool {
match self {
DecodedImage::Cpu(f) => f.keyframe,
#[cfg(target_os = "linux")]
DecodedImage::Dmabuf(f) => f.keyframe,
DecodedImage::VkFrame(f) => f.keyframe,
#[cfg(windows)]
DecodedImage::D3d11(f) => f.keyframe,
}
}
/// The decoded image's pixel dimensions. The presenter's resize indicator uses these
/// as the mid-stream-resize END signal: a frame arriving at the target size means the
/// new-mode picture is on glass (the ack alone lands before the host's rebuild does).
pub fn dimensions(&self) -> (u32, u32) {
match self {
DecodedImage::Cpu(f) => (f.width, f.height),
#[cfg(target_os = "linux")]
DecodedImage::Dmabuf(f) => (f.width, f.height),
DecodedImage::VkFrame(f) => (f.width, f.height),
#[cfg(windows)]
DecodedImage::D3d11(f) => (f.width, f.height),
}
}
}
/// The YCbCr→RGB conversion as three vec4 rows for a shader constant buffer / push-constant /// The YCbCr→RGB conversion as three vec4 rows for a shader constant buffer / push-constant
/// block: `rgb[i] = dot(r[i].xyz, yuv) + r[i].w` — bit-depth exact. The ONE coefficient /// block: `rgb[i] = dot(r[i].xyz, yuv) + r[i].w` — bit-depth exact. The ONE coefficient
/// implementation every presenter derives its CSC from (Vulkan push constants, the Windows /// implementation every presenter derives its CSC from (Vulkan push constants, the Windows
@@ -205,6 +261,8 @@ pub struct CpuFrame {
/// pixels are full-range RGB), but a PQ/BT.2020 stream keeps its transfer + primaries /// pixels are full-range RGB), but a PQ/BT.2020 stream keeps its transfer + primaries
/// baked in — the presenter tags the texture so GTK tone-maps it. /// baked in — the presenter tags the texture so GTK tone-maps it.
pub color: ColorDesc, pub color: ColorDesc,
/// Intra keyframe (IDR/I) — the pump's post-loss re-anchor signal. See [`VkVideoFrame`].
pub keyframe: bool,
} }
/// A decoded frame still on the GPU: dmabuf fds + plane layout for /// A decoded frame still on the GPU: dmabuf fds + plane layout for
@@ -222,6 +280,8 @@ pub struct DmabufFrame {
/// Signaling of the source frame — drives the `GdkDmabufTexture` color state (BT.709 /// Signaling of the source frame — drives the `GdkDmabufTexture` color state (BT.709
/// narrow for SDR, BT.2020 PQ for an HDR stream). /// narrow for SDR, BT.2020 PQ for an HDR stream).
pub color: ColorDesc, pub color: ColorDesc,
/// Intra keyframe (IDR/I) — the pump's post-loss re-anchor signal. See [`VkVideoFrame`].
pub keyframe: bool,
pub guard: DrmFrameGuard, pub guard: DrmFrameGuard,
} }
@@ -644,6 +704,9 @@ impl SoftwareDecoder {
stride: dst_linesize[0] as usize, stride: dst_linesize[0] as usize,
rgba, rgba,
color, color,
// `is_key()` reads the same intra flag `frame_is_keyframe` derives from pict_type
// for the hardware paths; ffmpeg-next handles the FFmpeg-version binding split.
keyframe: frame.is_key(),
}) })
} }
} }
@@ -844,6 +907,7 @@ impl VaapiDecoder {
// SAFETY: `self.frame` is the live decoded AVFrame (unref'd only after // SAFETY: `self.frame` is the live decoded AVFrame (unref'd only after
// this returns); plain CICP field reads. // this returns); plain CICP field reads.
color: ColorDesc::from_raw(self.frame), color: ColorDesc::from_raw(self.frame),
keyframe: frame_is_keyframe(self.frame),
guard, guard,
}) })
} }
@@ -1363,6 +1427,7 @@ impl VulkanDecoder {
width: (*self.frame).width as u32, width: (*self.frame).width as u32,
height: (*self.frame).height as u32, height: (*self.frame).height as u32,
color: ColorDesc::from_raw(self.frame), color: ColorDesc::from_raw(self.frame),
keyframe: frame_is_keyframe(self.frame),
guard: DrmFrameGuard(clone), guard: DrmFrameGuard(clone),
}) })
} }
+5
View File
@@ -99,6 +99,9 @@ pub struct D3d11Frame {
/// BT.709 full-range RGB — regardless of the stream's own CICP (a PQ stream was /// BT.709 full-range RGB — regardless of the stream's own CICP (a PQ stream was
/// tone-mapped). The presenter keys SDR/HDR handling off this, so it always reads SDR. /// tone-mapped). The presenter keys SDR/HDR handling off this, so it always reads SDR.
pub color: ColorDesc, pub color: ColorDesc,
/// Intra keyframe (IDR/I) — the pump's post-loss re-anchor signal. See
/// `crate::video::VkVideoFrame`.
pub keyframe: bool,
/// The ring slot's NT shared handle (`IDXGIResource1::CreateSharedHandle`), stable for the /// The ring slot's NT shared handle (`IDXGIResource1::CreateSharedHandle`), stable for the
/// ring's lifetime. Raw `isize` so the frame crosses the pump→presenter channel. /// ring's lifetime. Raw `isize` so the frame crosses the pump→presenter channel.
pub handle: isize, pub handle: isize,
@@ -692,6 +695,8 @@ impl D3d11vaDecoder {
matrix: 0, // identity — RGB matrix: 0, // identity — RGB
full_range: true, full_range: true,
}, },
// SAFETY: `self.frame` is the live decoded AVFrame for this call.
keyframe: crate::video::frame_is_keyframe(self.frame),
handle, handle,
generation, generation,
}) })
+34 -7
View File
@@ -176,7 +176,9 @@ fn row_spec(id: RowId, ctx: &Ctx) -> RowSpec {
RowId::Resolution => ( RowId::Resolution => (
Some("Stream"), Some("Stream"),
"Resolution", "Resolution",
if s.width == 0 { if s.match_window {
"Match window".into()
} else if s.width == 0 {
"Native".into() "Native".into()
} else { } else {
format!("{} × {}", s.width, s.height) format!("{} × {}", s.width, s.height)
@@ -259,7 +261,8 @@ fn row_spec(id: RowId, ctx: &Ctx) -> RowSpec {
fn detail(id: RowId) -> &'static str { fn detail(id: RowId) -> &'static str {
match id { match id {
RowId::Resolution => { RowId::Resolution => {
"The host creates a virtual display at exactly this size — no scaling." "The host creates a virtual display at exactly this size — no scaling. \
Match window follows this window, including mid-stream resizes."
} }
RowId::Refresh => "Native follows the display this window is on.", RowId::Refresh => "Native follows the display this window is on.",
RowId::Bitrate => "Automatic uses the host's default (20 Mbps).", RowId::Bitrate => "Automatic uses the host's default (20 Mbps).",
@@ -303,11 +306,20 @@ fn adjust(id: RowId, delta: i32, wrap: bool, ctx: &mut Ctx) -> bool {
let s = &mut *ctx.settings; let s = &mut *ctx.settings;
match id { match id {
RowId::Resolution => { RowId::Resolution => {
let cur = RESOLUTIONS // The D1 tri-state as one picker: Native, Match window, then the explicit
// sizes (RESOLUTIONS keeps its (0,0) = Native head; Match window is the
// virtual index 1, stored as the `match_window` flag with w/h cleared).
let cur = if s.match_window {
Some(1)
} else {
RESOLUTIONS
.iter() .iter()
.position(|(w, h)| (*w, *h) == (s.width, s.height)); .position(|(w, h)| (*w, *h) == (s.width, s.height))
step_option(cur, RESOLUTIONS.len(), delta, wrap).map(|i| { .map(|i| if i == 0 { 0 } else { i + 1 })
(s.width, s.height) = RESOLUTIONS[i]; };
step_option(cur, RESOLUTIONS.len() + 1, delta, wrap).map(|i| {
s.match_window = i == 1;
(s.width, s.height) = if i <= 1 { (0, 0) } else { RESOLUTIONS[i - 1] };
}) })
} }
RowId::Refresh => { RowId::Refresh => {
@@ -401,14 +413,29 @@ mod tests {
device_name: "t", device_name: "t",
t: 0.0, t: 0.0,
}; };
// Resolution starts at Native (index 0): left refuses, right steps. // Resolution starts at Native (index 0): left refuses, right steps — first onto
// Match window (the D1 tri-state's middle option), then the explicit sizes.
assert!(!adjust(RowId::Resolution, -1, false, &mut ctx)); assert!(!adjust(RowId::Resolution, -1, false, &mut ctx));
assert!(adjust(RowId::Resolution, 1, false, &mut ctx)); assert!(adjust(RowId::Resolution, 1, false, &mut ctx));
assert!(ctx.settings.match_window, "Native → Match window");
assert_eq!((ctx.settings.width, ctx.settings.height), (0, 0));
assert!(adjust(RowId::Resolution, 1, false, &mut ctx));
assert!(
!ctx.settings.match_window,
"explicit size clears the policy"
);
assert_eq!((ctx.settings.width, ctx.settings.height), (1280, 720)); assert_eq!((ctx.settings.width, ctx.settings.height), (1280, 720));
// Stepping back from an explicit size returns to Match window, then Native.
assert!(adjust(RowId::Resolution, -1, false, &mut ctx));
assert!(ctx.settings.match_window);
assert!(adjust(RowId::Resolution, -1, false, &mut ctx));
assert!(!ctx.settings.match_window);
assert_eq!(ctx.settings.width, 0, "back to Native");
// Cycle from the last option wraps to the first. // Cycle from the last option wraps to the first.
(ctx.settings.width, ctx.settings.height) = (3840, 2160); (ctx.settings.width, ctx.settings.height) = (3840, 2160);
assert!(adjust(RowId::Resolution, 1, true, &mut ctx)); assert!(adjust(RowId::Resolution, 1, true, &mut ctx));
assert_eq!(ctx.settings.width, 0, "wrapped to Native"); assert_eq!(ctx.settings.width, 0, "wrapped to Native");
assert!(!ctx.settings.match_window);
} }
#[test] #[test]
+119 -91
View File
@@ -19,7 +19,9 @@ use anyhow::{anyhow, Result};
use pf_client_core::gamepad::{MenuDir, MenuEvent, MenuPulse, PadInfo}; use pf_client_core::gamepad::{MenuDir, MenuEvent, MenuPulse, PadInfo};
use pf_client_core::trust; use pf_client_core::trust;
use pf_presenter::overlay::OverlayAction; use pf_presenter::overlay::OverlayAction;
use skia_safe::{Canvas, Color4f, Data, Paint, Rect, RuntimeEffect}; use skia_safe::{
gradient_shader, Canvas, Color4f, Data, Paint, Point, Rect, RuntimeEffect, TileMode,
};
use std::collections::VecDeque; use std::collections::VecDeque;
use std::time::Instant; use std::time::Instant;
@@ -240,6 +242,11 @@ impl Shell {
self.wake = None; self.wake = None;
if let Some(Some(intent)) = intent { if let Some(Some(intent)) = intent {
self.start_connect(intent); self.start_connect(intent);
// The wake takeover was already full-screen; skip the connect fade-in so the
// Waking → Connecting handoff is seamless (no flash of the home behind).
if let Some(c) = &mut self.connecting {
c.appear = 1.0;
}
} }
} }
} }
@@ -607,77 +614,68 @@ impl Shell {
t: f64, t: f64,
fonts: &Fonts, fonts: &Fonts,
) { ) {
// Resolve the connect/wake takeover — the two phases of reaching a host — into one
// full-screen shape (spinner, title, one detail line, its own hints). Connecting flows
// straight out of a wake (see `sync`) so they share the same backdrop and never blink
// between them. Mirrors the Android client's unified `ConnectOverlay`.
let takeover: Option<(f64, bool, String, String, Vec<Hint>)> =
if let Some(c) = &mut self.connecting { if let Some(c) = &mut self.connecting {
c.appear = approach(c.appear, 1.0, dt, 0.07); c.appear = approach(c.appear, 1.0, dt, 0.07);
let a = c.appear; if c.canceling {
canvas.draw_rect( Some((
Rect::from_wh(w as f32, h as f32), c.appear,
&Paint::new(Color4f::new(0.0, 0.0, 0.0, (0.45 * a) as f32), None),
);
let title = if c.canceling {
"Canceling…".to_string()
} else {
format!("Connecting to {}", c.title)
};
let hints = if c.canceling {
vec![]
} else {
vec![Hint::new(HintKey::Back, "Cancel")]
};
card(
canvas,
fonts,
w,
h,
k,
a,
t,
self.glyphs,
true, true,
&title, "Canceling…".to_string(),
"Starting the stream in this window.", String::new(),
&hints, vec![],
); ))
} else {
Some((
c.appear,
true,
format!("Connecting to {}", c.title),
"Starting the stream in this window.".to_string(),
vec![Hint::new(HintKey::Back, "Cancel")],
))
}
} else if let Some(wk) = &self.wake { } else if let Some(wk) = &self.wake {
let a = 1.0; // the wake card is service-driven; it appears settled // Service-driven, so it appears settled (no fade-in).
canvas.draw_rect(
Rect::from_wh(w as f32, h as f32),
&Paint::new(Color4f::new(0.0, 0.0, 0.0, 0.45), None),
);
if wk.timed_out { if wk.timed_out {
card( Some((
canvas, 1.0,
fonts,
w,
h,
k,
a,
t,
self.glyphs,
false, false,
&format!("{} didn't wake", wk.name), format!("{} didn't wake", wk.name),
"Check its power settings, or wake it manually and try again.", "Check its power settings, or wake it manually and try again.".to_string(),
&[ vec![
Hint::new(HintKey::Confirm, "Try Again"), Hint::new(HintKey::Confirm, "Try Again"),
Hint::new(HintKey::Back, "Cancel"), Hint::new(HintKey::Back, "Cancel"),
], ],
); ))
} else { } else {
card( Some((
canvas, 1.0,
fonts,
w,
h,
k,
a,
t,
self.glyphs,
true, true,
&format!("Waking {}", wk.name), format!("Waking {}", wk.name),
&format!("Waiting for it to come online · {} s", wk.seconds), format!("Waiting for it to come online · {} s", wk.seconds),
&[Hint::new(HintKey::Back, "Cancel")], // A wake-only wait (no dial after) offers "Stop Waiting"; a wake-&-connect
); // is a plain "Cancel".
vec![Hint::new(
HintKey::Back,
if wk.then_connect {
"Cancel"
} else {
"Stop Waiting"
},
)],
))
} }
} else {
None
};
if let Some((appear, spinner, title, body, hints)) = takeover {
self.draw_takeover(
canvas, w, h, k, appear, t, fonts, spinner, &title, &body, &hints,
);
} }
// The toast: a transient pill above the hint bar; slides in, fades out. // The toast: a transient pill above the hint bar; slides in, fades out.
@@ -799,71 +797,92 @@ impl LayerEnv<'_> {
} }
} }
/// A centered modal card: spinner (or not), a title, one detail line, and its own hint impl Shell {
/// row — the connect/wake overlays share this one shape. /// A full-screen connect/wake takeover: a fresh aurora over everything (so the carousel and
/// chrome fall away), a centered spinner (or none, when a wake has timed out), a title, one
/// detail line, and its own bottom hint row. `appear` fades the whole thing in over the home;
/// a wake that hands off to a connect passes 1.0 so the two never blink between them. The
/// console counterpart of the Android/Apple `ConnectOverlay` — one full-screen shape, not a
/// centered modal card.
#[allow(clippy::too_many_arguments)] #[allow(clippy::too_many_arguments)]
fn card( fn draw_takeover(
&self,
canvas: &Canvas, canvas: &Canvas,
fonts: &Fonts,
w: f64, w: f64,
h: f64, h: f64,
k: f64, k: f64,
appear: f64, appear: f64,
t: f64, t: f64,
glyphs: GlyphStyle, fonts: &Fonts,
spinner: bool, spinner: bool,
title: &str, title: &str,
body: &str, body: &str,
hints: &[Hint], hints: &[Hint],
) { ) {
let cw = (440.0 * k).min(w * 0.86);
let ch = 190.0 * k;
let cx = w / 2.0; let cx = w / 2.0;
let top = h / 2.0 - ch / 2.0 + (1.0 - appear) * 14.0 * k;
canvas.save_layer_alpha_f(None, appear as f32); canvas.save_layer_alpha_f(None, appear as f32);
let rect = Rect::from_xywh((cx - cw / 2.0) as f32, top as f32, cw as f32, ch as f32); // Opaque aurora — the same living backdrop the home wears, so the takeover reads as the
crate::theme::drop_shadow(canvas, rect, 22.0, k as f32, 0.5); // console taking over rather than a card popping up.
crate::theme::panel( self.draw_aurora(canvas, w, h, t);
canvas, // A soft pool of shade under the centre seats the white text against a bright aurora.
rect, let mut vignette = Paint::default();
22.0, vignette.set_shader(gradient_shader::radial(
Some(Color4f::new(0.07, 0.06, 0.12, 0.85)), Point::new(cx as f32, (h / 2.0) as f32),
PanelStroke::Plain(0.14), (w.max(h) * 0.42) as f32,
k as f32, gradient_shader::GradientShaderColors::Colors(&[
); Color4f::new(0.0, 0.0, 0.0, 0.5).to_color(),
let mut y = top + 44.0 * k; Color4f::new(0.0, 0.0, 0.0, 0.0).to_color(),
]),
None,
TileMode::Clamp,
None,
None,
));
canvas.draw_rect(Rect::from_wh(w as f32, h as f32), &vignette);
// Centre the spinner + title + detail as a group around the middle of the screen.
let title_y = h / 2.0 + if spinner { 14.0 * k } else { 0.0 };
if spinner { if spinner {
crate::theme::spinner(canvas, cx, y, 14.0 * k, t); crate::theme::spinner(canvas, cx, title_y - 52.0 * k, 22.0 * k, t);
y += 34.0 * k;
} else {
y += 6.0 * k;
} }
fonts.centered(canvas, title, W::SemiBold, 19.0 * k, WHITE, cx, y, cw * 0.9); fonts.centered(
canvas,
title,
W::SemiBold,
23.0 * k,
WHITE,
cx,
title_y,
w * 0.82,
);
if !body.is_empty() {
fonts.centered( fonts.centered(
canvas, canvas,
body, body,
W::Regular, W::Regular,
13.0 * k, 14.0 * k,
DIM, DIM,
cx, cx,
y + 30.0 * k, title_y + 32.0 * k,
cw * 0.86, w * 0.66,
); );
}
if !hints.is_empty() { if !hints.is_empty() {
// Centered inside the card's bottom band. // Centered near the bottom, where every console screen's legend sits.
let probe = hint_bar(canvas, fonts, hints, glyphs, -10_000.0, -10_000.0, k); let probe = hint_bar(canvas, fonts, hints, self.glyphs, -10_000.0, -10_000.0, k);
hint_bar( hint_bar(
canvas, canvas,
fonts, fonts,
hints, hints,
glyphs, self.glyphs,
cx - probe.0 / 2.0, cx - probe.0 / 2.0,
top + ch - 16.0 * k, h - 34.0 * k,
k, k,
); );
} }
canvas.restore(); canvas.restore();
} }
}
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
@@ -1154,6 +1173,15 @@ mod tests {
then_connect: true, then_connect: true,
})); }));
dump(&mut s, 10, 8, "08-waking", true); dump(&mut s, 10, 8, "08-waking", true);
console.set_wake(Some(WakeStatus {
key: "bb22".into(),
name: "Office Tower".into(),
seconds: 90,
timed_out: true,
online: false,
then_connect: true,
}));
dump(&mut s, 10, 8, "08b-wake-timed-out", true);
console.set_wake(None); console.set_wake(None);
s.set_connecting(Some("Elden Ring".into())); s.set_connecting(Some("Elden Ring".into()));
dump(&mut s, 10, 8, "09-connecting", true); dump(&mut s, 10, 8, "09-connecting", true);
+64 -2
View File
@@ -50,6 +50,10 @@ struct Drawn {
hint: Option<String>, hint: Option<String>,
/// The start banner's alpha, quantized — a fade step is a redraw, steady is not. /// The start banner's alpha, quantized — a fade step is a redraw, steady is not.
banner_step: u8, banner_step: u8,
/// The resize scrim's spinner phase, quantized — a nonzero, ever-changing step while a
/// mid-stream resize is in flight forces the per-frame redraw the spinner needs; `0`
/// when no resize is showing (so a still stream stays damage-free).
resize_step: u16,
} }
/// Where the console starts (the session binary's `--browse` forms). /// Where the console starts (the session binary's `--browse` forms).
@@ -85,6 +89,9 @@ pub struct SkiaOverlay {
streaming_since: Option<Instant>, streaming_since: Option<Instant>,
/// The banner's words (set per stream from the active-pad state). /// The banner's words (set per stream from the active-pad state).
banner_text: Option<String>, banner_text: Option<String>,
/// When the current mid-stream resize scrim began showing — drives its spinner phase.
/// `None` = no resize in flight (`FrameCtx::resizing` was false last frame).
resizing_since: Option<Instant>,
} }
struct Gpu { struct Gpu {
@@ -114,6 +121,7 @@ impl SkiaOverlay {
shell: None, shell: None,
streaming_since: None, streaming_since: None,
banner_text: None, banner_text: None,
resizing_since: None,
} }
} }
@@ -340,10 +348,15 @@ impl Overlay for SkiaOverlay {
})); }));
} }
// --- Stream chrome: stats OSD + capture hint + the start banner --------------- // --- Stream chrome: stats OSD + capture hint + start banner + resize scrim -----
let banner_alpha = self.banner_alpha(ctx); let banner_alpha = self.banner_alpha(ctx);
let banner_step = (banner_alpha * 32.0).round() as u8; let banner_step = (banner_alpha * 32.0).round() as u8;
if ctx.stats.is_none() && ctx.hint.is_none() && banner_step == 0 { let resize_phase = self.resize_phase(ctx);
// 120 steps/s: every ~16 ms frame lands on a fresh step, so the spinner keeps
// spinning through the damage gate; `+ 1` keeps an active resize's step nonzero
// even on its first frame (phase 0) so the guard below doesn't skip it.
let resize_step = resize_phase.map_or(0, |p| (p * 120.0) as u16 + 1);
if ctx.stats.is_none() && ctx.hint.is_none() && banner_step == 0 && resize_step == 0 {
self.drawn = Drawn::default(); // forget content so re-show re-renders self.drawn = Drawn::default(); // forget content so re-show re-renders
return Ok(None); return Ok(None);
} }
@@ -353,6 +366,7 @@ impl Overlay for SkiaOverlay {
stats: ctx.stats.map(str::to_owned), stats: ctx.stats.map(str::to_owned),
hint: ctx.hint.map(str::to_owned), hint: ctx.hint.map(str::to_owned),
banner_step, banner_step,
resize_step,
}; };
if want == self.drawn { if want == self.drawn {
// Unchanged — hand the presenter the already-rendered image. // Unchanged — hand the presenter the already-rendered image.
@@ -374,6 +388,10 @@ impl Overlay for SkiaOverlay {
let canvas = slot.surface.canvas(); let canvas = slot.surface.canvas();
canvas.clear(Color4f::new(0.0, 0.0, 0.0, 0.0)); canvas.clear(Color4f::new(0.0, 0.0, 0.0, 0.0));
let font = self.font.as_ref().expect("init ran"); let font = self.font.as_ref().expect("init ran");
// The resize scrim sits UNDER the OSD/hint so those stay legible over it.
if let Some(phase) = resize_phase {
draw_resize_scrim(canvas, font, ctx.width, ctx.height, phase);
}
if let Some(stats) = &want.stats { if let Some(stats) = &want.stats {
draw_osd_panel(canvas, font, stats, 12.0, 12.0); draw_osd_panel(canvas, font, stats, 12.0, 12.0);
} }
@@ -445,6 +463,23 @@ impl SkiaOverlay {
((BANNER_S - age) / BANNER_FADE_S).min(1.0) ((BANNER_S - age) / BANNER_FADE_S).min(1.0)
} }
/// The mid-stream-resize spinner's phase (elapsed seconds since the scrim came up), or
/// `None` when no resize is in flight. Latches the start on the first `resizing` frame
/// and clears it the moment the run loop drops the flag (the target frame landed or the
/// switch timed out), so the next resize starts its spinner from zero.
fn resize_phase(&mut self, ctx: &FrameCtx) -> Option<f64> {
if !ctx.resizing {
self.resizing_since = None;
return None;
}
Some(
self.resizing_since
.get_or_insert_with(Instant::now)
.elapsed()
.as_secs_f64(),
)
}
/// Make `slots[i]` a render target of exactly `width`×`height` (rebuilt on resize). /// Make `slots[i]` a render target of exactly `width`×`height` (rebuilt on resize).
fn ensure_slot(&mut self, i: usize, width: u32, height: u32) -> Result<()> { fn ensure_slot(&mut self, i: usize, width: u32, height: u32) -> Result<()> {
if self.slots[i] if self.slots[i]
@@ -540,6 +575,33 @@ fn draw_osd_panel(canvas: &Canvas, font: &Font, text: &str, x: f32, y: f32) {
} }
} }
/// The mid-stream-resize cover: a full-screen dark scrim, the shared rotating spinner, and
/// a "Resizing…" label centered over it — so the host's 0.32 s virtual-display + encoder
/// rebuild reads as a deliberate pause rather than the stream stretching to the changed
/// window. This is the presenter's analog of the Apple client's blur overlay: the overlay
/// composites its own RGBA quad and cannot sample the video to blur it, so an opaque scrim
/// hides the stretched in-between frame instead (same intent, one draw).
fn draw_resize_scrim(canvas: &Canvas, font: &Font, width: u32, height: u32, phase: f64) {
let (wf, hf) = (width as f32, height as f32);
canvas.draw_rect(
Rect::from_wh(wf, hf),
&Paint::new(Color4f::new(0.0, 0.0, 0.0, 0.55), None),
);
// Spinner slightly above center; the label sits below it.
let (cx, cy) = (f64::from(width) / 2.0, f64::from(height) / 2.0);
let r = (f64::from(width.min(height)) * 0.045).clamp(16.0, 44.0);
crate::theme::spinner(canvas, cx, cy - r, r, phase);
let (_, metrics) = font.metrics();
let label = "Resizing\u{2026}";
let tw = font.measure_str(label, None).0;
canvas.draw_str(
label,
Point::new((wf - tw) / 2.0, (cy + r * 0.9) as f32 - metrics.ascent),
font,
&Paint::new(Color4f::new(1.0, 1.0, 1.0, 0.9), None),
);
}
/// The capture hint / start banner: a centered pill near the bottom edge. /// The capture hint / start banner: a centered pill near the bottom edge.
fn draw_hint_pill(canvas: &Canvas, font: &Font, text: &str, width: u32, height: u32, alpha: f32) { fn draw_hint_pill(canvas: &Canvas, font: &Font, text: &str, width: u32, height: u32, alpha: f32) {
let (_, metrics) = font.metrics(); let (_, metrics) = font.metrics();
+5
View File
@@ -37,6 +37,11 @@ pub struct FrameCtx<'a> {
pub stats: Option<&'a str>, pub stats: Option<&'a str>,
/// The capture hint (bottom-center pill, "click to capture…"); `None` = hidden. /// The capture hint (bottom-center pill, "click to capture…"); `None` = hidden.
pub hint: Option<&'a str>, pub hint: Option<&'a str>,
/// A mid-stream Match-window resize is in flight (design/midstream-resolution-resize.md,
/// client UX): draw a full-screen scrim + spinner so the host's 0.32 s virtual-display
/// and encoder rebuild reads as an intentional pause rather than the stream stretching to
/// the changed window. Cleared the instant the sharp new-resolution frame is on glass.
pub resizing: bool,
/// The active gamepad's name (the console library's controller chip). /// The active gamepad's name (the console library's controller chip).
pub pad: Option<&'a str>, pub pad: Option<&'a str>,
/// The active pad's resolved kind — drives the console UI's button glyphs /// The active pad's resolved kind — drives the console UI's button glyphs
+449 -9
View File
@@ -52,6 +52,18 @@ pub struct SessionOpts {
/// stay stdout-only). An overlay whose `init` fails degrades to `None` with a /// stay stdout-only). An overlay whose `init` fails degrades to `None` with a
/// warning rather than killing the session. Browse mode requires one. /// warning rather than killing the session. Browse mode requires one.
pub overlay: Option<Box<dyn Overlay>>, pub overlay: Option<Box<dyn Overlay>>,
/// The window's starting logical size; `None` = the 1280×720 default. The binary
/// passes the persisted last-window size under the Match-window policy so the first
/// connect's mode already matches what the user will be looking at.
pub window_size: Option<(u32, u32)>,
/// Match-window resolution policy (design/midstream-resolution-resize.md D1/D2):
/// `Some` = the stream mode follows the window. At session start the params' mode
/// w/h are replaced by the window's physical pixel size; a mid-session resize sends
/// a debounced `Reconfigure` so the host's virtual display + encoder follow. The
/// callback receives the window's logical size at each resize-end — the binary
/// persists it for the next launch. `None` = never auto-resize (Auto-native /
/// Explicit keep today's behavior).
pub match_window: Option<Box<dyn FnMut(u32, u32)>>,
} }
pub enum Outcome { pub enum Outcome {
@@ -173,6 +185,25 @@ struct StreamState {
/// The last pump window, kept so a Ctrl+Alt+Shift+S tier cycle can re-render the /// The last pump window, kept so a Ctrl+Alt+Shift+S tier cycle can re-render the
/// OSD immediately instead of waiting up to 1 s for the next Stats event. /// OSD immediately instead of waiting up to 1 s for the next Stats event.
last_stats: Option<Stats>, last_stats: Option<Stats>,
/// Match-window (D2) debounce state: the last resize event's stamp. `Some` = a
/// resize is pending; the tick fires the request once ~400 ms pass with no further
/// size events (never per drag-frame — each accepted switch is a full host rebuild).
resize_pending: Option<Instant>,
/// When the last `Reconfigure` was sent — the ≥ 1 s spacing between requests (D2).
/// The accept ack round-trips in milliseconds (it precedes the host's rebuild), so
/// this spacing also serializes: at most ~one request is ever outstanding.
resize_sent_at: Option<Instant>,
/// The last size actually requested. Each distinct size is requested at most once:
/// this both implements "don't re-request a rejected size until it changes" (D2) and
/// keeps a host-side rollback (accept ack, rebuild failed, corrective ack restored
/// the old mode) from looping request → rollback → request forever.
resize_requested: Option<(u32, u32)>,
/// The connector mode last shown in the HUD/title — a change (an accepted switch's
/// ack, or a corrective rollback) refreshes both.
shown_mode: Option<Mode>,
/// Resize-in-progress overlay (scrim + spinner) — armed by [`resize_tick`] when it
/// requests a switch, cleared when a decoded frame reaches the target (or on timeout).
resize_overlay: ResizeIndicator,
} }
impl StreamState { impl StreamState {
@@ -217,6 +248,11 @@ impl StreamState {
hw_fails: 0, hw_fails: 0,
osd_text: String::new(), osd_text: String::new(),
last_stats: None, last_stats: None,
resize_pending: None,
resize_sent_at: None,
resize_requested: None,
shown_mode: None,
resize_overlay: ResizeIndicator::default(),
} }
} }
@@ -270,7 +306,10 @@ fn run_inner(mut opts: SessionOpts, mut mode: ModeCtl) -> Result<Option<Outcome>
.register_custom_event::<FrameWake>() .register_custom_event::<FrameWake>()
.map_err(|e| anyhow::anyhow!("register FrameWake event: {e}"))?; .map_err(|e| anyhow::anyhow!("register FrameWake event: {e}"))?;
let mut window = { let mut window = {
let mut b = video.window(&opts.window_title, 1280, 720); // Match-window (D1): open at the persisted last size, so the first connect's
// mode already matches the glass. 1280×720 stays the fallback/default.
let (ww, wh) = opts.window_size.unwrap_or((1280, 720));
let mut b = video.window(&opts.window_title, ww.max(320), wh.max(200));
match opts.window_pos { match opts.window_pos {
Some((x, y)) => b.position(x, y), Some((x, y)) => b.position(x, y),
None => b.position_centered(), None => b.position_centered(),
@@ -340,12 +379,15 @@ fn run_inner(mut opts: SessionOpts, mut mode: ModeCtl) -> Result<Option<Outcome>
let mut stream: Option<StreamState> = match &mut mode { let mut stream: Option<StreamState> = match &mut mode {
ModeCtl::Single(build) => { ModeCtl::Single(build) => {
let force_software = Arc::new(AtomicBool::new(false)); let force_software = Arc::new(AtomicBool::new(false));
let params = build( let mut params = build(
&gamepad, &gamepad,
native, native,
force_software.clone(), force_software.clone(),
presenter.vulkan_decode(), presenter.vulkan_decode(),
); );
if opts.match_window.is_some() {
apply_match_window(&mut params, &window);
}
Some(StreamState::new( Some(StreamState::new(
params, params,
force_software, force_software,
@@ -423,6 +465,14 @@ fn run_inner(mut opts: SessionOpts, mut mode: ModeCtl) -> Result<Option<Outcome>
WindowEvent::PixelSizeChanged(..) | WindowEvent::Resized(..) => { WindowEvent::PixelSizeChanged(..) | WindowEvent::Resized(..) => {
presenter.recreate_swapchain(&window)?; presenter.recreate_swapchain(&window)?;
presenter.present(&window, FrameInput::Redraw, overlay_frame.as_ref())?; presenter.present(&window, FrameInput::Redraw, overlay_frame.as_ref())?;
// Match-window (D2): (re)stamp the debounce — the request fires
// once ~400 ms pass with no further size events, never per
// drag-frame (each accepted switch is a full host rebuild).
if opts.match_window.is_some() {
if let Some(st) = stream.as_mut() {
st.resize_pending = Some(Instant::now());
}
}
} }
WindowEvent::Exposed => { WindowEvent::Exposed => {
presenter.present(&window, FrameInput::Redraw, overlay_frame.as_ref())?; presenter.present(&window, FrameInput::Redraw, overlay_frame.as_ref())?;
@@ -616,7 +666,10 @@ fn run_inner(mut opts: SessionOpts, mut mode: ModeCtl) -> Result<Option<Outcome>
presenter.vulkan_decode(), presenter.vulkan_decode(),
) { ) {
ActionOutcome::Handled => {} ActionOutcome::Handled => {}
ActionOutcome::Start(params) => { ActionOutcome::Start(mut params) => {
if opts.match_window.is_some() {
apply_match_window(&mut params, &window);
}
stream = Some(StreamState::new( stream = Some(StreamState::new(
*params, *params,
force_software, force_software,
@@ -753,6 +806,18 @@ fn run_inner(mut opts: SessionOpts, mut mode: ModeCtl) -> Result<Option<Outcome>
} }
} }
// --- Match-window (D2): debounced mode-follow + HUD/title refresh on a switch ----
if let Some(persist) = opts.match_window.as_mut() {
if let Some(st) = stream.as_mut() {
resize_tick(st, &mut window, &opts.window_title, persist.as_mut());
}
}
// Resize overlay timeout: a switch the host rejected/capped never delivers the exact
// target frame — drop the scrim so it can't linger. A no-op unless one is showing.
if let Some(st) = stream.as_mut() {
st.resize_overlay.tick(Instant::now());
}
// --- Console UI: damage-driven overlay re-render for this iteration -------------- // --- Console UI: damage-driven overlay re-render for this iteration --------------
if let Some(o) = overlay.as_mut() { if let Some(o) = overlay.as_mut() {
let (pw, ph) = window.size_in_pixels(); let (pw, ph) = window.size_in_pixels();
@@ -776,11 +841,15 @@ fn run_inner(mut opts: SessionOpts, mut mode: ModeCtl) -> Result<Option<Outcome>
}; };
let pad = gamepad.active(); let pad = gamepad.active();
let pads = gamepad.pads(); let pads = gamepad.pads();
let resizing = stream
.as_ref()
.is_some_and(|st| st.connector.is_some() && st.resize_overlay.active());
let ctx = FrameCtx { let ctx = FrameCtx {
width: pw, width: pw,
height: ph, height: ph,
stats, stats,
hint, hint,
resizing,
pad: pad.as_ref().map(|p| p.name.as_str()), pad: pad.as_ref().map(|p| p.name.as_str()),
pad_pref: pad.as_ref().map(|p| p.pref), pad_pref: pad.as_ref().map(|p| p.pref),
pads: &pads, pads: &pads,
@@ -814,6 +883,10 @@ fn run_inner(mut opts: SessionOpts, mut mode: ModeCtl) -> Result<Option<Outcome>
newest = Some(f); newest = Some(f);
} }
if let Some(f) = newest { if let Some(f) = newest {
// Resize END: a frame at the steered target size means the sharp new-mode
// picture is here — lift the scrim. A no-op unless a switch is in flight.
let (fw, fh) = f.image.dimensions();
st.resize_overlay.decoded(fw, fh);
let DecodedFrame { let DecodedFrame {
pts_ns, pts_ns,
decoded_ns, decoded_ns,
@@ -991,12 +1064,15 @@ fn run_inner(mut opts: SessionOpts, mut mode: ModeCtl) -> Result<Option<Outcome>
} }
} }
// Browse with no video driving presents (library / connecting): composite the // Composite the overlay every iteration when no video frame drove a present but
// overlay every iteration — FIFO vsync-throttles this to the display rate. // something on-screen still animates: browse-idle (library / connecting), OR a
if matches!(mode, ModeCtl::Browse(_)) // mid-stream resize scrim + spinner (the host's virtual-display + encoder rebuild
&& !presented_video // leaves a gap with no frames — without this the spinner would freeze). FIFO
&& stream.as_ref().is_none_or(|s| s.connector.is_none()) // vsync-throttles this to the display rate; the 15 ms wait keeps it smooth.
{ let resize_scrim = stream.as_ref().is_some_and(|s| s.resize_overlay.active());
let browse_idle = matches!(mode, ModeCtl::Browse(_))
&& stream.as_ref().is_none_or(|s| s.connector.is_none());
if !presented_video && (resize_scrim || browse_idle) {
presenter.present(&window, FrameInput::Redraw, overlay_frame.as_ref())?; presenter.present(&window, FrameInput::Redraw, overlay_frame.as_ref())?;
} }
}; };
@@ -1013,6 +1089,195 @@ fn run_inner(mut opts: SessionOpts, mut mode: ModeCtl) -> Result<Option<Outcome>
Ok(outcome) Ok(outcome)
} }
/// Match-window (D1): replace the params' requested w/h with the window's physical pixel
/// size — even-floored (the host's `validate_dimensions` rejects odd) and clamped to a
/// sane minimum — keeping the resolved refresh. Under `--fullscreen` the window IS the
/// display, so this degenerates to the display's native mode.
fn apply_match_window(params: &mut SessionParams, window: &sdl3::video::Window) {
let (pw, ph) = window.size_in_pixels();
params.mode.width = (pw & !1).max(320);
params.mode.height = (ph & !1).max(200);
tracing::info!(
w = params.mode.width,
h = params.mode.height,
"match-window: requesting the window's pixel size"
);
}
/// Match-window (D2) per-iteration tick: refresh the HUD line + window title when the
/// live mode moves (an accepted switch's ack, or a corrective rollback), then fire the
/// debounced `Reconfigure` once ~400 ms pass with no further resize events. The shared
/// trigger discipline:
/// * physical pixels, even-floored, clamped ≥ 320×200; the current refresh is kept;
/// * ≥ 1 s between requests (the accept ack round-trips in milliseconds — it precedes
/// the host's rebuild — so the spacing also keeps at most ~one request outstanding);
/// * each distinct size is requested at most ONCE (`resize_requested`): a rejected
/// size isn't re-asked until the window changes, and a host-side rollback (accepted,
/// rebuild failed, corrective ack restored the old mode) can't loop.
fn resize_tick(
st: &mut StreamState,
window: &mut sdl3::video::Window,
title_base: &str,
persist: &mut dyn FnMut(u32, u32),
) {
let Some(c) = &st.connector else {
return; // not connected yet — the pending stamp survives until we are
};
// HUD/title follow the live mode slot (updated by any accepted ack).
let m = c.mode();
if st.shown_mode.is_some_and(|prev| prev != m) {
st.mode_line = format!("{}×{}@{}", m.width, m.height, m.refresh_hz);
tracing::info!(mode = %st.mode_line, "stream mode switched");
let _ = window.set_title(&format!("{title_base} · {}", st.mode_line));
}
st.shown_mode = Some(m);
match resize_decision(
Instant::now(),
&mut st.resize_pending,
st.resize_sent_at,
st.resize_requested,
(m.width, m.height),
window.size_in_pixels(),
) {
ResizeAction::Wait => {}
ResizeAction::Settled(target) => {
// The debounce settled: persist the window's LOGICAL size for the next
// launch (its window is created in logical units) even when no request goes
// out (e.g. resized back to the streamed size).
let (lw, lh) = window.size();
persist(lw, lh);
let Some((w, h)) = target else { return };
tracing::info!(w, h, "window resized — requesting mode switch");
if c.request_mode(Mode {
width: w,
height: h,
refresh_hz: m.refresh_hz,
})
.is_err()
{
tracing::warn!("mode-switch request dropped — control channel closed");
}
st.resize_requested = Some((w, h));
st.resize_sent_at = Some(Instant::now());
// Show the scrim + spinner until a frame at this size lands (or the timeout):
// the live drag itself stays sharp; only the host's rebuild gap is covered.
st.resize_overlay.steering(w, h, Instant::now());
}
}
}
/// What one [`resize_decision`] tick decided.
#[derive(Debug, PartialEq, Eq)]
enum ResizeAction {
/// Nothing to do yet (no resize pending, still debouncing, or spacing defers — the
/// pending stamp is kept so a later tick retries).
Wait,
/// The debounce settled (pending cleared, the caller persists the window size), with
/// the mode to request — `None` when the size needs no switch (equal to the streamed
/// mode, or this exact size was already requested once).
Settled(Option<(u32, u32)>),
}
/// The D2 trigger discipline as a pure decision (unit-tested — CI can't open windows):
/// debounce to resize-end, ≥ 1 s between requests, physical pixels even-floored and
/// clamped ≥ 320×200, skip when equal to the streamed mode, and each distinct size
/// requested at most once (covers rejected sizes AND host-side rollbacks).
fn resize_decision(
now: Instant,
pending: &mut Option<Instant>,
sent_at: Option<Instant>,
requested: Option<(u32, u32)>,
current: (u32, u32),
pixel_size: (u32, u32),
) -> ResizeAction {
const DEBOUNCE: Duration = Duration::from_millis(400);
const SPACING: Duration = Duration::from_secs(1);
let Some(since) = *pending else {
return ResizeAction::Wait;
};
if now.duration_since(since) < DEBOUNCE {
return ResizeAction::Wait;
}
if sent_at.is_some_and(|at| now.duration_since(at) < SPACING) {
return ResizeAction::Wait; // keep the pending stamp — a later tick retries
}
*pending = None;
let target = ((pixel_size.0 & !1).max(320), (pixel_size.1 & !1).max(200));
if current == target || requested == Some(target) {
return ResizeAction::Settled(None);
}
ResizeAction::Settled(Some(target))
}
/// Resize-in-progress overlay state (design/midstream-resolution-resize.md — client UX),
/// ported from the Apple client's `ResizeIndicator`. A mid-stream Match-window switch takes
/// the host 0.32 s to rebuild its virtual display + encoder, and the first new-mode frame
/// is an IDR the decoder re-inits on. Rather than let the stream stretch to the changed
/// window during that gap, the presenter EMBRACES the delay: a deliberate scrim + spinner
/// the instant a switch is requested, cleared the instant the sharp new-resolution frame is
/// on screen — so the wait reads as intentional, not as lag.
///
/// Driven entirely by signals the presenter already has (no new protocol):
/// * START — [`resize_tick`] reports the size it just requested (`steering`).
/// * END — the decode pipeline reports each frame's dimensions; when they reach the
/// target the new picture is here (`decoded`). The accepted-switch ack alone can't
/// end it: the ack round-trips in milliseconds, ahead of the host's rebuild.
/// * TIMEOUT — the safety net for a switch that never delivers the exact target (a
/// gamescope reject, an advertised-mode cap, or a corrective ack landing a different
/// size); `tick` clears it after [`ResizeIndicator::TIMEOUT`].
///
/// Pure + clock-injected so the transition logic is unit-tested without a live session.
#[derive(Default)]
struct ResizeIndicator {
/// The size the follower is steering toward — cleared once a decoded frame reaches it.
/// `Some` ⇔ the scrim + spinner should be shown.
target: Option<(u32, u32)>,
/// When the current active span began — the timeout is measured from here.
since: Option<Instant>,
}
impl ResizeIndicator {
/// How long to keep the overlay up if the target frame never arrives.
const TIMEOUT: Duration = Duration::from_millis(2500);
/// Whether the scrim + spinner should be shown.
fn active(&self) -> bool {
self.target.is_some()
}
/// A switch to `w`×`h` was just requested — show the overlay now. The timeout re-arms
/// only when the target actually changes, so a drag that walks through several sizes
/// (each its own request) never trips the timeout mid-gesture.
fn steering(&mut self, w: u32, h: u32, now: Instant) {
if self.target != Some((w, h)) {
self.since = Some(now);
}
self.target = Some((w, h));
}
/// A decoded frame arrived at `w`×`h`. Clears the overlay once it matches the steered
/// target — the sharp new-resolution picture is on glass.
fn decoded(&mut self, w: u32, h: u32) {
if self.target == Some((w, h)) {
self.target = None;
self.since = None;
}
}
/// Timeout safety net: stop showing once [`TIMEOUT`](Self::TIMEOUT) has elapsed with no
/// matching frame (a rejected or host-capped switch never delivers the exact target).
fn tick(&mut self, now: Instant) {
if self
.since
.is_some_and(|s| now.duration_since(s) >= Self::TIMEOUT)
{
self.target = None;
self.since = None;
}
}
}
/// Apply the capture state to the window: pointer lock (relative mouse + hidden cursor) /// Apply the capture state to the window: pointer lock (relative mouse + hidden cursor)
/// and — on Windows — a keyboard grab, so system chords (Alt+Tab, the Windows key) reach /// and — on Windows — a keyboard grab, so system chords (Alt+Tab, the Windows key) reach
/// the host while captured instead of the local shell. SDL implements the grab there /// the host while captured instead of the local shell. SDL implements the grab there
@@ -1115,6 +1380,181 @@ fn stats_text(
mod tests { mod tests {
use super::*; use super::*;
#[test]
fn resize_decision_follows_the_d2_discipline() {
let t0 = Instant::now();
let ms = Duration::from_millis;
// No resize pending → nothing to do.
let mut pending = None;
assert_eq!(
resize_decision(t0, &mut pending, None, None, (1280, 720), (1000, 600)),
ResizeAction::Wait
);
// Still debouncing (a drag in progress) → wait, pending kept.
let mut pending = Some(t0);
assert_eq!(
resize_decision(
t0 + ms(399),
&mut pending,
None,
None,
(1280, 720),
(1000, 600)
),
ResizeAction::Wait
);
assert!(pending.is_some(), "pending survives the wait");
// Debounce settled → request the even-floored, clamped pixel size.
assert_eq!(
resize_decision(
t0 + ms(400),
&mut pending,
None,
None,
(1280, 720),
(1001, 601)
),
ResizeAction::Settled(Some((1000, 600))),
"odd pixels floor to even"
);
assert!(pending.is_none(), "pending consumed");
// Spacing: a request went out < 1 s ago → wait WITHOUT dropping the pending
// stamp, so a later tick retries.
let mut pending = Some(t0);
assert_eq!(
resize_decision(
t0 + ms(900),
&mut pending,
Some(t0),
Some((1000, 600)),
(1280, 720),
(800, 500)
),
ResizeAction::Wait
);
assert!(pending.is_some());
assert_eq!(
resize_decision(
t0 + ms(1000),
&mut pending,
Some(t0),
Some((1000, 600)),
(1280, 720),
(800, 500)
),
ResizeAction::Settled(Some((800, 500)))
);
// Equal to the streamed mode → settle (persist) but no request.
let mut pending = Some(t0);
assert_eq!(
resize_decision(
t0 + ms(400),
&mut pending,
None,
None,
(1280, 720),
(1280, 720)
),
ResizeAction::Settled(None)
);
// A size already requested once (rejected, or rolled back host-side) is never
// re-asked — no request → rollback → request loop.
let mut pending = Some(t0);
assert_eq!(
resize_decision(
t0 + ms(400),
&mut pending,
None,
Some((1000, 600)),
(1280, 720),
(1000, 600)
),
ResizeAction::Settled(None)
);
// Tiny windows clamp to the host's floor.
let mut pending = Some(t0);
assert_eq!(
resize_decision(
t0 + ms(400),
&mut pending,
None,
None,
(1280, 720),
(100, 80)
),
ResizeAction::Settled(Some((320, 200)))
);
}
#[test]
fn resize_indicator_shows_until_the_target_frame_or_timeout() {
let t0 = Instant::now();
let ms = Duration::from_millis;
// Idle at rest.
let mut ind = ResizeIndicator::default();
assert!(!ind.active());
// A requested switch shows the overlay immediately.
ind.steering(1000, 600, t0);
assert!(ind.active());
// A frame at a DIFFERENT size (a stale old-mode frame still draining) doesn't lift it.
ind.decoded(1280, 720);
assert!(ind.active(), "an off-target frame keeps the scrim up");
// The sharp new-resolution frame arrives → cleared.
ind.decoded(1000, 600);
assert!(!ind.active(), "the target frame lifts the scrim");
ind.tick(t0 + ms(10_000)); // a late tick after clearing is inert
assert!(!ind.active());
// A switch whose target frame never arrives (rejected / host-capped) times out.
let mut ind = ResizeIndicator::default();
ind.steering(1000, 600, t0);
ind.tick(t0 + ResizeIndicator::TIMEOUT - ms(1));
assert!(ind.active(), "still within the timeout window");
ind.tick(t0 + ResizeIndicator::TIMEOUT);
assert!(!ind.active(), "timeout lifts a switch that never delivered");
}
#[test]
fn resize_indicator_retargets_and_rearms_the_timeout_mid_drag() {
let t0 = Instant::now();
let ms = Duration::from_millis;
// A drag that walks through sizes (each a fresh request) re-arms the timeout, so a
// slow gesture never trips it: at t0 steer A, then near-timeout steer B, then a B
// frame lands well after A's timeout would have fired.
let mut ind = ResizeIndicator::default();
ind.steering(1000, 600, t0);
let near = t0 + ResizeIndicator::TIMEOUT - ms(1);
ind.steering(1200, 700, near); // new target → timeout re-armed from `near`
ind.tick(t0 + ResizeIndicator::TIMEOUT + ms(1)); // past A's window, within B's
assert!(
ind.active(),
"retarget re-armed the timeout — no mid-drag flicker"
);
// Re-steering the SAME size does NOT re-arm (so a repeated identical request can't
// hold the scrim open forever).
let mut ind = ResizeIndicator::default();
ind.steering(1000, 600, t0);
ind.steering(1000, 600, t0 + ms(500)); // same target, later — `since` unchanged
ind.tick(t0 + ResizeIndicator::TIMEOUT);
assert!(
!ind.active(),
"an unchanged target keeps the original timeout"
);
}
fn sample() -> (Stats, PresentedWindow) { fn sample() -> (Stats, PresentedWindow) {
( (
Stats { Stats {
+205
View File
@@ -13,6 +13,7 @@
use crate::config::{Config, FecConfig, FecScheme, ProtocolPhase, Role}; use crate::config::{Config, FecConfig, FecScheme, ProtocolPhase, Role};
use crate::error::PunktfunkStatus; use crate::error::PunktfunkStatus;
use crate::input::InputEvent; use crate::input::InputEvent;
use crate::reanchor::{GateVerdict, ReanchorGate};
use crate::session::Session; use crate::session::Session;
use crate::stats::Stats; use crate::stats::Stats;
use crate::transport::{loopback_pair, Transport, UdpTransport}; use crate::transport::{loopback_pair, Transport, UdpTransport};
@@ -2405,6 +2406,70 @@ pub unsafe extern "C" fn punktfunk_connection_request_keyframe(
}) })
} }
/// Ask the host to recover from loss by **reference-frame invalidation** rather than a full IDR:
/// report the range `[first_frame, last_frame]` of access units the client can no longer trust
/// (the first missing `frame_index` through the newest received). An RFI-capable host (AMD LTR /
/// NVENC) re-references a known-good picture before `first_frame` and emits a clean P-frame tagged
/// `USER_FLAG_RECOVERY_ANCHOR` — no 20-40x IDR spike; a host that can't RFI forces an IDR instead
/// (same effect as [`punktfunk_connection_request_keyframe`]). Non-blocking, fire-and-forget; the
/// recovered frame is the only ack, so THROTTLE it exactly like the keyframe request. Prefer this
/// over the keyframe request on loss so AMD/RFI hosts avoid the spike; keep the keyframe request as
/// the backstop for when the recovery frame itself is lost.
///
/// # Safety
/// `c` is a valid connection handle.
#[cfg(feature = "quic")]
#[no_mangle]
pub unsafe extern "C" fn punktfunk_connection_request_rfi(
c: *const PunktfunkConnection,
first_frame: u32,
last_frame: u32,
) -> PunktfunkStatus {
guard(|| {
let c = match unsafe { c.as_ref() } {
Some(c) => c,
None => return PunktfunkStatus::NullPointer,
};
match c.inner.request_rfi(first_frame, last_frame) {
Ok(()) => PunktfunkStatus::Ok,
Err(e) => e.status(),
}
})
}
/// Feed each received frame's `frame_index` (the [`PunktfunkFrame::frame_index`] field, in receive
/// order) so the client recovers from loss with a cheap reference-frame invalidation instead of a
/// full IDR. On a forward gap (a `frame_index` jump = the intervening frames were lost and the
/// following AUs reference a picture that never arrived) this fires a THROTTLED
/// [`punktfunk_connection_request_rfi`] for the lost range; an RFI-capable host (AMD LTR / NVENC)
/// then recovers with a clean P-frame instead of a 20-40x IDR spike. Call it for every received
/// frame — it is cheap and idempotent, and the [`punktfunk_connection_frames_dropped`]-driven
/// keyframe request stays the backstop. Writes whether a forward gap was detected this call to
/// `gap_out` (nullable — a client with a post-loss display freeze can use it to re-arm; most
/// clients pass NULL and ignore it).
///
/// # Safety
/// `c` is a valid connection handle; `gap_out` is writable or NULL.
#[cfg(feature = "quic")]
#[no_mangle]
pub unsafe extern "C" fn punktfunk_connection_note_frame_index(
c: *const PunktfunkConnection,
frame_index: u32,
gap_out: *mut bool,
) -> PunktfunkStatus {
guard(|| {
let c = match unsafe { c.as_ref() } {
Some(c) => c,
None => return PunktfunkStatus::NullPointer,
};
let gap = c.inner.note_frame_index(frame_index);
if !gap_out.is_null() {
unsafe { *gap_out = gap };
}
PunktfunkStatus::Ok
})
}
/// Cumulative access units the host→client reassembler dropped as unrecoverable (FEC couldn't /// Cumulative access units the host→client reassembler dropped as unrecoverable (FEC couldn't
/// rebuild them). A video loop polls this and calls [`punktfunk_connection_request_keyframe`] /// rebuild them). A video loop polls this and calls [`punktfunk_connection_request_keyframe`]
/// when it climbs — the correct loss trigger under the host's infinite GOP, where unrecoverable /// when it climbs — the correct loss trigger under the host's infinite GOP, where unrecoverable
@@ -2556,3 +2621,143 @@ pub unsafe extern "C" fn punktfunk_connection_close(c: *mut PunktfunkConnection)
drop(unsafe { Box::from_raw(c) }); drop(unsafe { Box::from_raw(c) });
} }
} }
// ---- Post-loss re-anchor freeze gate ----
//
// The shared [`ReanchorGate`](crate::reanchor::ReanchorGate) exposed for the Swift client (Rust
// embedders — Android/Windows/Linux — use the struct directly). After an unrecoverable reference
// loss the decoder silently conceals the missing-reference deltas (gray/garbage picture, no error);
// the client freezes on the last good frame and lifts only on a proven clean re-anchor. The gate
// takes time internally (`Instant::now`) so no timestamps cross the boundary. Drive it per session:
// `arm` on a loss (frame-index gap from `punktfunk_connection_note_frame_index`, a decoder
// wedge/demotion), `on_decoded` per decoded frame to gate presentation, `on_no_output` per AU that
// produced nothing, and `poll` each iteration for the dropped-count climb + overdue backstop. Route
// the returned keyframe intents through the client's existing request throttle.
/// Create a re-anchor gate seeded with the session's current `frames_dropped` (so the first
/// [`punktfunk_reanchor_gate_poll`] doesn't read the baseline as a loss). Free with
/// [`punktfunk_reanchor_gate_free`]. Never returns NULL.
#[no_mangle]
pub extern "C" fn punktfunk_reanchor_gate_new(frames_dropped: u64) -> *mut ReanchorGate {
Box::into_raw(Box::new(ReanchorGate::new(frames_dropped)))
}
/// Free a gate created by [`punktfunk_reanchor_gate_new`]. NULL is a no-op.
///
/// # Safety
/// `g` was returned by [`punktfunk_reanchor_gate_new`] and is not used after this call.
#[no_mangle]
pub unsafe extern "C" fn punktfunk_reanchor_gate_free(g: *mut ReanchorGate) {
if !g.is_null() {
drop(unsafe { Box::from_raw(g) });
}
}
/// Arm the freeze: a loss was detected (a frame-index gap, or a decoder wedge/demotion). Zeroes the
/// recovery-mark count and (re-)sets the backstop deadline. NULL is a no-op.
///
/// # Safety
/// `g` is a valid gate handle.
#[no_mangle]
pub unsafe extern "C" fn punktfunk_reanchor_gate_arm(g: *mut ReanchorGate) {
if let Some(g) = unsafe { g.as_mut() } {
g.arm(std::time::Instant::now());
}
}
/// Fold one decoded frame and write to `out_present` whether to display it (`true`) or withhold it as
/// a post-loss concealment (`false`). `flags` is the AU's `user_flags` word ([`PunktfunkFrame::flags`]):
/// the gate reads `FLAG_SOF` (the host's IDR marker), `USER_FLAG_RECOVERY_ANCHOR` and
/// `USER_FLAG_RECOVERY_POINT`. Pass `decoder_keyframe = false` where the platform decoder doesn't flag
/// IDRs (VideoToolbox/MediaCodec) — the wire `FLAG_SOF` covers it.
///
/// # Safety
/// `g` is a valid gate handle; `out_present` is writable or NULL.
#[no_mangle]
pub unsafe extern "C" fn punktfunk_reanchor_gate_on_decoded(
g: *mut ReanchorGate,
flags: u32,
decoder_keyframe: bool,
out_present: *mut bool,
) -> PunktfunkStatus {
guard(|| {
let g = match unsafe { g.as_mut() } {
Some(g) => g,
None => return PunktfunkStatus::NullPointer,
};
let present = g.on_decoded(flags, decoder_keyframe, std::time::Instant::now())
== GateVerdict::Present;
if !out_present.is_null() {
unsafe { *out_present = present };
}
PunktfunkStatus::Ok
})
}
/// A received AU produced no decoded frame. Writes to `out_request_kf` whether the no-output streak has
/// tripped and the client should (throttled) request a keyframe — the gate arms the freeze at the same
/// time.
///
/// # Safety
/// `g` is a valid gate handle; `out_request_kf` is writable or NULL.
#[no_mangle]
pub unsafe extern "C" fn punktfunk_reanchor_gate_on_no_output(
g: *mut ReanchorGate,
out_request_kf: *mut bool,
) -> PunktfunkStatus {
guard(|| {
let g = match unsafe { g.as_mut() } {
Some(g) => g,
None => return PunktfunkStatus::NullPointer,
};
let request = g.on_no_output(std::time::Instant::now());
if !out_request_kf.is_null() {
unsafe { *out_request_kf = request };
}
PunktfunkStatus::Ok
})
}
/// Periodic fold of the session's `frames_dropped` counter plus the overdue backstop. Writes to
/// `out_request_kf` whether the client should (throttled) request a keyframe (a drop-count climb armed
/// a fresh freeze, or the freeze is overdue and re-asks while it keeps holding).
///
/// # Safety
/// `g` is a valid gate handle; `out_request_kf` is writable or NULL.
#[no_mangle]
pub unsafe extern "C" fn punktfunk_reanchor_gate_poll(
g: *mut ReanchorGate,
frames_dropped: u64,
out_request_kf: *mut bool,
) -> PunktfunkStatus {
guard(|| {
let g = match unsafe { g.as_mut() } {
Some(g) => g,
None => return PunktfunkStatus::NullPointer,
};
let request = g.poll(frames_dropped, std::time::Instant::now());
if !out_request_kf.is_null() {
unsafe { *out_request_kf = request };
}
PunktfunkStatus::Ok
})
}
/// Whether the gate is currently withholding concealed frames (frozen on the last good picture).
/// Writes `false` on a NULL gate.
///
/// # Safety
/// `g` is a valid gate handle; `out_holding` is writable or NULL.
#[no_mangle]
pub unsafe extern "C" fn punktfunk_reanchor_gate_is_holding(
g: *const ReanchorGate,
out_holding: *mut bool,
) -> PunktfunkStatus {
guard(|| {
let holding = unsafe { g.as_ref() }.is_some_and(ReanchorGate::is_holding);
if !out_holding.is_null() {
unsafe { *out_holding = holding };
}
PunktfunkStatus::Ok
})
}
+361 -5
View File
@@ -19,7 +19,8 @@ use crate::packet::FLAG_PROBE;
use crate::quic::{ use crate::quic::{
accept_resync, endpoint, io, wall_clock_ns, window_loss_ppm, BitrateChanged, ClockEcho, accept_resync, endpoint, io, wall_clock_ns, window_loss_ppm, BitrateChanged, ClockEcho,
ClockResync, ColorInfo, HdrMeta, Hello, HidOutput, LossReport, ProbeRequest, ProbeResult, ClockResync, ColorInfo, HdrMeta, Hello, HidOutput, LossReport, ProbeRequest, ProbeResult,
Reconfigure, Reconfigured, RequestKeyframe, ResyncStep, RichInput, SetBitrate, Start, Welcome, Reconfigure, Reconfigured, RequestKeyframe, ResyncStep, RfiRequest, RichInput, SetBitrate,
Start, Welcome,
}; };
use crate::session::{Frame, Session}; use crate::session::{Frame, Session};
use crate::transport::UdpTransport; use crate::transport::UdpTransport;
@@ -49,6 +50,10 @@ enum CtrlRequest {
Mode(Mode), Mode(Mode),
Probe(ProbeRequest), Probe(ProbeRequest),
Keyframe, Keyframe,
/// Reference-frame-invalidation recovery: the client saw a `frame_index` gap and reports the
/// invalidation range so an RFI-capable host re-references a known-good picture instead of
/// forcing a full IDR. See [`RfiRequest`].
Rfi(RfiRequest),
Loss(LossReport), Loss(LossReport),
/// Adaptive bitrate: ask the host to re-target its encoder (kbps). Sent by the pump's /// Adaptive bitrate: ask the host to re-target its encoder (kbps). Sent by the pump's
/// [`BitrateController`] when the user's bitrate setting is Automatic. /// [`BitrateController`] when the user's bitrate setting is Automatic.
@@ -355,6 +360,83 @@ pub struct AudioPacket {
pub data: Vec<u8>, pub data: Vec<u8>,
} }
/// At most one client→host RFI request per this window, so a burst of frame-index gaps (a
/// full-screen pan shedding shards) can't storm the control stream. Matches the shared Vulkan pump's
/// recovery-request throttle; the host coalesces further.
const RFI_THROTTLE: Duration = Duration::from_millis(100);
/// State for [`NativeClient::note_frame_index`] — the client-side loss-range detector shared by every
/// embedder (Android, the C-ABI Apple client, the Windows shell pump) so none re-derives the wrapping
/// frame-index arithmetic. `next_expected` is the `frame_index` expected next in receive order;
/// `last_req` throttles the RFI requests a gap fires.
#[derive(Default)]
struct RfiRecovery {
next_expected: Option<u32>,
last_req: Option<Instant>,
}
/// What a forward gap should ask the host for: a precise RFI for a recoverable range, a plain
/// keyframe for a range wider than any encoder's reference history
/// ([`crate::packet::RFI_MAX_RANGE`] — a seconds-long outage, or a phantom index jump such as an
/// old host's speed-test burst consuming video indexes), or nothing (contiguous / straggler /
/// throttled).
#[derive(Debug, PartialEq, Eq)]
enum RecoveryAsk {
None,
Rfi(u32, u32),
Keyframe,
}
impl RfiRecovery {
/// Pure decision behind [`NativeClient::note_frame_index`]: fold one received `frame_index` (in
/// receive order) observed at `now`, advancing the expectation and returning `(gap, ask)`.
/// `gap` is whether this frame revealed a forward gap (the embedder arms its post-loss display
/// freeze on it); `ask` is the (throttled) recovery request to fire — an RFI naming the exact
/// lost span, or a keyframe when the span exceeds [`crate::packet::RFI_MAX_RANGE`] (RFI is
/// hopeless there: no encoder holds references that old, and a huge jump is more likely a
/// resync — e.g. the first real AU after an old host's speed test — than a real loss). Split
/// out from the connection so the wrapping arithmetic + [`RFI_THROTTLE`] are unit-testable
/// without a live session (see the tests below).
fn observe(&mut self, frame_index: u32, now: Instant) -> (bool, RecoveryAsk) {
match self.next_expected {
Some(exp) => {
// Wrapping split at the half-space: a small positive delta is a forward gap
// (missing frames); a delta in the top half is a straggler behind us.
let ahead = frame_index.wrapping_sub(exp);
if ahead == 0 {
self.next_expected = Some(frame_index.wrapping_add(1)); // contiguous
(false, RecoveryAsk::None)
} else if ahead < u32::MAX / 2 {
// Forward gap: [exp, frame_index-1] lost. Advance past this frame so the same
// gap isn't re-detected, then fire a throttled recovery ask for the lost range.
self.next_expected = Some(frame_index.wrapping_add(1));
let send = self
.last_req
.is_none_or(|t| now.duration_since(t) >= RFI_THROTTLE);
if send {
self.last_req = Some(now);
}
let ask = if !send {
RecoveryAsk::None
} else if ahead > crate::packet::RFI_MAX_RANGE {
RecoveryAsk::Keyframe
} else {
RecoveryAsk::Rfi(exp, frame_index.wrapping_sub(1))
};
(true, ask)
} else {
// Straggler behind the delivery point — leave the expectation.
(false, RecoveryAsk::None)
}
}
None => {
self.next_expected = Some(frame_index.wrapping_add(1));
(false, RecoveryAsk::None)
}
}
}
}
pub struct NativeClient { pub struct NativeClient {
// Each plane's receiver sits behind its own mutex so `NativeClient` is `Sync` and Rust // Each plane's receiver sits behind its own mutex so `NativeClient` is `Sync` and Rust
// embedders can share one `Arc<NativeClient>` across their plane threads (the same // embedders can share one `Arc<NativeClient>` across their plane threads (the same
@@ -398,6 +480,10 @@ pub struct NativeClient {
/// for the client stats HUDs (the unified spec's per-window `FEC` counter — proof FEC is /// for the client stats HUDs (the unified spec's per-window `FEC` counter — proof FEC is
/// earning its keep); readers window it by diffing successive reads. /// earning its keep); readers window it by diffing successive reads.
fec_recovered: Arc<AtomicU64>, fec_recovered: Arc<AtomicU64>,
/// Client-side RFI-on-loss detector state for [`note_frame_index`](Self::note_frame_index): the
/// next `frame_index` expected in receive order + the last RFI-request time (throttle). Lets every
/// embedder share one loss-range detector instead of re-deriving the wrapping frame arithmetic.
rfi: Mutex<RfiRecovery>,
/// Kernel ids of the client's latency-critical native threads: the internal data-plane pump /// Kernel ids of the client's latency-critical native threads: the internal data-plane pump
/// (UDP receive + FEC reassembly) plus any embedder plane threads registered via /// (UDP receive + FEC reassembly) plus any embedder plane threads registered via
/// [`NativeClient::register_hot_thread`]. The Android client feeds these to an ADPF hint session /// [`NativeClient::register_hot_thread`]. The Android client feeds these to an ADPF hint session
@@ -684,6 +770,7 @@ impl NativeClient {
worker: Some(worker), worker: Some(worker),
frames_dropped, frames_dropped,
fec_recovered, fec_recovered,
rfi: Mutex::new(RfiRecovery::default()),
hot_tids, hot_tids,
clock_offset, clock_offset,
mode: mode_slot, mode: mode_slot,
@@ -868,6 +955,62 @@ impl NativeClient {
.map_err(|_| PunktfunkError::Closed) .map_err(|_| PunktfunkError::Closed)
} }
/// Ask the host to recover from loss by **reference-frame invalidation** rather than a full IDR:
/// the client reports the range `[first_frame, last_frame]` of access units it can no longer trust
/// (from the first missing `frame_index` through the newest received). An RFI-capable host
/// re-references a known-good picture before `first_frame` (AMD LTR / NVENC RFI) and emits a clean
/// P-frame tagged [`crate::packet::USER_FLAG_RECOVERY_ANCHOR`]; a host that can't RFI forces an IDR
/// instead (same as [`request_keyframe`](Self::request_keyframe)). Non-blocking, fire-and-forget —
/// the recovered frame is the only ack; throttle it like the keyframe request. Prefer this over
/// `request_keyframe` on loss so AMD/RFI hosts avoid the IDR spike; the keyframe request remains
/// the backstop when the recovery frame itself is lost.
pub fn request_rfi(&self, first_frame: u32, last_frame: u32) -> Result<()> {
self.ctrl_tx
.try_send(CtrlRequest::Rfi(RfiRequest {
first_frame,
last_frame,
}))
.map_err(|_| PunktfunkError::Closed)
}
/// Feed each received AU's `frame_index` (in receive order) so the client recovers from loss with
/// a cheap reference-frame invalidation instead of always paying for a full IDR. On a **forward
/// gap** — a `frame_index` jump means the intervening frames were lost and the following AUs
/// reference a picture the decoder never got — this fires a **throttled**
/// [`request_rfi`](Self::request_rfi) for the lost range `[first_missing, frame_index-1]`. An
/// RFI-capable host (AMD LTR / NVENC) then re-references a known-good frame (a clean P-frame, no
/// 20-40x IDR spike); a host that can't RFI forces an IDR, same as the keyframe path.
///
/// Call it for EVERY received frame; it is cheap and idempotent, and the
/// [`frames_dropped`](Self::frames_dropped)-driven [`request_keyframe`](Self::request_keyframe)
/// loop stays the backstop for when the recovery frame itself is lost. Returns `true` when a
/// forward gap was detected on this call (whether or not the RFI was throttled), so a client with
/// a post-loss display freeze can (re-)arm it on the same signal.
///
/// This centralizes the loss-range detection so every embedder gets identical behavior. (The
/// in-process Vulkan session pump keeps its own copy because it gates a display freeze on the same
/// signal and shares one throttle across RFI + keyframe requests.)
pub fn note_frame_index(&self, frame_index: u32) -> bool {
// Decide (and update state) under the lock; fire the request after releasing it.
let (gap, ask) = self
.rfi
.lock()
.unwrap()
.observe(frame_index, Instant::now());
match ask {
RecoveryAsk::Rfi(first, last) => {
let _ = self.request_rfi(first, last);
}
// A gap wider than any encoder's reference history (RFI_MAX_RANGE) — a seconds-long
// outage or a phantom index jump: RFI can't repair it, resync on a keyframe instead.
RecoveryAsk::Keyframe => {
let _ = self.request_keyframe();
}
RecoveryAsk::None => {}
}
gap
}
/// Cumulative access units the host→client reassembler dropped as unrecoverable (FEC couldn't /// Cumulative access units the host→client reassembler dropped as unrecoverable (FEC couldn't
/// rebuild them). A video loop polls this and calls [`request_keyframe`](Self::request_keyframe) /// rebuild them). A video loop polls this and calls [`request_keyframe`](Self::request_keyframe)
/// when it increases — the correct loss trigger under infinite GOP, where unrecoverable loss /// when it increases — the correct loss trigger under infinite GOP, where unrecoverable loss
@@ -1297,7 +1440,12 @@ async fn worker_main(args: WorkerArgs) {
// when the matching bit is set, so `0` stays an 8-bit BT.709 stream. HOST_TIMING is // when the matching bit is set, so `0` stays an 8-bit BT.709 stream. HOST_TIMING is
// OR'd in unconditionally: every NativeClient build demuxes the 0xCF plane, and the // OR'd in unconditionally: every NativeClient build demuxes the 0xCF plane, and the
// bit only asks the host for observability datagrams (never changes the encode). // bit only asks the host for observability datagrams (never changes the encode).
video_caps: video_caps | crate::quic::VIDEO_CAP_HOST_TIMING, // PROBE_SEQ likewise: the shared reassembler keeps probe filler in its own window
// (every embedder inherits it), so the host may burst speed tests without consuming
// video frame indexes.
video_caps: video_caps
| crate::quic::VIDEO_CAP_HOST_TIMING
| crate::quic::VIDEO_CAP_PROBE_SEQ,
// Requested surround channel count; the host echoes the resolved value in Welcome. // Requested surround channel count; the host echoes the resolved value in Welcome.
audio_channels, audio_channels,
// The codecs this client can decode + its soft preference (0 = auto). The host // The codecs this client can decode + its soft preference (0 = auto). The host
@@ -1425,6 +1573,23 @@ async fn worker_main(args: WorkerArgs) {
// Touched pads only: an entry appears on the first gamepad event for that index, so the // Touched pads only: an entry appears on the first gamepad event for that index, so the
// refresh never conjures a virtual pad the embedder didn't drive. // refresh never conjures a virtual pad the embedder didn't drive.
let mut pads: [Option<GamepadSnapshot>; MAX_PADS] = [None; MAX_PADS]; let mut pads: [Option<GamepadSnapshot>; MAX_PADS] = [None; MAX_PADS];
// Per-pad wrapping seq that PERSISTS across a pad's remove/re-add on the same index (the
// snapshot itself is cleared to `None` on removal). A removal takes `seq[idx] + 1` so it
// supersedes every prior snapshot; the re-added pad's first snapshot takes the next value
// after that, so the host's seq gate accepts it instead of rejecting a restarted-at-0 seq.
let mut seq: [u8; MAX_PADS] = [0; MAX_PADS];
// Re-sends of a removal still owed on refresh ticks (the removal rides the lossy datagram
// plane; a single lost one would silently strand a ghost pad on the host — the exact bug
// the removal fixes). Mirrors the host's rumble stop burst: a few time-spread re-sends,
// each with a fresh (higher) seq, and canceled the moment the pad is driven again.
const REMOVE_RESENDS: u8 = 2;
let mut remove_owed: [u8; MAX_PADS] = [0; MAX_PADS];
// Per-pad declared controller kind ([`GamepadArrival`]) + its owed re-sends: the host needs
// the kind before the pad's first frame to build a matching virtual device (mixed types), so
// like the removal it rides the lossy plane with a small time-spread re-send burst.
const ARRIVAL_RESENDS: u8 = 2;
let mut arrival: [Option<u8>; MAX_PADS] = [None; MAX_PADS];
let mut arrival_owed: [u8; MAX_PADS] = [0; MAX_PADS];
let mut refresh = tokio::time::interval(Duration::from_millis(100)); let mut refresh = tokio::time::interval(Duration::from_millis(100));
refresh.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); refresh.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
loop { loop {
@@ -1436,24 +1601,89 @@ async fn worker_main(args: WorkerArgs) {
&& matches!(ev.kind, InputKind::GamepadButton | InputKind::GamepadAxis) && matches!(ev.kind, InputKind::GamepadButton | InputKind::GamepadAxis)
&& idx < MAX_PADS && idx < MAX_PADS
{ {
// The pad is being driven — cancel any owed removal (a re-plug on this
// index; its fresh snapshot seq already supersedes the removal's).
remove_owed[idx] = 0;
let snap = pads[idx].get_or_insert(GamepadSnapshot { let snap = pads[idx].get_or_insert(GamepadSnapshot {
pad: idx as u8, pad: idx as u8,
..Default::default() ..Default::default()
}); });
// Unknown axis ids don't send (the host's legacy fold drops them too). // Unknown axis ids don't send (the host's legacy fold drops them too).
if snap.fold(&ev) { if snap.fold(&ev) {
snap.seq = snap.seq.wrapping_add(1); seq[idx] = seq[idx].wrapping_add(1);
snap.seq = seq[idx];
let _ = input_conn let _ = input_conn
.send_datagram(snap.to_event().encode().to_vec().into()); .send_datagram(snap.to_event().encode().to_vec().into());
} }
continue; continue;
} }
if gamepad_snapshots && ev.kind == InputKind::GamepadRemove && idx < MAX_PADS {
// Stop refreshing the pad and forward a seq-stamped removal (in the shared
// seq space) so the host tears its virtual device down and no reordered
// snapshot can resurrect it; arm the re-send burst against datagram loss.
// Drop any owed kind declaration too — a re-plug on this index sends its own.
pads[idx] = None;
arrival[idx] = None;
arrival_owed[idx] = 0;
seq[idx] = seq[idx].wrapping_add(1);
remove_owed[idx] = REMOVE_RESENDS;
let rem = crate::input::InputEvent {
flags: crate::input::encode_gamepad_remove(idx as u8, seq[idx]),
..ev
};
let _ = input_conn.send_datagram(rem.encode().to_vec().into());
continue;
}
if gamepad_snapshots && ev.kind == InputKind::GamepadArrival && idx < MAX_PADS {
// Remember the declared kind (`code`) and forward it, arming a re-send burst
// so the host learns it before the pad's first frame even under loss.
arrival[idx] = Some(ev.code as u8);
arrival_owed[idx] = ARRIVAL_RESENDS;
let _ = input_conn.send_datagram(ev.encode().to_vec().into());
continue;
}
let _ = input_conn.send_datagram(ev.encode().to_vec().into()); let _ = input_conn.send_datagram(ev.encode().to_vec().into());
} }
_ = refresh.tick() => { _ = refresh.tick() => {
for snap in pads.iter_mut().flatten() { for idx in 0..MAX_PADS {
snap.seq = snap.seq.wrapping_add(1); // Re-send an owed kind declaration (independent of whether the pad has state
// yet — it may be idle-but-connected). Idempotent on the host.
if arrival_owed[idx] > 0 {
if let Some(kind) = arrival[idx] {
arrival_owed[idx] -= 1;
let arr = crate::input::InputEvent {
kind: InputKind::GamepadArrival,
_pad: [0; 3],
code: kind as u32,
x: 0,
y: 0,
flags: idx as u32,
};
let _ = input_conn.send_datagram(arr.encode().to_vec().into());
} else {
arrival_owed[idx] = 0;
}
}
if let Some(snap) = pads[idx].as_mut() {
seq[idx] = seq[idx].wrapping_add(1);
snap.seq = seq[idx];
let _ = input_conn.send_datagram(snap.to_event().encode().to_vec().into()); let _ = input_conn.send_datagram(snap.to_event().encode().to_vec().into());
} else if remove_owed[idx] > 0 {
// Idempotent removal re-send with a fresh seq (the host drops it as a
// no-op once the pad is already gone, but a re-plug's later snapshot
// still wins by seq).
remove_owed[idx] -= 1;
seq[idx] = seq[idx].wrapping_add(1);
let rem = crate::input::InputEvent {
kind: InputKind::GamepadRemove,
_pad: [0; 3],
code: 0,
x: 0,
y: 0,
flags: crate::input::encode_gamepad_remove(idx as u8, seq[idx]),
};
let _ = input_conn.send_datagram(rem.encode().to_vec().into());
}
} }
} }
} }
@@ -1511,6 +1741,7 @@ async fn worker_main(args: WorkerArgs) {
CtrlRequest::Mode(m) => Reconfigure { mode: m }.encode(), CtrlRequest::Mode(m) => Reconfigure { mode: m }.encode(),
CtrlRequest::Probe(p) => p.encode(), CtrlRequest::Probe(p) => p.encode(),
CtrlRequest::Keyframe => RequestKeyframe.encode(), CtrlRequest::Keyframe => RequestKeyframe.encode(),
CtrlRequest::Rfi(r) => r.encode(),
CtrlRequest::Loss(r) => r.encode(), CtrlRequest::Loss(r) => r.encode(),
CtrlRequest::SetBitrate(k) => SetBitrate { bitrate_kbps: k }.encode(), CtrlRequest::SetBitrate(k) => SetBitrate { bitrate_kbps: k }.encode(),
CtrlRequest::ClockResync => { CtrlRequest::ClockResync => {
@@ -1952,6 +2183,131 @@ mod host_port_tests {
} }
} }
#[cfg(test)]
mod rfi_recovery_tests {
//! The client-side loss-range detector shared by every embedder (Android, the C-ABI Apple
//! client, the Windows shell pump). `observe` is pure over `(frame_index, now)`, so the wrapping
//! frame arithmetic and the RFI throttle are exercised here without a live session.
use super::{RecoveryAsk, RfiRecovery, RFI_THROTTLE};
use std::time::{Duration, Instant};
// A fixed base instant; offsets model the throttle window deterministically (no sleeping).
fn base() -> Instant {
Instant::now()
}
#[test]
fn first_frame_arms_without_a_gap() {
let mut r = RfiRecovery::default();
// The opening frame only seeds the expectation — there is no prior frame to be missing.
assert_eq!(r.observe(100, base()), (false, RecoveryAsk::None));
assert_eq!(r.next_expected, Some(101));
}
#[test]
fn contiguous_frames_never_gap() {
let mut r = RfiRecovery::default();
let t = base();
r.observe(100, t);
assert_eq!(r.observe(101, t), (false, RecoveryAsk::None));
assert_eq!(r.observe(102, t), (false, RecoveryAsk::None));
assert_eq!(r.observe(103, t), (false, RecoveryAsk::None));
assert_eq!(r.next_expected, Some(104));
}
#[test]
fn forward_gap_reports_the_exact_lost_range() {
let mut r = RfiRecovery::default();
let t = base();
r.observe(100, t); // expecting 101 next
// 101..=104 were lost; 105 arrived. The RFI must name exactly the missing span.
assert_eq!(r.observe(105, t), (true, RecoveryAsk::Rfi(101, 104)));
// The expectation advances past the delivered frame so the same gap can't re-fire.
assert_eq!(r.next_expected, Some(106));
}
#[test]
fn single_frame_drop_names_a_unit_range() {
let mut r = RfiRecovery::default();
let t = base();
r.observe(100, t);
// Exactly one frame (101) lost → range is the single index [101, 101].
assert_eq!(r.observe(102, t), (true, RecoveryAsk::Rfi(101, 101)));
}
#[test]
fn throttle_suppresses_bursts_then_re_opens() {
let mut r = RfiRecovery::default();
let t0 = base();
r.observe(100, t0);
// First gap fires the request and stamps the throttle.
assert_eq!(r.observe(105, t0), (true, RecoveryAsk::Rfi(101, 104)));
// A second gap 50 ms later is still a gap, but the request is throttled away.
assert_eq!(
r.observe(110, t0 + Duration::from_millis(50)),
(true, RecoveryAsk::None)
);
// Past the window, the request re-opens for the still-accurate lost span.
assert_eq!(
r.observe(120, t0 + RFI_THROTTLE + Duration::from_millis(1)),
(true, RecoveryAsk::Rfi(111, 119))
);
}
#[test]
fn stragglers_behind_the_delivery_point_are_ignored() {
let mut r = RfiRecovery::default();
let t = base();
r.observe(100, t);
r.observe(105, t); // expecting 106 next
// A reordered late arrival (103, well behind 106) is neither a gap nor a request, and it
// must not rewind the expectation — otherwise the next in-order frame would false-gap.
assert_eq!(r.observe(103, t), (false, RecoveryAsk::None));
assert_eq!(r.next_expected, Some(106));
}
#[test]
fn wraparound_is_contiguous_across_u32_max() {
let mut r = RfiRecovery::default();
let t = base();
r.observe(u32::MAX - 1, t); // expecting u32::MAX next
assert_eq!(r.observe(u32::MAX, t), (false, RecoveryAsk::None)); // contiguous, wraps to 0
assert_eq!(r.next_expected, Some(0));
assert_eq!(r.observe(0, t), (false, RecoveryAsk::None)); // still contiguous across the wrap
assert_eq!(r.next_expected, Some(1));
}
#[test]
fn gap_range_wraps_across_u32_max() {
let mut r = RfiRecovery::default();
let t = base();
r.observe(u32::MAX - 1, t); // expecting u32::MAX next
// u32::MAX was lost and 1 arrived → the lost span wraps: [u32::MAX, 0].
assert_eq!(r.observe(1, t), (true, RecoveryAsk::Rfi(u32::MAX, 0)));
assert_eq!(r.next_expected, Some(2));
}
#[test]
fn huge_gap_resyncs_via_keyframe_not_rfi() {
let mut r = RfiRecovery::default();
let t = base();
r.observe(100, t); // expecting 101 next
// A jump wider than any encoder's reference history (RFI_MAX_RANGE): no valid
// reference exists for an RFI, and the jump may be a phantom (an old host's
// speed-test burst consuming video indexes) — ask for the IDR resync instead.
let jump = 100 + crate::packet::RFI_MAX_RANGE + 2;
assert_eq!(r.observe(jump, t), (true, RecoveryAsk::Keyframe));
// The expectation still advances past the delivered frame (no re-fire on the next one).
assert_eq!(r.next_expected, Some(jump + 1));
assert_eq!(r.observe(jump + 1, t), (false, RecoveryAsk::None));
// A huge gap consumes the shared throttle too — an immediate follow-up gap stays quiet.
assert_eq!(
r.observe(jump + 10, t + Duration::from_millis(1)),
(true, RecoveryAsk::None)
);
}
}
#[cfg(test)] #[cfg(test)]
mod frame_channel_tests { mod frame_channel_tests {
use super::{FrameChannel, FramePop, FRAME_QUEUE_HARD_CAP}; use super::{FrameChannel, FramePop, FRAME_QUEUE_HARD_CAP};
+55 -2
View File
@@ -51,6 +51,39 @@ pub enum InputKind {
/// [`HOST_CAP_GAMEPAD_STATE`](crate::quic::HOST_CAP_GAMEPAD_STATE); older hosts keep /// [`HOST_CAP_GAMEPAD_STATE`](crate::quic::HOST_CAP_GAMEPAD_STATE); older hosts keep
/// receiving the per-transition events. /// receiving the per-transition events.
GamepadState = 12, GamepadState = 12,
/// A pad was unplugged client-side (the native plane's answer to GameStream's
/// `activeGamepadMask`, which the per-transition/snapshot planes otherwise lack — see
/// [`encode_gamepad_remove`]). `flags` packs `seq << 24 | pad`: the low byte is the pad
/// index, the high byte a per-pad wrapping seq sharing the [`GamepadSnapshot`] sequence
/// space. The host clears the pad's `active_mask` bit so its virtual device is torn down,
/// seq-gated against snapshots so one the network reordered past the removal can't resurrect
/// the pad, and the shared seq space keeps the same index reusable by a later re-plug. Sent
/// only to a host that advertised [`HOST_CAP_GAMEPAD_STATE`](crate::quic::HOST_CAP_GAMEPAD_STATE);
/// an older host ignores the unknown tag (the pad then lingers until session end — the
/// pre-existing behaviour).
GamepadRemove = 13,
/// Declares which controller KIND a pad presents so a session can MIX types (pad 0 a
/// DualSense, pad 1 an Xbox pad). `code` = the [`GamepadPref`](crate::config::GamepadPref)
/// wire byte, `flags` = pad index. Sent when the client opens a pad slot — before that pad's
/// first input — and re-sent a few times against datagram loss (like [`GamepadRemove`]). The
/// host resolves the kind to a buildable backend and routes that pad's virtual device to it; a
/// pad the client never declares (an older client, or a fully-lost declaration) falls back to
/// the session-default kind from the handshake. Idempotent (no seq): re-declaring the same kind
/// is a no-op. Meaningful only to a host that advertised
/// [`HOST_CAP_GAMEPAD_STATE`](crate::quic::HOST_CAP_GAMEPAD_STATE); an older host ignores the
/// unknown tag (every pad then uses the session-default kind — the pre-existing behaviour).
GamepadArrival = 14,
}
/// Pack a [`InputKind::GamepadRemove`] `flags` word (`seq << 24 | pad`) — the same low-byte-pad /
/// high-byte-seq layout as [`GamepadSnapshot::to_event`], so a removal seq-gates against snapshots.
pub fn encode_gamepad_remove(pad: u8, seq: u8) -> u32 {
((seq as u32) << 24) | (pad as u32)
}
/// Unpack a [`InputKind::GamepadRemove`] `flags` word into `(pad, seq)`.
pub fn decode_gamepad_remove(flags: u32) -> (u8, u8) {
(flags as u8, (flags >> 24) as u8)
} }
/// The gamepad wire contract for [`InputKind::GamepadButton`]/[`InputKind::GamepadAxis`]. /// The gamepad wire contract for [`InputKind::GamepadButton`]/[`InputKind::GamepadAxis`].
@@ -123,6 +156,8 @@ impl InputKind {
10 => TouchMove, 10 => TouchMove,
11 => TouchUp, 11 => TouchUp,
12 => GamepadState, 12 => GamepadState,
13 => GamepadRemove,
14 => GamepadArrival,
_ => return None, _ => return None,
}) })
} }
@@ -321,8 +356,26 @@ mod tests {
}; };
assert_eq!(InputEvent::decode(&e.encode()), Some(e)); assert_eq!(InputEvent::decode(&e.encode()), Some(e));
} }
// 13 (one past GamepadState) is not a valid kind. // GamepadRemove + GamepadArrival are valid kinds; 15 (one past them) is not.
assert_eq!(InputKind::from_u8(13), None); assert_eq!(InputKind::from_u8(13), Some(InputKind::GamepadRemove));
assert_eq!(InputKind::from_u8(14), Some(InputKind::GamepadArrival));
assert_eq!(InputKind::from_u8(15), None);
}
#[test]
fn gamepad_remove_flags_roundtrip() {
for (pad, seq) in [(0u8, 0u8), (3, 200), (15, 255), (7, 1)] {
let flags = encode_gamepad_remove(pad, seq);
assert_eq!(decode_gamepad_remove(flags), (pad, seq));
}
// Layout matches the snapshot's pad/seq packing (low byte pad, high byte seq).
let snap = GamepadSnapshot {
pad: 9,
seq: 123,
..Default::default()
};
let (pad, seq) = decode_gamepad_remove(snap.to_event().flags);
assert_eq!((pad, seq), (9, 123));
} }
#[test] #[test]
+5 -1
View File
@@ -38,6 +38,7 @@ pub mod input;
pub mod packet; pub mod packet;
#[cfg(feature = "quic")] #[cfg(feature = "quic")]
pub mod quic; pub mod quic;
pub mod reanchor;
pub mod session; pub mod session;
pub mod stats; pub mod stats;
pub mod transport; pub mod transport;
@@ -61,7 +62,10 @@ pub use stats::Stats;
/// TTL of a v2 envelope; `punktfunk_connection_next_rumble` is unchanged and drops it). Additive — /// TTL of a v2 envelope; `punktfunk_connection_next_rumble` is unchanged and drops it). Additive —
/// the wire is backward-compatible (the envelope is a length-tolerant tail on 0xCA), so /// the wire is backward-compatible (the envelope is a length-tolerant tail on 0xCA), so
/// [`WIRE_VERSION`] is unchanged. /// [`WIRE_VERSION`] is unchanged.
pub const ABI_VERSION: u32 = 5; /// v6: added the `punktfunk_reanchor_gate_*` surface (post-loss freeze-until-reanchor gate for the
/// Swift client; Rust embedders use [`reanchor::ReanchorGate`] directly). Additive, client-local —
/// no wire change, so [`WIRE_VERSION`] is unchanged.
pub const ABI_VERSION: u32 = 6;
/// The punktfunk/1 **wire** version — what `Hello`/`Welcome` carry and hosts equality-check. /// The punktfunk/1 **wire** version — what `Hello`/`Welcome` carry and hosts equality-check.
/// Deliberately its own constant: [`ABI_VERSION`] tracks the embeddable **C surface** /// Deliberately its own constant: [`ABI_VERSION`] tracks the embeddable **C surface**
+240 -33
View File
@@ -35,6 +35,34 @@ pub const FLAG_SOF: u8 = 0x4;
/// feeding them to the decoder. Punktfunk/1 only (GameStream never sets it). /// feeding them to the decoder. Punktfunk/1 only (GameStream never sets it).
pub const FLAG_PROBE: u8 = 0x8; pub const FLAG_PROBE: u8 = 0x8;
/// Application `user_flags` bit (the u32 [`PacketHeader::user_flags`] word, surfaced to the client
/// as [`crate::session::Frame::flags`]) — NOT a transport packet flag. Marks the access unit that
/// **completes an intra-refresh wave**: the picture is loss-free from here even though the frame is
/// a coded `P` (no IDR, so the decoder never sets `AV_FRAME_FLAG_KEY`). The client lifts its
/// post-loss display freeze on this bit as well as on a real keyframe — the only bitstream-invisible
/// clean point it can honor without forcing a full IDR. Lives above the low nibble because the host
/// reuses `FLAG_PIC`/`FLAG_SOF`/`FLAG_PROBE` bit values inside `user_flags`; `0x10` clears all four.
pub const USER_FLAG_RECOVERY_POINT: u32 = 0x10;
/// Application `user_flags` bit — a **definitive single-frame clean re-anchor**. Unlike
/// [`USER_FLAG_RECOVERY_POINT`] (an intra-refresh wave boundary, where the first boundary after a loss
/// is only half-healed so the client waits for the second), this marks an access unit the host coded
/// to reference a **known-good** picture on purpose — an AMD **LTR reference-frame-invalidation**
/// recovery frame (`ForceLTRReferenceBitfield`): a clean P-frame off a long-term reference the client
/// already has, not an IDR. The picture is loss-free the instant this AU decodes, so the client lifts
/// its post-loss freeze on the **first** such mark. Coded `P` (no IDR), so the decoder never sets
/// `AV_FRAME_FLAG_KEY` — this host flag is the only signal.
pub const USER_FLAG_RECOVERY_ANCHOR: u32 = 0x20;
/// Widest lost-frame range (frames, wrapping `last - first`) a reference-frame-invalidation
/// recovery may be asked to repair; anything wider goes straight to the keyframe path on BOTH
/// ends. RFI can only re-reference history the encoder still holds — NVENC keeps a 5-frame DPB,
/// AMD LTR ~1 s of marks — and a genuine loss this wide (>1 s even at 240 fps) has no valid
/// reference anywhere, so an RFI request for it is either hopeless or (worse) a phantom range
/// from a desynced counter. Shared by the host's RFI dispatch (range → keyframe fallback) and the
/// client-side gap detectors (huge gap → resync + keyframe request, no RFI).
pub const RFI_MAX_RANGE: u32 = 256;
/// Crypto framing overhead [`Session`](crate::session::Session) adds when encrypting: /// Crypto framing overhead [`Session`](crate::session::Session) adds when encrypting:
/// an 8-byte sequence prefix plus the GCM tag. /// an 8-byte sequence prefix plus the GCM tag.
pub const CRYPTO_OVERHEAD: usize = 8 + crate::crypto::TAG_LEN; pub const CRYPTO_OVERHEAD: usize = 8 + crate::crypto::TAG_LEN;
@@ -98,8 +126,18 @@ const _: () = assert!(HEADER_LEN == 40, "PacketHeader must be 40 bytes / unpadde
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
/// Splits encoded access units into FEC-protected shard packets. Host-side only. /// Splits encoded access units into FEC-protected shard packets. Host-side only.
///
/// Frame numbering: a caller can pass an **explicit** `frame_index` to
/// [`packetize_each`](Self::packetize_each) (the punktfunk/1 encode loop owns the video numbering
/// so the encoder's reference-frame-invalidation bookkeeping stays 1:1 with the wire across
/// encoder rebuilds/resets), or pass `None` to draw from the internal counter (the legacy path —
/// synthetic/spike/ABI sessions where no encoder cares). Speed-test probe filler draws from a
/// **separate** index space ([`alloc_probe_index`](Self::alloc_probe_index)) so a burst never
/// consumes video indexes — see [`crate::quic::VIDEO_CAP_PROBE_SEQ`].
pub struct Packetizer { pub struct Packetizer {
next_frame_index: u32, next_frame_index: u32,
/// Probe-space frame counter (see [`alloc_probe_index`](Self::alloc_probe_index)).
next_probe_index: u32,
next_seq: u32, next_seq: u32,
shard_payload: usize, shard_payload: usize,
fec: crate::config::FecConfig, fec: crate::config::FecConfig,
@@ -115,6 +153,7 @@ impl Packetizer {
pub fn new(config: &Config) -> Self { pub fn new(config: &Config) -> Self {
Packetizer { Packetizer {
next_frame_index: 0, next_frame_index: 0,
next_probe_index: 0,
next_seq: 0, next_seq: 0,
shard_payload: config.shard_payload, shard_payload: config.shard_payload,
fec: config.fec, fec: config.fec,
@@ -123,6 +162,17 @@ impl Packetizer {
} }
} }
/// Allocate the next **probe-space** frame index (speed-test filler). A separate counter from
/// the video `frame_index`es so a multi-thousand-AU probe burst never advances the video
/// numbering — the client routes [`FLAG_PROBE`]-flagged shards into its own reassembly window
/// (see [`Reassembler`]), so the two spaces never collide. Only used against clients that
/// advertise [`crate::quic::VIDEO_CAP_PROBE_SEQ`].
pub fn alloc_probe_index(&mut self) -> u32 {
let i = self.next_probe_index;
self.next_probe_index = i.wrapping_add(1);
i
}
/// Live-adjust the FEC recovery percentage (adaptive FEC). Takes effect on the next /// Live-adjust the FEC recovery percentage (adaptive FEC). Takes effect on the next
/// [`packetize`](Self::packetize); the wire is self-describing (each packet carries its block's /// [`packetize`](Self::packetize); the wire is self-describing (each packet carries its block's
/// data/recovery counts), so the receiver needs no notification. Clamped to ≤ 90. /// data/recovery counts), so the receiver needs no notification. Clamped to ≤ 90.
@@ -146,7 +196,7 @@ impl Packetizer {
coder: &dyn ErasureCoder, coder: &dyn ErasureCoder,
) -> Result<Vec<Vec<u8>>> { ) -> Result<Vec<Vec<u8>>> {
let mut packets = Vec::new(); let mut packets = Vec::new();
self.packetize_each(frame, pts_ns, user_flags, coder, |hdr, body| { self.packetize_each(frame, pts_ns, user_flags, None, coder, |hdr, body| {
let mut pkt = Vec::with_capacity(HEADER_LEN + body.len()); let mut pkt = Vec::with_capacity(HEADER_LEN + body.len());
pkt.extend_from_slice(hdr.as_bytes()); pkt.extend_from_slice(hdr.as_bytes());
pkt.extend_from_slice(body); pkt.extend_from_slice(body);
@@ -162,17 +212,27 @@ impl Packetizer {
/// shard straight into a pooled wire buffer and seal in place /// shard straight into a pooled wire buffer and seal in place
/// ([`Session::seal_frame`](crate::session::Session::seal_frame)). An `emit` error aborts /// ([`Session::seal_frame`](crate::session::Session::seal_frame)). An `emit` error aborts
/// the frame mid-way (packet numbering has already advanced — callers treat it as fatal). /// the frame mid-way (packet numbering has already advanced — callers treat it as fatal).
///
/// `frame_index`: `Some(i)` stamps the AU with the caller's index — the punktfunk/1 encode
/// loop numbers video AUs itself so the encoder's RFI bookkeeping (LTR marks, DPB timestamps)
/// is 1:1 with what the client sees, surviving encoder rebuilds/resets that restart internal
/// counters. `None` draws from the internal counter (the legacy/self-numbering path). A
/// session must not mix the two styles for the same index space.
pub fn packetize_each( pub fn packetize_each(
&mut self, &mut self,
frame: &[u8], frame: &[u8],
pts_ns: u64, pts_ns: u64,
user_flags: u32, user_flags: u32,
frame_index: Option<u32>,
coder: &dyn ErasureCoder, coder: &dyn ErasureCoder,
mut emit: impl FnMut(&PacketHeader, &[u8]) -> Result<()>, mut emit: impl FnMut(&PacketHeader, &[u8]) -> Result<()>,
) -> Result<()> { ) -> Result<()> {
let payload = self.shard_payload; let payload = self.shard_payload;
let frame_index = self.next_frame_index; let frame_index = frame_index.unwrap_or_else(|| {
self.next_frame_index = self.next_frame_index.wrapping_add(1); let i = self.next_frame_index;
self.next_frame_index = i.wrapping_add(1);
i
});
// At least one (zero-padded) data shard even for an empty frame. // At least one (zero-padded) data shard even for an empty frame.
let total_data = frame.len().div_ceil(payload).max(1); let total_data = frame.len().div_ceil(payload).max(1);
@@ -324,10 +384,13 @@ impl ReassemblerLimits {
} }
} }
/// Buffers incoming shards, recovers lost ones via FEC, and emits whole access units. /// One frame-index space's reassembly state: the in-flight frames, the recently-emitted memory,
/// Client-side only. /// and the loss-window anchor. The [`Reassembler`] keeps two — video and speed-test probe filler —
pub struct Reassembler { /// because the two ride **separate index counters** on a [`VIDEO_CAP_PROBE_SEQ`]-aware host
limits: ReassemblerLimits, /// (a probe burst must neither advance the video loss window nor be dropped as "stale" against
/// it). [`VIDEO_CAP_PROBE_SEQ`]: crate::quic::VIDEO_CAP_PROBE_SEQ
#[derive(Default)]
struct ReassemblyWindow {
frames: HashMap<u32, FrameBuf>, frames: HashMap<u32, FrameBuf>,
/// Recently-emitted frames, so stray/late shards can't resurrect them. Pruned to /// Recently-emitted frames, so stray/late shards can't resurrect them. Pruned to
/// the reorder window alongside `frames`. /// the reorder window alongside `frames`.
@@ -338,13 +401,27 @@ pub struct Reassembler {
newest_frame: Option<(u32, u64)>, newest_frame: Option<(u32, u64)>,
} }
/// Buffers incoming shards, recovers lost ones via FEC, and emits whole access units.
/// Client-side only.
pub struct Reassembler {
limits: ReassemblerLimits,
/// The video stream's window — its aged-out incomplete frames count into `frames_dropped`
/// (the client's loss-recovery trigger).
video: ReassemblyWindow,
/// Speed-test probe filler ([`FLAG_PROBE`] in `user_flags`). Routed by the flag, so it also
/// captures an OLD host's probe frames (which still carry video-space indexes — they complete
/// fine here, and keeping them out of the video window means a burst can no longer advance the
/// video loss anchor). Aged-out probe frames are NOT `frames_dropped` — probe loss is measured
/// bytes-wise by the probe accumulator and must not fire video recovery.
probe: ReassemblyWindow,
}
impl Reassembler { impl Reassembler {
pub fn new(limits: ReassemblerLimits) -> Self { pub fn new(limits: ReassemblerLimits) -> Self {
Reassembler { Reassembler {
limits, limits,
frames: HashMap::new(), video: ReassemblyWindow::default(),
completed: HashSet::new(), probe: ReassemblyWindow::default(),
newest_frame: None,
} }
} }
@@ -405,18 +482,28 @@ impl Reassembler {
} }
let payload = pkt[HEADER_LEN..HEADER_LEN + shard_bytes].to_vec(); let payload = pkt[HEADER_LEN..HEADER_LEN + shard_bytes].to_vec();
self.advance_window(hdr.frame_index, hdr.pts_ns, stats); // Route by index space: speed-test probe filler (FLAG_PROBE in user_flags) reassembles in
// its own window so its indexes never interact with the video loss window — a probe burst
// can neither advance the video anchor nor be dropped as stale against it (and its aged-out
// frames never count as `frames_dropped`, which would fire video loss recovery).
let is_probe = hdr.user_flags & (FLAG_PROBE as u32) != 0;
let win = if is_probe {
&mut self.probe
} else {
&mut self.video
};
win.advance_window(hdr.frame_index, hdr.pts_ns, stats, !is_probe);
// Drop shards for frames we've already emitted (e.g. the recovery shards of a // Drop shards for frames we've already emitted (e.g. the recovery shards of a
// frame that completed early via the all-originals-present fast path) or that // frame that completed early via the all-originals-present fast path) or that
// have fallen out of the loss window. // have fallen out of the loss window.
if self.completed.contains(&hdr.frame_index) || self.is_stale(hdr.frame_index, hdr.pts_ns) { if win.completed.contains(&hdr.frame_index) || win.is_stale(hdr.frame_index, hdr.pts_ns) {
drop(stats); drop(stats);
return Ok(None); return Ok(None);
} }
// First packet of a frame establishes its geometry; later packets must agree. // First packet of a frame establishes its geometry; later packets must agree.
let frame = self let frame = win
.frames .frames
.entry(hdr.frame_index) .entry(hdr.frame_index)
.or_insert_with(|| FrameBuf { .or_insert_with(|| FrameBuf {
@@ -502,8 +589,8 @@ impl Reassembler {
// Whole frame ready? // Whole frame ready?
if frame.block_data.len() == frame.block_count { if frame.block_data.len() == frame.block_count {
let frame = self.frames.remove(&hdr.frame_index).unwrap(); let frame = win.frames.remove(&hdr.frame_index).unwrap();
self.completed.insert(hdr.frame_index); win.completed.insert(hdr.frame_index);
// Reserve based on the bytes we actually hold, not the (already-bounded but // Reserve based on the bytes we actually hold, not the (already-bounded but
// still caller-supplied) frame_bytes, so a small frame can't over-reserve. // still caller-supplied) frame_bytes, so a small frame can't over-reserve.
let actual: usize = frame.block_data.values().map(|b| b.len()).sum(); let actual: usize = frame.block_data.values().map(|b| b.len()).sum();
@@ -522,11 +609,30 @@ impl Reassembler {
Ok(None) Ok(None)
} }
/// Drop all in-flight state — every partially-assembled frame and the completed/abandoned
/// index memory, in both index spaces — as if the session just started. Used by the client's
/// backlog flush ([`Session::flush_backlog`](crate::session::Session::flush_backlog)): after
/// the socket backlog is discarded wholesale, the partial frames here can never complete
/// (their remaining shards were just thrown away) and the window anchors (`newest_frame`)
/// point into the discarded past.
pub fn reset(&mut self) {
self.video = ReassemblyWindow::default();
self.probe = ReassemblyWindow::default();
}
}
impl ReassemblyWindow {
/// Track the newest frame, declare incomplete frames that fell out of the loss window /// Track the newest frame, declare incomplete frames that fell out of the loss window
/// ([`LOSS_WINDOW_NS`] behind the newest pts, or [`HARD_LOSS_WINDOW`] indices) lost — counting /// ([`LOSS_WINDOW_NS`] behind the newest pts, or [`HARD_LOSS_WINDOW`] indices) lost — for the
/// them dropped, which is what drives the client's recovery-keyframe request — and prune the /// video window (`count_drops`) counting them dropped, which is what drives the client's
/// completed-index memory to [`REORDER_WINDOW`]. /// recovery-keyframe request — and prune the completed-index memory to [`REORDER_WINDOW`].
fn advance_window(&mut self, frame_index: u32, pts_ns: u64, stats: &StatsCounters) { fn advance_window(
&mut self,
frame_index: u32,
pts_ns: u64,
stats: &StatsCounters,
count_drops: bool,
) {
let (newest, newest_pts) = match self.newest_frame { let (newest, newest_pts) = match self.newest_frame {
// `frame_index` is newer iff it's within the forward half of the index space. // `frame_index` is newer iff it's within the forward half of the index space.
Some((n, p)) if frame_index.wrapping_sub(n) > u32::MAX / 2 => (n, p), Some((n, p)) if frame_index.wrapping_sub(n) > u32::MAX / 2 => (n, p),
@@ -548,29 +654,17 @@ impl Reassembler {
keep keep
}); });
let pruned = before - self.frames.len(); let pruned = before - self.frames.len();
if pruned > 0 { if pruned > 0 && count_drops {
StatsCounters::add(&stats.frames_dropped, pruned as u64); StatsCounters::add(&stats.frames_dropped, pruned as u64);
} }
self.completed self.completed
.retain(|&idx| newest.wrapping_sub(idx) <= REORDER_WINDOW); .retain(|&idx| newest.wrapping_sub(idx) <= REORDER_WINDOW);
} }
/// Drop all in-flight state — every partially-assembled frame and the completed/abandoned
/// index memory — as if the session just started. Used by the client's backlog flush
/// ([`Session::flush_backlog`](crate::session::Session::flush_backlog)): after the socket
/// backlog is discarded wholesale, the partial frames here can never complete (their remaining
/// shards were just thrown away) and the window anchor (`newest_frame`) points into the
/// discarded past.
pub fn reset(&mut self) {
self.frames.clear();
self.completed.clear();
self.newest_frame = None;
}
/// True if this packet's frame lies outside the loss window (behind the newest frame by more /// True if this packet's frame lies outside the loss window (behind the newest frame by more
/// than [`LOSS_WINDOW_NS`] of capture time or [`HARD_LOSS_WINDOW`] indices) — its shards /// than [`LOSS_WINDOW_NS`] of capture time or [`HARD_LOSS_WINDOW`] indices) — its shards
/// arrive too late to be useful, and accepting one would only create a frame buffer the next /// arrive too late to be useful, and accepting one would only create a frame buffer the next
/// [`advance_window`] immediately declares lost. /// [`advance_window`](Self::advance_window) immediately declares lost.
fn is_stale(&self, frame_index: u32, pts_ns: u64) -> bool { fn is_stale(&self, frame_index: u32, pts_ns: u64) -> bool {
match self.newest_frame { match self.newest_frame {
Some((n, newest_pts)) => { Some((n, newest_pts)) => {
@@ -750,6 +844,119 @@ mod tests {
assert_eq!(stats.snapshot().frames_dropped, 1, "no double-count"); assert_eq!(stats.snapshot().frames_dropped, 1, "no double-count");
} }
/// The explicit-index path stamps the caller's `frame_index` and leaves the internal video
/// counter untouched — the punktfunk/1 encode loop owns the numbering, and mixing must not
/// perturb the legacy self-numbering path (tests/ABI/synthetic).
#[test]
fn explicit_frame_index_is_stamped_and_internal_counter_untouched() {
use crate::config::{FecConfig, FecScheme, ProtocolPhase, Role};
let cfg = Config {
role: Role::Host,
phase: ProtocolPhase::P2Punktfunk,
fec: FecConfig {
scheme: FecScheme::Gf16,
fec_percent: 0,
max_data_per_block: 8,
},
shard_payload: 16,
max_frame_bytes: 4096,
encrypt: false,
key: [0u8; 16],
salt: [0u8; 4],
loopback_drop_period: 0,
};
let coder = coder_for(FecScheme::Gf16);
let mut pk = Packetizer::new(&cfg);
let mut seen = Vec::new();
pk.packetize_each(&[1u8; 16], 0, 0, Some(4242), coder.as_ref(), |hdr, _| {
seen.push(hdr.frame_index);
Ok(())
})
.unwrap();
assert_eq!(seen, vec![4242]);
// The legacy wrapper still numbers from the untouched internal counter.
let pkts = pk.packetize(&[1u8; 16], 0, 0, coder.as_ref()).unwrap();
let hdr = PacketHeader::read_from_bytes(&pkts[0][..HEADER_LEN]).unwrap();
assert_eq!(hdr.frame_index, 0);
// The probe space is a third, independent counter.
assert_eq!(pk.alloc_probe_index(), 0);
assert_eq!(pk.alloc_probe_index(), 1);
}
/// Probe filler (FLAG_PROBE in user_flags) reassembles in its OWN window: a probe frame whose
/// index is far behind the video stream's completes anyway (an old client's single window
/// would drop it as stale), and video frames complete undisturbed around it.
#[test]
fn probe_frames_reassemble_in_their_own_window() {
let mut r = Reassembler::new(limits());
let coder = coder_for(FecScheme::Gf8);
let stats = StatsCounters::default();
// Establish a video stream far into its index space.
let mut v = base_header();
v.frame_index = 100_000;
v.pts_ns = 1_000_000_000;
assert!(r
.push(&packet(v), coder.as_ref(), &stats)
.unwrap()
.is_some());
// A probe frame at index 0 — 100k "behind" the video window — must still complete.
let mut p = base_header();
p.frame_index = 0;
p.pts_ns = 1_000_000_100;
p.user_flags = FLAG_PROBE as u32;
let got = r.push(&packet(p), coder.as_ref(), &stats).unwrap();
assert!(got.is_some(), "probe frame must complete in its own window");
assert_eq!(got.unwrap().flags & FLAG_PROBE as u32, FLAG_PROBE as u32);
// The probe burst must not have advanced the VIDEO window: the next video frame is
// contiguous and completes, with nothing counted dropped.
let mut v2 = base_header();
v2.frame_index = 100_001;
v2.pts_ns = 1_000_000_200;
assert!(r
.push(&packet(v2), coder.as_ref(), &stats)
.unwrap()
.is_some());
assert_eq!(stats.snapshot().frames_dropped, 0);
}
/// An incomplete probe frame aging out of the probe window is NOT a video `frames_dropped`
/// (which would fire the client's loss recovery) — probe loss is measured bytes-wise by the
/// probe accumulator.
#[test]
fn aged_out_probe_frames_do_not_count_as_dropped() {
let mut r = Reassembler::new(limits());
let coder = coder_for(FecScheme::Gf8);
let stats = StatsCounters::default();
// Probe frame 0: one of two shards — incomplete.
let mut p = base_header();
p.user_flags = FLAG_PROBE as u32;
p.data_shards = 2;
p.frame_bytes = 32;
assert!(r
.push(&packet(p), coder.as_ref(), &stats)
.unwrap()
.is_none());
// A much newer probe frame ages it out of the probe window.
let mut p2 = base_header();
p2.user_flags = FLAG_PROBE as u32;
p2.frame_index = 1;
p2.pts_ns = LOSS_WINDOW_NS + 1;
assert!(r
.push(&packet(p2), coder.as_ref(), &stats)
.unwrap()
.is_some());
assert_eq!(
stats.snapshot().frames_dropped,
0,
"probe-window drops must not fire video loss recovery"
);
}
#[test] #[test]
fn rejects_wrong_shard_bytes_and_oversized_frame() { fn rejects_wrong_shard_bytes_and_oversized_frame() {
let coder = coder_for(FecScheme::Gf8); let coder = coder_for(FecScheme::Gf8);
+54
View File
@@ -107,6 +107,18 @@ pub const VIDEO_CAP_444: u8 = 0x04;
/// host ignores it and simply never sends any); a client that doesn't set it keeps the combined /// host ignores it and simply never sends any); a client that doesn't set it keeps the combined
/// stage. Purely observability — never changes what the host encodes. /// stage. Purely observability — never changes what the host encodes.
pub const VIDEO_CAP_HOST_TIMING: u8 = 0x08; pub const VIDEO_CAP_HOST_TIMING: u8 = 0x08;
/// [`Hello::video_caps`] bit: the client's reassembler keeps **speed-test probe filler in its own
/// frame-index space** (a second reassembly window keyed on the [`crate::packet::FLAG_PROBE`]
/// user-flag), so probe bursts no longer consume video `frame_index`es. Without this, a mid-session
/// speed test burns thousands of video indexes that are invisible to every client-side gap detector
/// (probe frames are filtered before the pump sees them) — the first real AU afterwards reads as a
/// phantom multi-thousand-frame loss (spurious freeze + a nonsense RFI). It also lets the host's
/// encode loop own the video numbering outright (the wire-index contract
/// [`crate::packet::Packetizer::packetize_each`] documents), which reference-frame invalidation
/// depends on. The host runs mid-session probe bursts ONLY against clients that set this bit — an
/// older client gets a declined (zeroed) [`ProbeResult`] instead of a measurement its single-window
/// reassembler would silently drop as stale.
pub const VIDEO_CAP_PROBE_SEQ: u8 = 0x10;
/// QUIC application error code a punktfunk/1 client closes the control connection with on a /// QUIC application error code a punktfunk/1 client closes the control connection with on a
/// **deliberate quit** (a user "stop", not a network drop). The host reads it off the connection's /// **deliberate quit** (a user "stop", not a network drop). The host reads it off the connection's
@@ -355,6 +367,24 @@ pub struct Reconfigured {
#[derive(Clone, Copy, Debug, PartialEq, Eq)] #[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct RequestKeyframe; pub struct RequestKeyframe;
/// `client → host`: reference-frame-invalidation recovery — the loss-aware sibling of
/// [`RequestKeyframe`]. The client detected a `frame_index` gap and reports the range `[first_frame,
/// last_frame]` of access units it can no longer trust (from the first missing index through the
/// newest received). Instead of a full IDR (a 20-40× spike that deepens the loss it recovers), a host
/// whose encoder supports RFI re-references a known-good picture *before* `first_frame` — an AMD LTR
/// force-reference or an NVENC `nvEncInvalidateRefFrames` — emitting a single clean P-frame it tags
/// [`crate::packet::USER_FLAG_RECOVERY_ANCHOR`] so the client lifts its freeze on it. A host that
/// can't RFI (no valid reference / libavcodec backend) forces an IDR instead, exactly as for a bare
/// [`RequestKeyframe`]; a host that predates this ignores the unknown message and the client's
/// keyframe backstop still recovers. Fire-and-forget — the recovered frame is the only ack.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct RfiRequest {
/// First access-unit `frame_index` the client can no longer trust (the gap start).
pub first_frame: u32,
/// Newest received `frame_index` at the time of the report (the invalidation range end).
pub last_frame: u32,
}
/// `client → host`, periodic: the client's observed data-plane loss, so the host can size FEC to /// `client → host`, periodic: the client's observed data-plane loss, so the host can size FEC to
/// the link instead of a flat percentage (adaptive FEC). `loss_ppm` is parts-per-million of shards /// the link instead of a flat percentage (adaptive FEC). `loss_ppm` is parts-per-million of shards
/// that arrived missing-but-recovered (plus a bump when frames went unrecoverable) over the report /// that arrived missing-but-recovered (plus a bump when frames went unrecoverable) over the report
@@ -467,6 +497,8 @@ pub const MSG_LOSS_REPORT: u8 = 0x04;
pub const MSG_SET_BITRATE: u8 = 0x05; pub const MSG_SET_BITRATE: u8 = 0x05;
/// Type byte of [`BitrateChanged`]. /// Type byte of [`BitrateChanged`].
pub const MSG_BITRATE_CHANGED: u8 = 0x06; pub const MSG_BITRATE_CHANGED: u8 = 0x06;
/// Type byte of [`RfiRequest`].
pub const MSG_RFI_REQUEST: u8 = 0x07;
/// Type byte of [`ProbeRequest`]. /// Type byte of [`ProbeRequest`].
pub const MSG_PROBE_REQUEST: u8 = 0x20; pub const MSG_PROBE_REQUEST: u8 = 0x20;
/// Type byte of [`ProbeResult`]. /// Type byte of [`ProbeResult`].
@@ -1032,6 +1064,28 @@ impl RequestKeyframe {
} }
} }
impl RfiRequest {
pub fn encode(&self) -> Vec<u8> {
// magic[0..4] type[4] first_frame[5..9] last_frame[9..13]
let mut b = Vec::with_capacity(13);
b.extend_from_slice(CTL_MAGIC);
b.push(MSG_RFI_REQUEST);
b.extend_from_slice(&self.first_frame.to_le_bytes());
b.extend_from_slice(&self.last_frame.to_le_bytes());
b
}
pub fn decode(b: &[u8]) -> Result<RfiRequest> {
if b.len() != 13 || &b[0..4] != CTL_MAGIC || b[4] != MSG_RFI_REQUEST {
return Err(PunktfunkError::InvalidArg("bad RfiRequest"));
}
Ok(RfiRequest {
first_frame: u32::from_le_bytes(b[5..9].try_into().unwrap()),
last_frame: u32::from_le_bytes(b[9..13].try_into().unwrap()),
})
}
}
impl LossReport { impl LossReport {
pub fn encode(&self) -> Vec<u8> { pub fn encode(&self) -> Vec<u8> {
// magic[0..4] type[4] loss_ppm[5..9] // magic[0..4] type[4] loss_ppm[5..9]
+29
View File
@@ -633,6 +633,35 @@ fn request_keyframe_roundtrip() {
assert!(RequestKeyframe::decode(&[bytes.as_slice(), &[0]].concat()).is_err()); assert!(RequestKeyframe::decode(&[bytes.as_slice(), &[0]].concat()).is_err());
} }
#[test]
fn rfi_request_roundtrip() {
for (first_frame, last_frame) in [(0u32, 0u32), (40, 47), (5, 5), (1_000_000, u32::MAX)] {
let r = RfiRequest {
first_frame,
last_frame,
};
assert_eq!(RfiRequest::decode(&r.encode()).unwrap(), r);
}
// Disjoint from the bare keyframe request (its loss-unaware sibling) and others: type byte + length.
assert!(RfiRequest::decode(&RequestKeyframe.encode()).is_err());
assert!(RequestKeyframe::decode(
&RfiRequest {
first_frame: 1,
last_frame: 2
}
.encode()
)
.is_err());
// Exact length — no trailing bytes.
let bytes = RfiRequest {
first_frame: 3,
last_frame: 9,
}
.encode();
assert!(RfiRequest::decode(&[bytes.as_slice(), &[0]].concat()).is_err());
assert!(RfiRequest::decode(&bytes[..bytes.len() - 1]).is_err());
}
#[test] #[test]
fn loss_report_roundtrip() { fn loss_report_roundtrip() {
for loss_ppm in [0u32, 1, 12_345, 50_000, 1_000_000] { for loss_ppm in [0u32, 1, 12_345, 50_000, 1_000_000] {
+473
View File
@@ -0,0 +1,473 @@
//! Post-loss display freeze — the shared "freeze-until-reanchor" gate.
//!
//! After an unrecoverable reference loss the hardware decoder does **not** error: it *conceals* the
//! reference-missing delta frames (on RADV, the DPB-and-output-COINCIDE path paints a gray plate with
//! the new frame's motion on top) and returns Ok. Displaying that is the "gray frames mid-stream"
//! artifact. Instead every client freezes on the last good picture — withholds the concealed frames
//! from its presenter, which keeps redrawing the held frame — and lifts the freeze ONLY on a proven
//! clean re-anchor: a real IDR, an LTR-RFI recovery anchor ([`USER_FLAG_RECOVERY_ANCHOR`]), or the
//! second intra-refresh recovery mark ([`USER_FLAG_RECOVERY_POINT`]) since the loss.
//!
//! This module owns that decision so every embedder shares ONE implementation instead of re-deriving
//! it (the Linux/Deck pump in `pf-client-core`, the Windows in-process pump, the Android decode loops,
//! and — over the C ABI — the Apple client). The state machine is time-driven but takes `now` as a
//! parameter so it is unit-testable without a clock; the C ABI wrappers supply `Instant::now()`.
//!
//! [`USER_FLAG_RECOVERY_POINT`]: crate::packet::USER_FLAG_RECOVERY_POINT
//! [`USER_FLAG_RECOVERY_ANCHOR`]: crate::packet::USER_FLAG_RECOVERY_ANCHOR
use crate::packet::{FLAG_SOF, USER_FLAG_RECOVERY_ANCHOR, USER_FLAG_RECOVERY_POINT};
use std::time::{Duration, Instant};
/// Consecutive no-output AUs that force a keyframe request. ~50 ms at 60 Hz — long enough not to fire
/// on a one-frame decoder hiccup, short enough that a lost initial IDR (or a mid-GOP join) unfreezes
/// almost immediately instead of never.
pub const NO_OUTPUT_KEYFRAME_STREAK: u32 = 3;
/// Longest the gate holds the last good frame waiting for a post-loss re-anchor keyframe before it
/// re-asks. After a reference loss the hardware decoder does not error — it conceals the
/// reference-missing deltas (on RADV, the DPB-and-output-COINCIDE path renders them as a gray plate
/// with the new frame's motion painted over it) and returns Ok, so displaying them is the "gray frames
/// mid-stream" artifact. We instead freeze on the last good picture until a fresh IDR re-anchors decode
/// — the behaviour NVIDIA already shows (its DISTINCT output image + different concealment reads as a
/// brief freeze, not gray). This cap only bounds the freeze when recovery genuinely stalls (host
/// ignores the request, or an RFI recovery that never emits a keyframe): the freeze is NEVER lifted to
/// the concealed picture — the deadline re-asks for a keyframe and keeps holding, so a glitch can never
/// become a permanent freeze while a clean re-anchor is what un-freezes. A recovery IDR round-trips well
/// under this on any live link.
pub const REANCHOR_FREEZE_MAX: Duration = Duration::from_millis(500);
/// How many host intra-refresh recovery marks ([`USER_FLAG_RECOVERY_POINT`]) must arrive since the
/// latest loss before the gate lifts its freeze on an IDR-free stream. TWO, not one: with a continuous
/// rolling wave the host marks phase-fixed wave boundaries, so the FIRST boundary after a loss is only
/// partially healed — stripes swept BEFORE the loss still reference the lost frame — and lifting there
/// would flash a partially-stale picture. The SECOND boundary guarantees a full wave swept entirely
/// after the loss, so the picture is clean. This stays correct under repeated loss because every fresh
/// arm resets the count. The cost is up to ~2 wave periods of holding the last good frame — the
/// deliberate "hold longer, never show garbage" trade.
///
/// [`USER_FLAG_RECOVERY_POINT`]: crate::packet::USER_FLAG_RECOVERY_POINT
pub const REANCHOR_MARKS_TO_LIFT: u32 = 2;
/// Backstop patience while a host intra-refresh heal is visibly in progress. Each recovery mark pushes
/// the freeze deadline out by this much, so a live mark stream (the host actively healing via its wave)
/// keeps the gate patiently holding the last good frame instead of tripping the IDR floor mid-heal.
/// Must exceed the inter-mark interval (one wave period, ~0.5 s) with margin; if the marks STOP (heal
/// stalled, or the host isn't running intra-refresh) the deadline lapses and the normal recovery-IDR
/// floor fires, so a real stall still recovers.
pub const RECOVERY_MARK_PATIENCE: Duration = Duration::from_millis(1500);
/// Frames skipped when `got` arrives while `expected` was the next index, or `None` if `got` is
/// contiguous (`== expected`) or a straggler we have already passed. Frame indices are u32 counters
/// that wrap, so the "ahead" test is a wrapping subtraction split at the half-space: a small positive
/// delta is a forward gap (missing frames whose dependents will decode against absent references); a
/// delta in the top half is an index behind us.
pub fn index_gap(expected: u32, got: u32) -> Option<u32> {
let ahead = got.wrapping_sub(expected);
(ahead != 0 && ahead < u32::MAX / 2).then_some(ahead)
}
/// Fold one decoded frame into the re-anchor state and decide whether it lifts the post-loss freeze.
///
/// `is_keyframe` — a real IDR (always a clean re-anchor). `has_anchor` — this AU carried
/// [`USER_FLAG_RECOVERY_ANCHOR`](crate::packet::USER_FLAG_RECOVERY_ANCHOR), the host's definitive
/// single-frame re-anchor from an LTR-RFI recovery (a clean P-frame coded against a known-good
/// reference), so it lifts on the FIRST occurrence exactly like an IDR — no two-mark wait. `has_mark` —
/// this AU carried [`USER_FLAG_RECOVERY_POINT`](crate::packet::USER_FLAG_RECOVERY_POINT), a
/// host-signalled intra-refresh wave boundary (only *half* a re-anchor). `marks` — recovery marks seen
/// since the latest loss.
///
/// Returns `(lift, new_marks)`: `lift` clears the freeze; `new_marks` is the running count (reset to 0
/// on a lift). The two-mark rule ([`REANCHOR_MARKS_TO_LIFT`]) lives here so it is unit-tested
/// independent of the pump's channel/decoder plumbing — the first wave boundary after a loss is only
/// partially healed, so a single mark must NOT lift. An anchor (or IDR) is a *whole* re-anchor and
/// lifts immediately.
fn reanchor_after_frame(
is_keyframe: bool,
has_anchor: bool,
has_mark: bool,
marks: u32,
) -> (bool, u32) {
let marks = if has_mark {
marks.saturating_add(1)
} else {
marks
};
if is_keyframe || has_anchor || marks >= REANCHOR_MARKS_TO_LIFT {
(true, 0)
} else {
(false, marks)
}
}
/// Whether a decoded frame should be shown or withheld while the gate is (or isn't) frozen.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum GateVerdict {
/// Present this frame — the gate is not frozen, or this frame is the clean re-anchor that lifts it.
Present,
/// Withhold this frame — it is a post-loss concealment; the presenter keeps the last good picture.
Hold,
}
/// The shared post-loss freeze state machine. A client feeds it three kinds of event — an *arm* (a
/// loss was detected: a frame-index gap, a dropped-count climb, or a decoder wedge/demotion), each
/// *decoded frame* ([`on_decoded`](Self::on_decoded), which decides present-vs-hold and interprets the
/// re-anchor wire flags), and each *no-output* AU ([`on_no_output`](Self::on_no_output)) — plus a
/// periodic [`poll`](Self::poll) that folds the dropped counter and fires the overdue backstop.
///
/// The gate emits *intents* only: [`on_no_output`](Self::on_no_output) and [`poll`](Self::poll) return
/// `true` when the client should ask the host for a keyframe. The client routes that through its own
/// ~100 ms request throttle (and the precise RFI-vs-keyframe range decision stays in the loss-range
/// tracker behind [`crate::client::NativeClient::note_frame_index`]) — the gate never touches the wire.
#[derive(Debug, Clone)]
pub struct ReanchorGate {
/// Frozen on the last good frame, withholding the decoder's concealed output until a clean
/// re-anchor. Armed by any loss signal; cleared only by [`on_decoded`](Self::on_decoded) lifting.
awaiting: bool,
/// Host intra-refresh recovery marks seen since the latest arm (see [`REANCHOR_MARKS_TO_LIFT`]).
/// Reset to 0 whenever the freeze is (re-)armed, so a fresh loss always waits out two fresh marks.
marks: u32,
/// When the freeze becomes overdue and [`poll`](Self::poll) re-asks for a keyframe (holding, never
/// resuming to the concealed picture). `None` when not frozen.
deadline: Option<Instant>,
/// Consecutive received AUs that produced no decoded frame — a decoder wedged on missing references
/// with no reassembler drop to trigger recovery. A short streak forces a fresh IDR.
no_output_streak: u32,
/// The last `frames_dropped` value [`poll`](Self::poll) observed; a climb means the reassembler
/// declared an AU unrecoverable and the following deltas will conceal, so arm.
last_dropped: u64,
}
impl ReanchorGate {
/// Seed the gate with the session's current `frames_dropped` so the first [`poll`](Self::poll)
/// doesn't read the baseline as a loss.
pub fn new(frames_dropped: u64) -> Self {
ReanchorGate {
awaiting: false,
marks: 0,
deadline: None,
no_output_streak: 0,
last_dropped: frames_dropped,
}
}
/// Arm the freeze: a loss was detected (a frame-index gap, a dropped-count climb, or a decoder
/// wedge/demotion). Zeroes the mark count so a fresh loss waits out two fresh recovery marks, and
/// (re-)sets the backstop deadline. Idempotent while already frozen (re-arming just re-zeroes the
/// marks and pushes the deadline — the correct behaviour when a second loss lands mid-freeze).
pub fn arm(&mut self, now: Instant) {
self.awaiting = true;
self.marks = 0;
self.deadline = Some(now + REANCHOR_FREEZE_MAX);
}
/// Fold one decoded frame and decide whether to present or withhold it.
///
/// `wire_flags` is the AU's `user_flags` word ([`crate::session::Frame::flags`] /
/// `PunktfunkFrame.flags`); the gate reads [`FLAG_SOF`](crate::packet::FLAG_SOF) (the host sets it
/// only on IDR AUs — the codec-agnostic keyframe signal the platform decoders don't expose),
/// [`USER_FLAG_RECOVERY_ANCHOR`] and [`USER_FLAG_RECOVERY_POINT`]. `decoder_keyframe` is an optional
/// belt from decoders that flag IDRs themselves (libavcodec's `AV_FRAME_FLAG_KEY` on Linux/Windows);
/// pass `false` where the decoder doesn't (Android MediaCodec, Apple VideoToolbox) and rely on the
/// wire `FLAG_SOF`.
///
/// A decoded frame always clears the no-output streak. When frozen, a live mark stream pushes the
/// backstop out ([`RECOVERY_MARK_PATIENCE`]) so a healing wave isn't pre-empted by a mid-heal IDR.
///
/// [`USER_FLAG_RECOVERY_ANCHOR`]: crate::packet::USER_FLAG_RECOVERY_ANCHOR
/// [`USER_FLAG_RECOVERY_POINT`]: crate::packet::USER_FLAG_RECOVERY_POINT
pub fn on_decoded(
&mut self,
wire_flags: u32,
decoder_keyframe: bool,
now: Instant,
) -> GateVerdict {
self.no_output_streak = 0;
let is_keyframe = decoder_keyframe || (wire_flags & FLAG_SOF as u32 != 0);
let has_anchor = wire_flags & USER_FLAG_RECOVERY_ANCHOR != 0;
let has_mark = wire_flags & USER_FLAG_RECOVERY_POINT != 0;
if has_mark && self.awaiting {
self.deadline = Some(now + RECOVERY_MARK_PATIENCE);
}
let (lift, marks) = reanchor_after_frame(is_keyframe, has_anchor, has_mark, self.marks);
self.marks = marks;
if lift {
self.awaiting = false;
self.deadline = None;
}
if self.awaiting {
GateVerdict::Hold
} else {
GateVerdict::Present
}
}
/// A received AU produced no decoded frame (decode error, or the decoder swallowed a
/// reference-missing delta). Returns `true` when the streak has tripped and the client should
/// (throttled) request a keyframe — arming the freeze at the same time, since the stream is broken
/// regardless of whether the throttle lets the request through this iteration.
pub fn on_no_output(&mut self, now: Instant) -> bool {
self.no_output_streak += 1;
if self.no_output_streak >= NO_OUTPUT_KEYFRAME_STREAK {
self.arm(now);
self.no_output_streak = 0;
true
} else {
false
}
}
/// Periodic fold of the session's `frames_dropped` counter plus the overdue backstop. Returns
/// `true` when the client should (throttled) request a keyframe: either the drop count climbed (a
/// fresh unrecoverable loss — arm the freeze) or the freeze has held a full [`REANCHOR_FREEZE_MAX`]
/// window with no re-anchor (re-ask and keep holding — NEVER resume to the concealed picture; a
/// genuinely dead stream is the QUIC idle-timeout watchdog's job, not the gate's).
pub fn poll(&mut self, frames_dropped: u64, now: Instant) -> bool {
let mut want_keyframe = false;
if frames_dropped > self.last_dropped {
self.last_dropped = frames_dropped;
self.arm(now);
want_keyframe = true;
}
if self.awaiting && self.deadline.is_some_and(|d| now >= d) {
self.deadline = Some(now + REANCHOR_FREEZE_MAX);
want_keyframe = true;
}
want_keyframe
}
/// Whether the gate is currently withholding concealed frames (frozen on the last good picture).
pub fn is_holding(&self) -> bool {
self.awaiting
}
}
#[cfg(test)]
mod tests {
use super::*;
// Simulate the gate's re-anchor state across a sequence of decoded frames: each `(is_keyframe,
// has_mark)` pair is folded through `reanchor_after_frame`, returning the frame index (0-based) at
// which the freeze first lifts, or `None` if it never does. A reset to 0 models a fresh loss
// re-arming the freeze (the gate zeroes the count at every arm site).
fn lift_at(frames: &[(bool, bool)]) -> Option<usize> {
let mut marks = 0u32;
for (i, &(is_kf, has_mark)) in frames.iter().enumerate() {
// The intra-refresh-mark model never carries an LTR-RFI anchor (that path is exercised by
// `an_rfi_anchor_lifts_immediately`), so `has_anchor` is always false here.
let (lift, m) = reanchor_after_frame(is_kf, false, has_mark, marks);
marks = m;
if lift {
return Some(i);
}
}
None
}
#[test]
fn a_single_recovery_mark_does_not_lift() {
// The first wave boundary after a loss is only half-healed — one mark must hold the freeze.
assert_eq!(REANCHOR_MARKS_TO_LIFT, 2);
assert_eq!(lift_at(&[(false, true)]), None);
assert_eq!(
lift_at(&[(false, false), (false, true), (false, false)]),
None
);
}
#[test]
fn the_second_recovery_mark_lifts() {
// Two marks = a full wave swept after the loss → clean re-anchor.
assert_eq!(lift_at(&[(false, true), (false, true)]), Some(1));
assert_eq!(
lift_at(&[(false, false), (false, true), (false, false), (false, true)]),
Some(3)
);
}
#[test]
fn a_real_keyframe_lifts_immediately() {
// An IDR is always a clean anchor — no marks needed.
assert_eq!(lift_at(&[(true, false)]), Some(0));
assert_eq!(lift_at(&[(false, true), (true, false)]), Some(1));
}
#[test]
fn a_fresh_gap_resets_the_mark_count() {
// The gate zeroes `marks` at each arm site, so one mark before a new gap plus one after must
// NOT lift — the model resets the running count to imitate that.
let mut marks = 0u32;
let (_, m) = reanchor_after_frame(false, false, true, marks); // mark #1 (pre-gap)
marks = m;
assert_eq!(marks, 1);
marks = 0; // a new gap re-arms the freeze → count reset
let (lift, m) = reanchor_after_frame(false, false, true, marks); // first mark of the new wave
assert!(!lift, "a single post-gap mark must not lift");
assert_eq!(m, 1);
}
#[test]
fn an_rfi_anchor_lifts_immediately() {
// An LTR-RFI recovery anchor is a WHOLE re-anchor (a clean P-frame off a known-good reference),
// so — like an IDR — it lifts on the FIRST occurrence, no two-mark wait.
let (lift, marks) = reanchor_after_frame(false, true, false, 0);
assert!(lift, "an RFI anchor must lift the freeze immediately");
assert_eq!(marks, 0, "a lift resets the running mark count");
// Even with zero prior marks and no keyframe, the anchor alone is sufficient.
let (lift, _) = reanchor_after_frame(false, true, true, 1);
assert!(lift, "an anchor lifts regardless of the pending mark count");
}
#[test]
fn contiguous_indices_are_not_a_gap() {
assert_eq!(index_gap(5, 5), None);
assert_eq!(index_gap(0, 0), None);
}
#[test]
fn a_forward_jump_reports_the_skip_count() {
assert_eq!(index_gap(5, 6), Some(1)); // one frame missing
assert_eq!(index_gap(5, 9), Some(4));
}
#[test]
fn a_straggler_behind_us_is_not_a_gap() {
// The reassembler emitted a newer frame first; the late one must not re-arm.
assert_eq!(index_gap(9, 5), None);
assert_eq!(index_gap(1, 0), None);
}
#[test]
fn the_index_counter_wraps_cleanly() {
// last frame = u32::MAX, so the next expected wraps to 0.
assert_eq!(index_gap(0, 0), None);
// waiting on u32::MAX, frame 0 arrived → MAX was skipped.
assert_eq!(index_gap(u32::MAX, 0), Some(1));
assert_eq!(index_gap(u32::MAX, 2), Some(3));
// an old frame arriving just after the wrap is still a straggler.
assert_eq!(index_gap(0, u32::MAX), None);
}
// ---- gate-level sequence tests (the whole behavioural contract) ----
const SOF: u32 = FLAG_SOF as u32; // IDR wire flag
const ANCHOR: u32 = USER_FLAG_RECOVERY_ANCHOR;
const POINT: u32 = USER_FLAG_RECOVERY_POINT;
fn t0() -> Instant {
Instant::now()
}
#[test]
fn a_clean_link_never_holds() {
// Disarmed gate presents every frame, keyframe or not, and never asks for anything.
let mut g = ReanchorGate::new(0);
let now = t0();
assert_eq!(g.on_decoded(0, false, now), GateVerdict::Present);
assert_eq!(g.on_decoded(SOF, true, now), GateVerdict::Present);
assert!(!g.is_holding());
assert!(!g.poll(0, now));
}
#[test]
fn a_gap_holds_until_the_wire_keyframe_lifts() {
// Android/Apple path: no decoder keyframe flag, lift comes from the wire FLAG_SOF alone.
let mut g = ReanchorGate::new(0);
let now = t0();
g.arm(now); // frame-index gap
assert!(g.is_holding());
assert_eq!(g.on_decoded(0, false, now), GateVerdict::Hold); // concealed delta withheld
assert_eq!(g.on_decoded(0, false, now), GateVerdict::Hold);
assert_eq!(g.on_decoded(SOF, false, now), GateVerdict::Present); // IDR re-anchors
assert!(!g.is_holding());
assert_eq!(g.on_decoded(0, false, now), GateVerdict::Present); // stays presenting
}
#[test]
fn a_gap_lifts_on_the_first_rfi_anchor() {
let mut g = ReanchorGate::new(0);
let now = t0();
g.arm(now);
assert_eq!(g.on_decoded(0, false, now), GateVerdict::Hold);
assert_eq!(g.on_decoded(ANCHOR, false, now), GateVerdict::Present);
assert!(!g.is_holding());
}
#[test]
fn a_gap_lifts_on_the_second_recovery_mark() {
let mut g = ReanchorGate::new(0);
let now = t0();
g.arm(now);
assert_eq!(g.on_decoded(POINT, false, now), GateVerdict::Hold); // first boundary: half-healed
assert_eq!(g.on_decoded(0, false, now), GateVerdict::Hold);
assert_eq!(g.on_decoded(POINT, false, now), GateVerdict::Present); // second: clean
}
#[test]
fn a_second_gap_mid_freeze_resets_the_marks() {
let mut g = ReanchorGate::new(0);
let now = t0();
g.arm(now);
assert_eq!(g.on_decoded(POINT, false, now), GateVerdict::Hold); // mark #1
g.arm(now); // a fresh loss re-arms → mark count zeroed
assert_eq!(g.on_decoded(POINT, false, now), GateVerdict::Hold); // this is mark #1 of the new wave
assert_eq!(g.on_decoded(POINT, false, now), GateVerdict::Present); // #2 lifts
}
#[test]
fn the_dropped_climb_arms_and_asks() {
let mut g = ReanchorGate::new(5);
let now = t0();
assert!(!g.poll(5, now), "no climb → no ask"); // baseline
assert!(g.poll(6, now), "a climb asks for a keyframe");
assert!(g.is_holding(), "and arms the freeze");
assert!(
!g.poll(6, now),
"same value → no repeat ask from the drop path"
);
}
#[test]
fn the_no_output_streak_trips_at_three() {
let mut g = ReanchorGate::new(0);
let now = t0();
assert!(!g.on_no_output(now));
assert!(!g.on_no_output(now));
assert!(g.on_no_output(now), "third no-output trips the streak");
assert!(g.is_holding());
// A decoded frame resets the streak.
g.on_decoded(SOF, false, now); // lifts + resets streak
assert!(!g.on_no_output(now));
assert!(!g.on_no_output(now));
assert!(g.on_no_output(now));
}
#[test]
fn an_overdue_freeze_re_asks_but_keeps_holding() {
let mut g = ReanchorGate::new(0);
let start = t0();
g.arm(start);
// Before the deadline: holding, no re-ask.
assert!(!g.poll(0, start));
assert!(g.is_holding());
// Past REANCHOR_FREEZE_MAX with no re-anchor: re-ask, still holding.
let later = start + REANCHOR_FREEZE_MAX + Duration::from_millis(1);
assert!(g.poll(0, later), "overdue freeze re-asks for a keyframe");
assert!(g.is_holding(), "but never resumes to the concealed picture");
}
#[test]
fn a_live_mark_stream_pushes_the_deadline_out() {
// A healing wave (marks arriving) must not be pre-empted by the overdue IDR floor.
let mut g = ReanchorGate::new(0);
let start = t0();
g.arm(start);
// A mark past the original freeze deadline pushes it out by RECOVERY_MARK_PATIENCE.
let t = start + REANCHOR_FREEZE_MAX + Duration::from_millis(10);
// mark #1 pushes the deadline out; at a time that WOULD have been overdue on the ORIGINAL
// deadline, poll does not re-ask.
assert_eq!(g.on_decoded(POINT, false, t), GateVerdict::Hold);
assert!(!g.poll(0, t + Duration::from_millis(1)));
assert!(g.is_holding());
}
}

Some files were not shown because too many files have changed in this diff Show More