DualSense haptics + speaker work in-game: four fixes between the game and the pad #302

Merged
enricobuehler merged 4 commits from worktree-pad-audio-fidelity into main 2026-08-18 15:02:20 +00:00
10 changed files with 259 additions and 21 deletions
@@ -34,6 +34,29 @@ pub(crate) struct PadUsbCapturer {
pad: u8,
}
/// Map the pad's **hardware** quad onto the wire's **logical** layout.
///
/// The isochronous endpoint carries the DualSense's own channel map — `ch0` = headphone LEFT,
/// `ch1` = headphone RIGHT *and* the built-in mono speaker, `ch2`/`ch3` = the voice coils
/// (confirmed twice independently: the UCM split positions `[AUX1,AUX1,AUX2,AUX3]` and the
/// on-glass channel sweep). The 0xD1 wire contract instead puts the *speaker pair* on ch0/1.
/// Forwarding the hardware quad verbatim therefore ships headphone-left (silence, or content no
/// remote pad can render — the jack is on the other end of the stream) as wire speaker-left, and
/// the actual speaker channel as wire speaker-right — which the client then plays into the ONE
/// split-sink channel that a current PipeWire never wires to the physical speaker.
/// Field-diagnosed 2026-08-18: haptics felt, speaker dead, the tone measured on exactly one
/// channel at each hop.
///
/// So: duplicate the hardware speaker channel (`ch1`) across the wire's speaker pair, pass the
/// coils through. Headphone-left is dropped deliberately — the remote pad's jack is not a wire
/// surface, and a game that routes to the jack has the pad's audio *off* the speaker anyway.
fn normalize_hw_quad(mut chunk: Vec<f32>) -> Vec<f32> {
for frame in chunk.chunks_exact_mut(4) {
frame[0] = frame[1];
}
chunk
}
impl PadUsbCapturer {
/// Claim wire pad `pad`'s USB audio stream.
///
@@ -55,7 +78,7 @@ impl AudioCapturer for PadUsbCapturer {
fn next_chunk_within(&mut self, budget: Duration) -> Result<Vec<f32>> {
match self.rx.recv_timeout(budget.min(IDLE_TIMEOUT)) {
Ok(chunk) => Ok(chunk),
Ok(chunk) => Ok(normalize_hw_quad(chunk)),
// Nothing arrived in the budget. The game isn't writing (or the stream is stopped) —
// a quiet pad, not a dead one, exactly as the sink capturer reports it.
Err(RecvTimeoutError::Timeout) => Ok(Vec::new()),
@@ -110,15 +133,29 @@ mod tests {
assert!(c.next_chunk_within(Duration::from_millis(10)).is_err());
}
/// Samples pass through untouched the handler already produced interleaved `f32`.
/// The hardware quad is normalized to the wire layout: hw ch1 (the pad's one real speaker
/// channel) is duplicated across the wire speaker pair, the coils pass through, and hw ch0
/// (headphone-left — not a wire surface) is dropped. Forwarding the quad verbatim shipped
/// the speaker on wire ch1 only, which the client's split-sink render never got to the
/// physical speaker (field, 2026-08-18: haptics felt, speaker dead).
#[test]
fn delivers_the_published_chunk_verbatim() {
fn normalizes_the_hardware_quad_to_the_wire_layout() {
let (tx, mut c) = capturer();
tx.send(vec![0.5, -0.5, 0.25, -0.25]).expect("send");
tx.send(vec![0.9, 0.5, 0.25, -0.25, 0.8, 0.4, 0.2, -0.2])
.expect("send");
assert_eq!(
c.next_chunk_within(Duration::from_millis(50))
.expect("chunk"),
vec![0.5, -0.5, 0.25, -0.25]
vec![0.5, 0.5, 0.25, -0.25, 0.4, 0.4, 0.2, -0.2]
);
}
/// The normalizer itself, on one frame: `[hpL, spk, coilA, coilB]` → `[spk, spk, coilA, coilB]`.
#[test]
fn normalize_duplicates_the_speaker_channel() {
assert_eq!(
normalize_hw_quad(vec![0.9, 0.5, 0.25, -0.25]),
vec![0.5, 0.5, 0.25, -0.25]
);
}
}
+24 -15
View File
@@ -2,7 +2,7 @@
//! Windows: WASAPI loopback of a pre-provisioned endpoint ([`crate::audio::pad_endpoint`]);
//! Linux: the per-pad PipeWire sink we mint (`crate::audio::pad_sink`) — → 4-ch de-interleave
//! into the speaker (front) and voice-coil haptics (back) pairs → per-kind silence gate →
//! stereo Opus (48 kHz, CBR, LowDelay)
//! stereo Opus (48 kHz, CBR; LowDelay for haptics, Audio for the speaker)
//! → [`PAD_AUDIO_MAGIC`](punktfunk_core::quic::PAD_AUDIO_MAGIC) datagrams. One thread per
//! arriving pad, spawned/reaped by the input thread ([`super::input`]) as arrivals declare
//! renderers and pads leave. Modeled on the session audio thread ([`super::audio`]): the same
@@ -49,10 +49,16 @@ const GATE_OPEN_PEAK: f32 = 1e-3;
#[cfg(any(target_os = "windows", target_os = "linux", test))]
const GATE_HANGOVER_MS: u32 = 250;
/// Per-kind Opus bitrate — a stereo voice-coil / pad-speaker pair needs far less than the
/// session plane's 128 kbps; 64 kbps CBR keeps every frame comfortably under one MTU.
/// The haptics lane's Opus bitrate — voice-coil content is band-limited rumble; 64 kbps CBR
/// keeps every frame comfortably under one MTU.
#[cfg(any(target_os = "windows", target_os = "linux"))]
const PAD_AUDIO_BITRATE: i32 = 64_000;
const HAPTICS_BITRATE: i32 = 64_000;
/// The speaker lane's Opus bitrate. The pad speaker carries real programme audio (voice lines,
/// effects), and 64 kbps CELT-only in 10 ms frames is audibly artifacty there — field report
/// 2026-08-18: "sounds insanely compressed". 96 kbps CBR is still ~120 bytes per frame, far
/// under one MTU.
#[cfg(any(target_os = "windows", target_os = "linux"))]
const SPEAKER_BITRATE: i32 = 96_000;
/// The per-kind silence gate — the steady-state-cost feature: an idle pad endpoint (games
/// rarely render pad audio) must cost ZERO encodes and ZERO datagrams, not a permanent 200 Hz
@@ -471,32 +477,35 @@ struct Lane {
encode_errs: u64,
}
/// Build one stereo encoder per enabled kind: 48 kHz LowDelay hard-CBR like the session audio
/// plane ([`super::audio`]), at the pad plane's 64 kbps.
/// Build one stereo encoder per enabled kind: 48 kHz hard-CBR like the session audio plane
/// ([`super::audio`]), each lane tuned to its content. Haptics are felt latency — LowDelay
/// (CELT-only, 2.5 ms lookahead) at 64 kbps. The speaker is programme audio — the full
/// `Application::Audio` coder at 96 kbps; its ~4 ms of extra algorithmic delay is inaudible on
/// a speaker but the CELT-only artifacts were not (field, 2026-08-18).
#[cfg(any(target_os = "windows", target_os = "linux"))]
fn build_lanes(kinds: u8) -> Result<Vec<Lane>, opus::Error> {
let mut lanes = Vec::new();
for (bit, kind, frame_ms) in [
for (bit, kind, frame_ms, app, bitrate) in [
(
KIND_BIT_HAPTICS,
punktfunk_core::quic::PAD_AUDIO_KIND_HAPTICS,
HAPTICS_FRAME_MS,
opus::Application::LowDelay,
HAPTICS_BITRATE,
),
(
KIND_BIT_SPEAKER,
punktfunk_core::quic::PAD_AUDIO_KIND_SPEAKER,
SPEAKER_FRAME_MS,
opus::Application::Audio,
SPEAKER_BITRATE,
),
] {
if kinds & bit == 0 {
continue;
}
let mut enc = opus::Encoder::new(
crate::audio::SAMPLE_RATE,
opus::Channels::Stereo,
opus::Application::LowDelay,
)?;
enc.set_bitrate(opus::Bitrate::Bits(PAD_AUDIO_BITRATE)).ok();
let mut enc = opus::Encoder::new(crate::audio::SAMPLE_RATE, opus::Channels::Stereo, app)?;
enc.set_bitrate(opus::Bitrate::Bits(bitrate)).ok();
enc.set_vbr(false).ok();
lanes.push(Lane {
kind,
@@ -537,7 +546,7 @@ fn pad_audio_thread<C: crate::audio::AudioCapturer>(
return; // spawn() refuses kinds == 0 — belt and braces
}
let mut framer = PadFramer::new(kinds);
// One Opus frame per datagram; 64 kbps CBR at ≤10 ms is ~80 bytes — sized with the session
// One Opus frame per datagram; 96 kbps CBR at ≤10 ms is ~120 bytes — sized with the session
// plane's slack.
let mut opus_buf = vec![0u8; 1500];
// Reopen-with-backoff (the audio.rs discipline): a capture death (endpoint invalidated,
@@ -546,7 +555,7 @@ fn pad_audio_thread<C: crate::audio::AudioCapturer>(
let mut capturer: Option<C> = None;
let mut last_failed: Option<std::time::Instant> = None;
// Datagrams the wire refused as oversized (`design/hi-res-audio.md` §4.8). Vanishingly
// unlikely on this plane — a 64 kbps CBR Opus frame at ≤10 ms is ~80 bytes — but it used to
// unlikely on this plane — a ≤96 kbps CBR Opus frame at ≤10 ms is ≤~120 bytes — but it used to
// be indistinguishable from the connection ending, which is the actual defect being fixed.
let mut oversized_drops: u64 = 0;
tracing::info!(
+6
View File
@@ -28,10 +28,16 @@ num-derive = "0.4"
num-traits = "0.2"
# `time` is for the interrupt-IN pacing added in device.rs (punktfunk modification — see NOTICE).
tokio = { version = "1", features = ["rt", "net", "io-util", "sync", "time"] }
# Upstream gated its struct derives behind a `serde` feature; kept (off by default) so the
# `#[cfg(feature = "serde")]` attributes stay valid and the vendored diff stays minimal.
serde = { version = "1", features = ["derive"], optional = true }
[dev-dependencies]
# `#[tokio::test(start_paused = true)]` for the ISO pacing tests — paused virtual time is the
# only way to pin an absolute-deadline pacer exactly.
tokio = { version = "1", features = ["rt", "macros", "test-util", "time"] }
[features]
default = []
serde = ["dep:serde"]
+7
View File
@@ -36,6 +36,13 @@ Modifications by the punktfunk project:
`actual_length = 0`; `vhci_hcd` copies that field into the URB verbatim, so
every synchronous writer (`write()` on hidraw, `HIDIOCSFEATURE`) was told it
transferred 0 bytes and treated the write as failed.
- Isochronous completion is paced against an absolute per-endpoint deadline
ledger (`UsbDevice::iso_deadlines`) rather than a relative
`sleep(interval × packets)` per URB. The relative sleep added scheduling
overhead on top of every period, so the simulated device's audio clock ran
measurably slow (~26 % under load) — the PCM backed up into xruns and
anything clocked off the device dragged. Late completions now catch up;
a stall beyond 20 ms re-anchors instead of fast-forwarding.
Only the USB/IP server *simulation* path is retained: the device model, the
USB/IP wire protocol, and the `UsbInterfaceHandler` trait. The original MIT
+134 -1
View File
@@ -52,6 +52,13 @@ pub struct UsbDevice {
#[cfg_attr(feature = "serde", serde(skip))]
pub device_handler: Option<Arc<Mutex<Box<dyn UsbDeviceHandler + Send>>>>,
/// Per-endpoint isochronous completion deadlines (punktfunk addition) — the absolute-time
/// ledger [`handle_iso_urb`](Self::handle_iso_urb) paces against. Keyed by endpoint address.
/// Shared across clones because the clones all present the same device: whoever services the
/// endpoint advances the one clock.
#[cfg_attr(feature = "serde", serde(skip))]
pub(crate) iso_deadlines: Arc<Mutex<HashMap<u8, tokio::time::Instant>>>,
pub usb_version: Version,
pub(crate) ep0_in: UsbEndpoint,
@@ -321,6 +328,17 @@ impl UsbDevice {
/// above is paced), so completing instantly would both spin the loopback link and tell the
/// kernel the device consumed a whole URB's worth of samples in no time, running the stream's
/// clock away and xrunning it continuously.
///
/// **Paced against an absolute per-endpoint deadline, not relative sleeps.** A plain
/// `sleep(interval × packets)` per URB adds every source of slop — tokio timer granularity,
/// socket I/O, handler lock waits — ON TOP of the nominal period, so the device's clock runs
/// systematically slow (measured ~26 % slow on a busy graph, 2026-08-18: `hw_ptr` advanced
/// ~35.7 k frames/s against a 48 kHz stream — the PCM backs up, latency grows into xruns,
/// and anything clocked off this device drags). The ledger makes late completions *catch up*:
/// each URB advances the endpoint's deadline by exactly its nominal duration and sleeps until
/// that absolute instant, so overhead eats into the next sleep instead of accumulating. If
/// the stream stalls long enough that the ledger is far behind (stop/start, unlink storm),
/// it re-anchors to now rather than fast-forwarding a burst of instant completions.
pub(crate) async fn handle_iso_urb(
&self,
ep: UsbEndpoint,
@@ -331,7 +349,22 @@ impl UsbDevice {
// ISO on ep0 is not a thing; treat it as an unsupported transfer rather than panicking.
return Err(std::io::Error::other("isochronous transfer to ep0"));
};
tokio::time::sleep(self.service_interval(ep) * packets.len() as u32).await;
// Allow this much catch-up before deciding the stream stalled and re-anchoring. Two USB
// frames of slack keeps ordinary scheduling jitter inside the ledger (where it averages
// out) without letting a restarted stream burn through a stale deadline backlog.
const RESYNC_SLACK: std::time::Duration = std::time::Duration::from_millis(20);
let step = self.service_interval(ep) * packets.len() as u32;
let deadline = {
let mut ledger = self.iso_deadlines.lock().unwrap();
let now = tokio::time::Instant::now();
let due = ledger.entry(ep.address).or_insert(now);
if *due + RESYNC_SLACK < now {
*due = now;
}
*due += step;
*due
};
tokio::time::sleep_until(deadline).await;
let mut handler = intf.handler.lock().unwrap();
handler.handle_iso_urb(intf, ep, packets)
}
@@ -717,6 +750,106 @@ mod pacing_tests {
);
}
/// A no-op ISO handler so the pacing tests can drive `handle_iso_urb` without a device model.
#[derive(Debug)]
struct NullIso;
impl crate::UsbInterfaceHandler for NullIso {
fn handle_urb(
&mut self,
_interface: &UsbInterface,
_ep: UsbEndpoint,
_transfer_buffer_length: u32,
_setup: crate::SetupPacket,
_req: &[u8],
) -> Result<Vec<u8>> {
Ok(Vec::new())
}
fn handle_iso_urb(
&mut self,
_interface: &UsbInterface,
_ep: UsbEndpoint,
packets: &[IsoPacket<'_>],
) -> Result<Vec<Vec<u8>>> {
Ok(vec![Vec::new(); packets.len()])
}
fn get_class_specific_descriptor(&self) -> Vec<u8> {
Vec::new()
}
fn as_any(&mut self) -> &mut dyn std::any::Any {
self
}
}
fn null_intf() -> UsbInterface {
UsbInterface {
interface_class: 1,
interface_subclass: 2,
interface_protocol: 0,
endpoints: vec![iso_ep(4)],
string_interface: 0,
class_specific_descriptor: Vec::new(),
alt_settings: Vec::new(),
handler: Arc::new(Mutex::new(
Box::new(NullIso) as Box<dyn crate::UsbInterfaceHandler + Send>
)),
}
}
/// The completion pace must hold the NOMINAL rate over many URBs — a relative
/// `sleep(interval × packets)` per URB adds scheduling overhead on top of every period and
/// the device's audio clock runs measurably slow (~26 % on a busy graph, field 2026-08-18).
/// Under tokio's paused clock the ledger's `sleep_until` deadlines auto-advance with zero
/// slop, so 50 URBs × 8 packets × 1 ms must take exactly 400 ms of virtual time — and the
/// deadline arithmetic (not per-call `now()`) is what guarantees the same under real slop.
#[tokio::test(start_paused = true)]
async fn iso_pacing_holds_the_nominal_rate_across_urbs() {
let d = dev(UsbSpeed::High);
let intf = null_intf();
let buf = [0u8; 392];
let start = tokio::time::Instant::now();
for _ in 0..50 {
let packets: Vec<IsoPacket<'_>> = (0..8)
.map(|_| IsoPacket {
data: &buf,
requested_len: 392,
})
.collect();
d.handle_iso_urb(iso_ep(4), Some(&intf), &packets)
.await
.expect("iso urb");
}
assert_eq!(
start.elapsed(),
std::time::Duration::from_millis(400),
"50 URBs × 8 packets × 1 ms must complete in exactly their nominal duration"
);
}
/// After a stall longer than the resync slack, the ledger re-anchors to now instead of
/// fast-forwarding a burst of instant completions through the stale backlog.
#[tokio::test(start_paused = true)]
async fn iso_pacing_reanchors_after_a_stall() {
let d = dev(UsbSpeed::High);
let intf = null_intf();
let buf = [0u8; 392];
let one = |d: &UsbDevice, intf: &UsbInterface| {
let packets = vec![IsoPacket {
data: &buf,
requested_len: 392,
}];
let d = d.clone();
let intf = intf.clone();
async move { d.handle_iso_urb(iso_ep(4), Some(&intf), &packets).await }
};
one(&d, &intf).await.expect("prime the ledger");
// Stall well past the slack, then resume: the next URB must take ~its nominal 1 ms from
// NOW, not complete instantly against the stale deadline.
tokio::time::sleep(std::time::Duration::from_millis(500)).await;
let start = tokio::time::Instant::now();
one(&d, &intf).await.expect("resumed urb");
assert_eq!(start.elapsed(), std::time::Duration::from_millis(1));
}
/// `bmAttributes` carries the synchronisation and usage type above the transfer type, so a real
/// UAC endpoint is `0x05`/`0x09` rather than a bare `0x01`. Decoding the whole byte returns
/// `None` for those and used to reach `unimplemented!()`; only bits 1..0 may be decoded.
+2
View File
@@ -213,6 +213,8 @@ package_punktfunk-host() {
install -Dm0755 "$T/punktfunk-encode-worker" "$pkgdir/usr/bin/punktfunk-encode-worker"
# /dev/uinput + /dev/uhid -> input group (virtual gamepads + DualSense UHID)
install -Dm0644 "$R/scripts/60-punktfunk.rules" "$pkgdir/usr/lib/udev/rules.d/60-punktfunk.rules"
install -Dm0644 "$R/scripts/60-punktfunk-dualsense.conf" \
"$pkgdir/usr/share/wireplumber/wireplumber.conf.d/60-punktfunk-dualsense.conf"
# Managed gamescope takeover on DM-autologin boxes: root helper + polkit action so the host can
# stop/restore the display manager for the stream. Arch has no /usr/libexec — install under
# /usr/lib/punktfunk and rewrite the policy's exec.path annotation to match (the host probes both).
+1
View File
@@ -82,6 +82,7 @@ install -Dm0644 packaging/linux/punktfunk-update.service \
install -Dm0644 packaging/linux/49-punktfunk-update.rules \
"$STAGE/usr/share/polkit-1/rules.d/49-punktfunk-update.rules"
install -Dm0644 scripts/60-punktfunk.rules "$STAGE/usr/lib/udev/rules.d/60-punktfunk.rules"
install -Dm0644 scripts/60-punktfunk-dualsense.conf "$STAGE/usr/share/wireplumber/wireplumber.conf.d/60-punktfunk-dualsense.conf"
# Managed gamescope takeover on DM-autologin boxes: root helper + polkit action so the host can
# stop/restore the display manager for the stream (the helper derives the DM unit itself).
install -Dm0755 scripts/pf-dm-helper "$STAGE/usr/libexec/punktfunk/pf-dm-helper"
+2
View File
@@ -176,6 +176,8 @@ in
# udev: /dev/uinput + /dev/uhid (virtual gamepads) + the vhci sysfs perms for the virtual Deck.
install -Dm0644 scripts/60-punktfunk.rules "$out/lib/udev/rules.d/60-punktfunk.rules"
# WirePlumber: hold a DualSense's sound card open + keep it off the graph clock.
install -Dm0644 scripts/60-punktfunk-dualsense.conf "$out/share/wireplumber/wireplumber.conf.d/60-punktfunk-dualsense.conf"
# KWin Desktop-mode authorization (zkde_screencast + fake_input). Point Exec at the store binary.
install -Dm0644 packaging/linux/io.unom.Punktfunk.Host.desktop \
+6
View File
@@ -323,6 +323,11 @@ install -Dm0755 target/release/punktfunk-encode-worker %{buildroot}%{_bindir}/pu
# udev rule — /dev/uinput access for virtual gamepads (input group).
install -Dm0644 scripts/60-punktfunk.rules %{buildroot}%{_udevrulesdir}/60-punktfunk.rules
%{_datadir}/wireplumber/wireplumber.conf.d/60-punktfunk-dualsense.conf
# WirePlumber policy — hold a DualSense's sound card open (GE-Proton's raw-open self-race) and
# keep it from driving the graph clock. See the file's own comments.
install -Dm0644 scripts/60-punktfunk-dualsense.conf %{buildroot}%{_datadir}/wireplumber/wireplumber.conf.d/60-punktfunk-dualsense.conf
# Managed gamescope takeover on DM-autologin boxes (Nobara's plasmalogin): a root helper + polkit
# action let the host stop/restore the display manager for the stream without a hand-installed
@@ -577,6 +582,7 @@ install -Dm0644 scripts/punktfunk-scripting.service %{buildroot}%{_userunitdir}/
%{_unitdir}/user@.service.d/50-punktfunk-nice.conf
%{_bindir}/punktfunk-tray
%{_udevrulesdir}/60-punktfunk.rules
%{_datadir}/wireplumber/wireplumber.conf.d/60-punktfunk-dualsense.conf
%dir %{_libexecdir}/punktfunk
%{_libexecdir}/punktfunk/pf-dm-helper
%{_libexecdir}/punktfunk/pf-update
+35
View File
@@ -0,0 +1,35 @@
# WirePlumber policy for DualSense sound cards on a punktfunk host (virtual usbip pads AND
# physically plugged pads — the failure modes are identical).
#
# 1. `node.always-process` + no suspend: PipeWire must HOLD the pad's ALSA device open at all
# times. GE-Proton's DS5 haptic router opens the sink's backing `hw:` device RAW whenever it
# is free — and then its own path re-probe EBUSYs against its own handle, invalidates the
# stream, and spins a refresh loop at 100 Hz (haptics dead, speaker dead). On SteamOS, where
# that code was developed, PipeWire always holds the device, so GE lands on its well-tested
# Pulse-routing fallback immediately. This rule reproduces that environment. Field-diagnosed
# 2026-08-18 (Spider-Man Remastered, GE-Proton 11-5).
#
# 2. `priority.driver = 1`: an always-processing node is a permanent graph-driver candidate, and
# a USB pad's audio clock (virtual or real) must never clock the whole graph — the day this
# was diagnosed, the virtual pad's clock drove the desktop capture to 50 % delivery. Keep the
# pad a follower.
#
# Install: /usr/share/wireplumber/wireplumber.conf.d/ (the user instance reads the shared dirs).
monitor.alsa.rules = [
{
matches = [
# Both product-string spellings a DS5 family pad ships with: newer firmware / Edge say
# "DualSense[ Edge] Wireless Controller", earlier firmware says just "Wireless Controller".
{ node.name = "~alsa_output.usb-Sony_Interactive_Entertainment_DualSense.*" }
{ node.name = "~alsa_output.usb-Sony_Interactive_Entertainment_Wireless_Controller.*" }
]
actions = {
update-props = {
session.suspend-timeout-seconds = 0
node.pause-on-idle = false
node.always-process = true
priority.driver = 1
}
}
}
]