From 3a51551f9783d219c09d29f15acae54cd9f34998 Mon Sep 17 00:00:00 2001 From: enricobuehler Date: Thu, 11 Jun 2026 09:08:04 +0200 Subject: [PATCH] feat(apple): mic uplink + touch events in PunktfunkKit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adopts the new ABI surface (still v2, additive): - PunktfunkConnection.sendMic(_:seq:ptsNs:) — Opus mic frames (48 kHz) to the host's virtual PipeWire source; enqueue-only, empty data = DTX silence. Wiring the actual Mac microphone (AVAudioEngine input → Opus) into the app is the follow-up, alongside audio playback (README note 5). - PunktfunkInputEvent.touchDown/touchMove/touchUp — absolute pixels + surface size in flags, host injects via libei ei_touchscreen. Built for the iOS variant; nothing on macOS emits them yet. - Loopback round trip now also sends touch events and mic frames (incl. a DTX frame) through the wrapper. Co-Authored-By: Claude Fable 5 --- clients/apple/README.md | 4 +- .../PunktfunkKit/PunktfunkConnection.swift | 41 +++++++++++++++++++ .../LoopbackIntegrationTests.swift | 9 +++- 3 files changed, 52 insertions(+), 2 deletions(-) diff --git a/clients/apple/README.md b/clients/apple/README.md index 187dde3..5e346a2 100644 --- a/clients/apple/README.md +++ b/clients/apple/README.md @@ -118,7 +118,9 @@ signing, bundle id `io.unom.punktfunk`. Notes: host stamps `pts_ns` with its capture wall clock; across machines you need a clock offset estimate from the QUIC RTT). 5. **Audio**: `nextAudio()` yields raw Opus packets (48 kHz stereo, one 5 ms frame each, - sequence-numbered). Decode with libopus or `AVAudioConverter`/`kAudioFormatOpus` into an + sequence-numbered). The inverse direction exists too: `sendMic(_:seq:ptsNs:)` uplinks + the client's mic as Opus frames into a virtual PipeWire source on the host (wire it + to AVAudioEngine input + an Opus encoder alongside playback). Decode with libopus or `AVAudioConverter`/`kAudioFormatOpus` into an `AVAudioEngine` source node; conceal gaps (drop/dup) rather than blocking — the Rust side buffers 320 ms and drops the newest packet when the puller lags. Wall-clock `ptsNs` shares the host clock with video AUs for A/V sync. Wiring this into diff --git a/clients/apple/Sources/PunktfunkKit/PunktfunkConnection.swift b/clients/apple/Sources/PunktfunkKit/PunktfunkConnection.swift index 77dfae9..341387d 100644 --- a/clients/apple/Sources/PunktfunkKit/PunktfunkConnection.swift +++ b/clients/apple/Sources/PunktfunkKit/PunktfunkConnection.swift @@ -334,6 +334,21 @@ public final class PunktfunkConnection { } } + /// Send one Opus mic frame (48 kHz) to the host, where it feeds a virtual + /// microphone source the host's apps can record. Non-blocking enqueue, safe + /// alongside the pull threads (same discipline as `send`). `seq`/`ptsNs` are the + /// caller's own counters (host uses them only for diagnostics); empty `opus` is a + /// DTX silence frame. + public func sendMic(_ opus: Data, seq: UInt32, ptsNs: UInt64) { + abiLock.lock() + defer { abiLock.unlock() } + guard let h = handle, !closeRequested else { return } + opus.withUnsafeBytes { p in + _ = punktfunk_connection_send_mic( + h, p.bindMemory(to: UInt8.self).baseAddress, UInt(opus.count), seq, ptsNs) + } + } + deinit { close() } /// Snapshot the handle unless close is pending (callers hold their plane lock). @@ -387,4 +402,30 @@ public extension PunktfunkInputEvent { static func gamepadAxis(_ axis: UInt32, value: Int32, pad: UInt32 = 0) -> PunktfunkInputEvent { make(PUNKTFUNK_INPUT_KIND_GAMEPAD_AXIS.rawValue, code: axis, x: value, y: 0, flags: pad) } + + // Touch (host-side: libei ei_touchscreen on the virtual output). `id` distinguishes + // fingers and is reusable after touchUp; coordinates are absolute pixels on the + // client's touch surface, whose size rides in `flags` so the host can rescale — + // the surface dimensions must each fit in 16 bits. Built for the iOS variant + // (UITouch → these); nothing on macOS emits them yet. + + static func touchDown( + id: UInt32, x: Int32, y: Int32, surfaceWidth: UInt32, surfaceHeight: UInt32 + ) -> PunktfunkInputEvent { + make( + PUNKTFUNK_INPUT_KIND_TOUCH_DOWN.rawValue, code: id, x: x, y: y, + flags: ((surfaceWidth & 0xFFFF) << 16) | (surfaceHeight & 0xFFFF)) + } + + static func touchMove( + id: UInt32, x: Int32, y: Int32, surfaceWidth: UInt32, surfaceHeight: UInt32 + ) -> PunktfunkInputEvent { + make( + PUNKTFUNK_INPUT_KIND_TOUCH_MOVE.rawValue, code: id, x: x, y: y, + flags: ((surfaceWidth & 0xFFFF) << 16) | (surfaceHeight & 0xFFFF)) + } + + static func touchUp(id: UInt32) -> PunktfunkInputEvent { + make(PUNKTFUNK_INPUT_KIND_TOUCH_UP.rawValue, code: id, x: 0, y: 0) + } } diff --git a/clients/apple/Tests/PunktfunkKitTests/LoopbackIntegrationTests.swift b/clients/apple/Tests/PunktfunkKitTests/LoopbackIntegrationTests.swift index badd0b1..b2f478f 100644 --- a/clients/apple/Tests/PunktfunkKitTests/LoopbackIntegrationTests.swift +++ b/clients/apple/Tests/PunktfunkKitTests/LoopbackIntegrationTests.swift @@ -42,10 +42,17 @@ final class LoopbackIntegrationTests: XCTestCase { } XCTAssertGreaterThanOrEqual(lastIndex, 24) - // Input goes the other way (enqueue-only; the host logs the count on close). + // Input goes the other way (enqueue-only; the host logs the count on close) — + // including the touch kinds and the mic uplink plane (the synthetic host counts + // the datagrams; injection/decoding are Linux-side concerns). conn.send(.mouseMove(dx: 1, dy: 2)) conn.send(.key(0x41, down: true)) conn.send(.key(0x41, down: false)) + conn.send(.touchDown(id: 0, x: 100, y: 200, surfaceWidth: 1280, surfaceHeight: 720)) + conn.send(.touchMove(id: 0, x: 110, y: 210, surfaceWidth: 1280, surfaceHeight: 720)) + conn.send(.touchUp(id: 0)) + conn.sendMic(Data([0xFC, 0xFF, 0xFE]), seq: 0, ptsNs: 1) // tiny opus-ish frame + conn.sendMic(Data(), seq: 1, ptsNs: 2) // DTX silence frame conn.close() XCTAssertThrowsError(try conn.nextAU(timeoutMs: 10)) { error in