diff --git a/clients/apple/README.md b/clients/apple/README.md index 187dde3..5e346a2 100644 --- a/clients/apple/README.md +++ b/clients/apple/README.md @@ -118,7 +118,9 @@ signing, bundle id `io.unom.punktfunk`. Notes: host stamps `pts_ns` with its capture wall clock; across machines you need a clock offset estimate from the QUIC RTT). 5. **Audio**: `nextAudio()` yields raw Opus packets (48 kHz stereo, one 5 ms frame each, - sequence-numbered). Decode with libopus or `AVAudioConverter`/`kAudioFormatOpus` into an + sequence-numbered). The inverse direction exists too: `sendMic(_:seq:ptsNs:)` uplinks + the client's mic as Opus frames into a virtual PipeWire source on the host (wire it + to AVAudioEngine input + an Opus encoder alongside playback). Decode with libopus or `AVAudioConverter`/`kAudioFormatOpus` into an `AVAudioEngine` source node; conceal gaps (drop/dup) rather than blocking — the Rust side buffers 320 ms and drops the newest packet when the puller lags. Wall-clock `ptsNs` shares the host clock with video AUs for A/V sync. Wiring this into diff --git a/clients/apple/Sources/PunktfunkKit/PunktfunkConnection.swift b/clients/apple/Sources/PunktfunkKit/PunktfunkConnection.swift index 77dfae9..341387d 100644 --- a/clients/apple/Sources/PunktfunkKit/PunktfunkConnection.swift +++ b/clients/apple/Sources/PunktfunkKit/PunktfunkConnection.swift @@ -334,6 +334,21 @@ public final class PunktfunkConnection { } } + /// Send one Opus mic frame (48 kHz) to the host, where it feeds a virtual + /// microphone source the host's apps can record. Non-blocking enqueue, safe + /// alongside the pull threads (same discipline as `send`). `seq`/`ptsNs` are the + /// caller's own counters (host uses them only for diagnostics); empty `opus` is a + /// DTX silence frame. + public func sendMic(_ opus: Data, seq: UInt32, ptsNs: UInt64) { + abiLock.lock() + defer { abiLock.unlock() } + guard let h = handle, !closeRequested else { return } + opus.withUnsafeBytes { p in + _ = punktfunk_connection_send_mic( + h, p.bindMemory(to: UInt8.self).baseAddress, UInt(opus.count), seq, ptsNs) + } + } + deinit { close() } /// Snapshot the handle unless close is pending (callers hold their plane lock). @@ -387,4 +402,30 @@ public extension PunktfunkInputEvent { static func gamepadAxis(_ axis: UInt32, value: Int32, pad: UInt32 = 0) -> PunktfunkInputEvent { make(PUNKTFUNK_INPUT_KIND_GAMEPAD_AXIS.rawValue, code: axis, x: value, y: 0, flags: pad) } + + // Touch (host-side: libei ei_touchscreen on the virtual output). `id` distinguishes + // fingers and is reusable after touchUp; coordinates are absolute pixels on the + // client's touch surface, whose size rides in `flags` so the host can rescale — + // the surface dimensions must each fit in 16 bits. Built for the iOS variant + // (UITouch → these); nothing on macOS emits them yet. + + static func touchDown( + id: UInt32, x: Int32, y: Int32, surfaceWidth: UInt32, surfaceHeight: UInt32 + ) -> PunktfunkInputEvent { + make( + PUNKTFUNK_INPUT_KIND_TOUCH_DOWN.rawValue, code: id, x: x, y: y, + flags: ((surfaceWidth & 0xFFFF) << 16) | (surfaceHeight & 0xFFFF)) + } + + static func touchMove( + id: UInt32, x: Int32, y: Int32, surfaceWidth: UInt32, surfaceHeight: UInt32 + ) -> PunktfunkInputEvent { + make( + PUNKTFUNK_INPUT_KIND_TOUCH_MOVE.rawValue, code: id, x: x, y: y, + flags: ((surfaceWidth & 0xFFFF) << 16) | (surfaceHeight & 0xFFFF)) + } + + static func touchUp(id: UInt32) -> PunktfunkInputEvent { + make(PUNKTFUNK_INPUT_KIND_TOUCH_UP.rawValue, code: id, x: 0, y: 0) + } } diff --git a/clients/apple/Tests/PunktfunkKitTests/LoopbackIntegrationTests.swift b/clients/apple/Tests/PunktfunkKitTests/LoopbackIntegrationTests.swift index badd0b1..b2f478f 100644 --- a/clients/apple/Tests/PunktfunkKitTests/LoopbackIntegrationTests.swift +++ b/clients/apple/Tests/PunktfunkKitTests/LoopbackIntegrationTests.swift @@ -42,10 +42,17 @@ final class LoopbackIntegrationTests: XCTestCase { } XCTAssertGreaterThanOrEqual(lastIndex, 24) - // Input goes the other way (enqueue-only; the host logs the count on close). + // Input goes the other way (enqueue-only; the host logs the count on close) — + // including the touch kinds and the mic uplink plane (the synthetic host counts + // the datagrams; injection/decoding are Linux-side concerns). conn.send(.mouseMove(dx: 1, dy: 2)) conn.send(.key(0x41, down: true)) conn.send(.key(0x41, down: false)) + conn.send(.touchDown(id: 0, x: 100, y: 200, surfaceWidth: 1280, surfaceHeight: 720)) + conn.send(.touchMove(id: 0, x: 110, y: 210, surfaceWidth: 1280, surfaceHeight: 720)) + conn.send(.touchUp(id: 0)) + conn.sendMic(Data([0xFC, 0xFF, 0xFE]), seq: 0, ptsNs: 1) // tiny opus-ish frame + conn.sendMic(Data(), seq: 1, ptsNs: 2) // DTX silence frame conn.close() XCTAssertThrowsError(try conn.nextAU(timeoutMs: 10)) { error in