feat(apple): mic uplink + touch events in PunktfunkKit
ci / rust (push) Has been cancelled

Adopts the new ABI surface (still v2, additive):

- PunktfunkConnection.sendMic(_:seq:ptsNs:) — Opus mic frames (48 kHz) to the host's
  virtual PipeWire source; enqueue-only, empty data = DTX silence. Wiring the actual
  Mac microphone (AVAudioEngine input → Opus) into the app is the follow-up, alongside
  audio playback (README note 5).
- PunktfunkInputEvent.touchDown/touchMove/touchUp — absolute pixels + surface size in
  flags, host injects via libei ei_touchscreen. Built for the iOS variant; nothing on
  macOS emits them yet.
- Loopback round trip now also sends touch events and mic frames (incl. a DTX frame)
  through the wrapper.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
2026-06-11 09:08:04 +02:00
parent 6575dddac7
commit 3a51551f97
3 changed files with 52 additions and 2 deletions
+3 -1
View File
@@ -118,7 +118,9 @@ signing, bundle id `io.unom.punktfunk`. Notes:
host stamps `pts_ns` with its capture wall clock; across machines you need a clock
offset estimate from the QUIC RTT).
5. **Audio**: `nextAudio()` yields raw Opus packets (48 kHz stereo, one 5 ms frame each,
sequence-numbered). Decode with libopus or `AVAudioConverter`/`kAudioFormatOpus` into an
sequence-numbered). The inverse direction exists too: `sendMic(_:seq:ptsNs:)` uplinks
the client's mic as Opus frames into a virtual PipeWire source on the host (wire it
to AVAudioEngine input + an Opus encoder alongside playback). Decode with libopus or `AVAudioConverter`/`kAudioFormatOpus` into an
`AVAudioEngine` source node; conceal gaps (drop/dup) rather than blocking — the Rust
side buffers 320 ms and drops the newest packet when the puller lags. Wall-clock
`ptsNs` shares the host clock with video AUs for A/V sync. Wiring this into
@@ -334,6 +334,21 @@ public final class PunktfunkConnection {
}
}
/// Send one Opus mic frame (48 kHz) to the host, where it feeds a virtual
/// microphone source the host's apps can record. Non-blocking enqueue, safe
/// alongside the pull threads (same discipline as `send`). `seq`/`ptsNs` are the
/// caller's own counters (host uses them only for diagnostics); empty `opus` is a
/// DTX silence frame.
public func sendMic(_ opus: Data, seq: UInt32, ptsNs: UInt64) {
abiLock.lock()
defer { abiLock.unlock() }
guard let h = handle, !closeRequested else { return }
opus.withUnsafeBytes { p in
_ = punktfunk_connection_send_mic(
h, p.bindMemory(to: UInt8.self).baseAddress, UInt(opus.count), seq, ptsNs)
}
}
deinit { close() }
/// Snapshot the handle unless close is pending (callers hold their plane lock).
@@ -387,4 +402,30 @@ public extension PunktfunkInputEvent {
static func gamepadAxis(_ axis: UInt32, value: Int32, pad: UInt32 = 0) -> PunktfunkInputEvent {
make(PUNKTFUNK_INPUT_KIND_GAMEPAD_AXIS.rawValue, code: axis, x: value, y: 0, flags: pad)
}
// Touch (host-side: libei ei_touchscreen on the virtual output). `id` distinguishes
// fingers and is reusable after touchUp; coordinates are absolute pixels on the
// client's touch surface, whose size rides in `flags` so the host can rescale
// the surface dimensions must each fit in 16 bits. Built for the iOS variant
// (UITouch these); nothing on macOS emits them yet.
static func touchDown(
id: UInt32, x: Int32, y: Int32, surfaceWidth: UInt32, surfaceHeight: UInt32
) -> PunktfunkInputEvent {
make(
PUNKTFUNK_INPUT_KIND_TOUCH_DOWN.rawValue, code: id, x: x, y: y,
flags: ((surfaceWidth & 0xFFFF) << 16) | (surfaceHeight & 0xFFFF))
}
static func touchMove(
id: UInt32, x: Int32, y: Int32, surfaceWidth: UInt32, surfaceHeight: UInt32
) -> PunktfunkInputEvent {
make(
PUNKTFUNK_INPUT_KIND_TOUCH_MOVE.rawValue, code: id, x: x, y: y,
flags: ((surfaceWidth & 0xFFFF) << 16) | (surfaceHeight & 0xFFFF))
}
static func touchUp(id: UInt32) -> PunktfunkInputEvent {
make(PUNKTFUNK_INPUT_KIND_TOUCH_UP.rawValue, code: id, x: 0, y: 0)
}
}
@@ -42,10 +42,17 @@ final class LoopbackIntegrationTests: XCTestCase {
}
XCTAssertGreaterThanOrEqual(lastIndex, 24)
// Input goes the other way (enqueue-only; the host logs the count on close).
// Input goes the other way (enqueue-only; the host logs the count on close)
// including the touch kinds and the mic uplink plane (the synthetic host counts
// the datagrams; injection/decoding are Linux-side concerns).
conn.send(.mouseMove(dx: 1, dy: 2))
conn.send(.key(0x41, down: true))
conn.send(.key(0x41, down: false))
conn.send(.touchDown(id: 0, x: 100, y: 200, surfaceWidth: 1280, surfaceHeight: 720))
conn.send(.touchMove(id: 0, x: 110, y: 210, surfaceWidth: 1280, surfaceHeight: 720))
conn.send(.touchUp(id: 0))
conn.sendMic(Data([0xFC, 0xFF, 0xFE]), seq: 0, ptsNs: 1) // tiny opus-ish frame
conn.sendMic(Data(), seq: 1, ptsNs: 2) // DTX silence frame
conn.close()
XCTAssertThrowsError(try conn.nextAU(timeoutMs: 10)) { error in