Files
punktfunk/clients/apple/Sources/LumenKit/InputCapture.swift
T
enricobuehler 3ea096ace9
ci / rust (push) Has been cancelled
feat: M4 groundwork — lumen/1 client connector in the C ABI + SwiftUI client scaffold
The shared-core architecture pays off: platform clients now link ONE Rust library that
does the entire lumen/1 protocol, and only add decode/present/input on top.

lumen-core:
- client.rs (quic feature): NativeClient — QUIC handshake + UDP data plane + input
  datagrams on internal threads; embedder surface = connect / next_frame / send_input.
- abi.rs: lumen_connect / lumen_connection_next_au (borrow-until-next-call, matching
  lumen_client_poll_frame semantics) / lumen_connection_send_input / lumen_connection_mode /
  lumen_connection_close. Guarded in the generated header by LUMEN_FEATURE_QUIC (cbindgen
  [defines] mapping), so the checked-in header is stable across feature sets.
- error.rs: append-only LumenStatus additions Timeout (-9) and Closed (-10).
- TESTED end-to-end through the C ABI: in-process lumen/1 host, lumen_connect pulls 25
  byte-verified frames, sends input, closes (m3.rs::c_abi_connection_roundtrip).

Apple client (clients/apple — SCAFFOLD, written on Linux, first Xcode build pending):
- scripts/build-xcframework.sh: cargo per Apple target → universal staticlib + header
  (LUMEN_FEATURE_QUIC pre-defined) + modulemap → LumenCore.xcframework.
- Package.swift (LumenKit) + Swift sources: LumenConnection (ABI wrapper), AnnexB
  (in-band VPS/SPS/PPS → CMVideoFormatDescription, Annex-B → AVCC CMSampleBuffers with
  DisplayImmediately), StreamView (SwiftUI over AVSampleBufferDisplayLayer — stage-1
  presenter that hardware-decodes compressed HEVC itself), InputCapture (GCMouse raw
  deltas + GCKeyboard HID→VK).
- README.md is the full handoff for the next (Mac-side) agent: build steps, ABI contract,
  first-light test recipe against the Linux host, stage-2 (VT+Metal pacing) plan, and the
  known host-side gaps (single-session m3-host, no lumen/1 audio yet, gamepad kinds not
  yet routed in m3's injector, seed-stage trust).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-06-10 07:28:41 +00:00

104 lines
4.3 KiB
Swift
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
// Input capture lumen/1 datagrams, via the GameController framework.
//
// GCMouse delivers RAW deltas (not the accelerated cursor) exactly what the host-side
// injector expects for relative motion. GCKeyboard gives HID keycodes which we map to the
// Windows VK space the host's vk_to_evdev table consumes (same space Moonlight uses).
// Gamepads (GCController) come later the host's uinput pads already speak the
// GamepadButton/GamepadAxis event kinds.
//
// SCAFFOLD: written on the Linux host, not yet compiled against Xcode. The VK map covers
// the common keys; extend alongside lumen-host/src/inject.rs::vk_to_evdev.
#if os(macOS)
import Foundation
import GameController
import LumenCore
public final class InputCapture {
private let connection: LumenConnection
private var observers: [NSObjectProtocol] = []
public init(connection: LumenConnection) {
self.connection = connection
}
/// Begin forwarding the current (and future) mouse/keyboard to the host.
public func start() {
if let mouse = GCMouse.current { attach(mouse: mouse) }
if let keyboard = GCKeyboard.coalesced { attach(keyboard: keyboard) }
observers.append(NotificationCenter.default.addObserver(
forName: .GCMouseDidConnect, object: nil, queue: .main
) { [weak self] n in
if let m = n.object as? GCMouse { self?.attach(mouse: m) }
})
observers.append(NotificationCenter.default.addObserver(
forName: .GCKeyboardDidConnect, object: nil, queue: .main
) { [weak self] n in
if let k = n.object as? GCKeyboard { self?.attach(keyboard: k) }
})
}
public func stop() {
observers.forEach(NotificationCenter.default.removeObserver(_:))
observers.removeAll()
}
private func attach(mouse: GCMouse) {
guard let input = mouse.mouseInput else { return }
let conn = connection
input.mouseMovedHandler = { _, dx, dy in
// GC gives +y up; the host expects screen-space (+y down).
conn.send(.mouseMove(dx: Int32(dx), dy: Int32(-dy)))
}
input.leftButton.pressedChangedHandler = { _, _, pressed in
conn.send(.mouseButton(1, down: pressed))
}
input.rightButton?.pressedChangedHandler = { _, _, pressed in
conn.send(.mouseButton(3, down: pressed))
}
input.middleButton?.pressedChangedHandler = { _, _, pressed in
conn.send(.mouseButton(2, down: pressed))
}
input.scroll.valueChangedHandler = { _, _, dy in
if dy != 0 { conn.send(.scroll(Int32(dy * 120))) }
}
}
private func attach(keyboard: GCKeyboard) {
let conn = connection
keyboard.keyboardInput?.keyChangedHandler = { _, _, keyCode, pressed in
if let vk = Self.hidToVK[keyCode.rawValue] {
conn.send(.key(vk, down: pressed))
}
}
}
/// HID usage (GCKeyCode raw) Windows VK (the host maps VK evdev).
static let hidToVK: [Int: UInt32] = {
var m: [Int: UInt32] = [:]
// az: HID 0x04..0x1D VK 'A'..'Z'.
for i in 0..<26 { m[0x04 + i] = UInt32(0x41 + i) }
// 19, 0: HID 0x1E..0x27 VK '1'..'9','0'.
for i in 0..<9 { m[0x1E + i] = UInt32(0x31 + i) }
m[0x27] = 0x30
m[0x28] = 0x0D // return
m[0x29] = 0x1B // escape
m[0x2A] = 0x08 // backspace
m[0x2B] = 0x09 // tab
m[0x2C] = 0x20 // space
m[0x2D] = 0xBD; m[0x2E] = 0xBB // - =
m[0x2F] = 0xDB; m[0x30] = 0xDD; m[0x31] = 0xDC // [ ] backslash
m[0x33] = 0xBA; m[0x34] = 0xDE; m[0x35] = 0xC0 // ; ' `
m[0x36] = 0xBC; m[0x37] = 0xBE; m[0x38] = 0xBF // , . /
// F1..F12: HID 0x3A..0x45 VK 0x70..0x7B.
for i in 0..<12 { m[0x3A + i] = UInt32(0x70 + i) }
m[0x4F] = 0x27; m[0x50] = 0x25; m[0x51] = 0x28; m[0x52] = 0x26 // arrows R L D U
m[0x49] = 0x2D; m[0x4A] = 0x24; m[0x4B] = 0x21 // insert home pageup
m[0x4C] = 0x2E; m[0x4D] = 0x23; m[0x4E] = 0x22 // delete end pagedown
m[0xE0] = 0xA2; m[0xE1] = 0xA0; m[0xE2] = 0xA4; m[0xE3] = 0x5B // Lctrl Lshift Lalt Lcmd
m[0xE4] = 0xA3; m[0xE5] = 0xA1; m[0xE6] = 0xA5; m[0xE7] = 0x5C // Rctrl Rshift Ralt Rcmd
return m
}()
}
#endif