feat: M4 stage 1 — the SwiftUI client is real: compiles, tested, first light on glass
ci / rust (push) Has been cancelled

The clients/apple scaffold is now a working macOS client, validated live against this
repo's host across the LAN: gamescope virtual output → NVENC HEVC → lumen/1 (GF(2¹⁶) FEC +
AES-GCM over UDP, QUIC control) → VideoToolbox → AVSampleBufferDisplayLayer at 720p60,
mouse/keyboard flowing back as QUIC datagrams into the host's gamescope EIS injector
(~3.7k events injected in one session).

LumenKit:
- LumenConnection: the predicted cbindgen compile fixes (C17 header spells the typedefs as
  integers while the enum constants import as a distinct Swift type — bridge by rawValue);
  close() is now safe from any thread (a close flag + pumpLock held across the blocking
  poll enforce the C contract "never close with a next_au in flight"; flag prevents
  lock-starvation by back-to-back polls).
- StreamView: per-pump cancellation token (reconnects can't double-pump), flush + re-gate
  on the next in-band parameter sets when the layer fails, no stale enqueue after restart.
- InputCapture: fractional-delta accumulation (sub-pixel motion isn't truncated away),
  pressed-state tracking with release-all on focus loss and stop() (nothing sticks down
  host-side), global-singleton ownership guard (GC has one handler slot per process),
  X1/X2 buttons, horizontal scroll, full keypad/CapsLock/ISO-102nd/PrintScreen/Menu VKs.
- LumenClient app shell (swift run LumenClient): connect form, fps/Mb-s HUD,
  LUMEN_AUTOCONNECT/LUMEN_MODE for scripted first-light runs.
- Tests: Annex-B byte-level units; real-codec round trip (VTCompressionSession-encoded
  HEVC rebuilt as the host's wire shape → AnnexB → VTDecompressionSession → pixels);
  test-loopback.sh (Swift client vs a real local m3-host over loopback — the Swift twin of
  c_abi_connection_roundtrip); RemoteFirstLightTests (full pipeline over the LAN).

Host/build fixes that fell out:
- The workspace builds on non-Linux again: gamestream audio (opus) and sendmmsg batching
  are now platform-gated with stubs/fallback, per the crate's "compiles everywhere" rule.
- Horizontal scroll was inverted end-to-end: the injectors negated BOTH axes onto the
  ei/wl axes, but GameStream's horizontal convention is positive = right
  (moonlight-qt/Sunshine pass it through unnegated) — only vertical flips now. This also
  un-inverts real Moonlight clients.
- AnnexB drops all zeros preceding a start code (trailing_zero_8bits padding), ffmpeg's
  policy, instead of leaking them into the preceding NAL.
- build-xcframework.sh: deployment targets pinned to the package floor + an otool guard —
  cargo does not fingerprint MACOSX_DEPLOYMENT_TARGET, so warm caches can silently ship
  too-new minos objects.

Adversarially reviewed (5-dimension multi-agent pass, every finding refutation-verified):
14 confirmed findings, all fixed above; the send-while-polling core-contract gap flagged
here is closed by the lumen/1 session-planes work (&self pulls + per-plane borrow slots).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
2026-06-10 14:38:01 +02:00
parent 520d7342dd
commit bf8a974e8b
23 changed files with 1212 additions and 180 deletions
@@ -0,0 +1,122 @@
// Connect form live stream. Stage-1 UX: pick host + mode, see frames, type/aim.
import AppKit
import LumenKit
import SwiftUI
struct ContentView: View {
@StateObject private var model = SessionModel()
@AppStorage("lumen.host") private var host = "192.168.1.70"
@AppStorage("lumen.port") private var port = 9777
@AppStorage("lumen.width") private var width = 1920
@AppStorage("lumen.height") private var height = 1080
@AppStorage("lumen.hz") private var hz = 60
var body: some View {
Group {
if let conn = model.connection {
stream(conn)
} else {
connectForm
}
}
.onAppear { autoConnectIfAsked() }
.onDisappear { model.disconnect() } // window closed mid-session (Cmd+N spawns more)
}
/// Development hook: LUMEN_AUTOCONNECT=host[:port] connects immediately at the saved
/// (or LUMEN_MODE=WxHxHz) mode lets scripts drive first-light runs. (IPv4/hostname
/// only; an IPv6 literal would need bracket parsing.)
private func autoConnectIfAsked() {
guard let target = ProcessInfo.processInfo.environment["LUMEN_AUTOCONNECT"],
!target.isEmpty, model.connection == nil, !model.connecting
else { return }
let parts = target.split(separator: ":")
host = String(parts[0])
if parts.count == 2, let p = Int(parts[1]) { port = p }
if let mode = ProcessInfo.processInfo.environment["LUMEN_MODE"] {
let dims = mode.split(separator: "x").compactMap { Int($0) }
if dims.count == 3 {
width = dims[0]
height = dims[1]
hz = dims[2]
}
}
model.connect(
host: host, port: UInt16(clamping: port),
width: UInt32(clamping: width), height: UInt32(clamping: height),
hz: UInt32(clamping: hz))
}
private func stream(_ conn: LumenConnection) -> some View {
StreamView(
connection: conn,
onFrame: { [meter = model.meter] au in meter.note(byteCount: au.data.count) },
onSessionEnd: { [weak model] in
Task { @MainActor in model?.sessionEnded() }
}
)
.overlay(alignment: .topTrailing) { hud(conn) }
.frame(minWidth: 640, minHeight: 360)
.background(Color.black)
}
private func hud(_ conn: LumenConnection) -> some View {
VStack(alignment: .trailing, spacing: 4) {
Text("\(conn.width)×\(conn.height)@\(conn.refreshHz) \(model.fps) fps \(model.mbps, specifier: "%.1f") Mb/s")
.font(.system(.caption, design: .monospaced))
Button("Disconnect") { model.disconnect() }
.font(.caption)
}
.padding(8)
.background(.black.opacity(0.5), in: RoundedRectangle(cornerRadius: 6))
.foregroundStyle(.white)
.padding(10)
}
private var connectForm: some View {
VStack(spacing: 14) {
Text("lumen").font(.largeTitle.weight(.semibold))
Form {
TextField("Host", text: $host)
TextField("Port", value: $port, format: .number.grouping(.never))
HStack {
TextField("Width", value: $width, format: .number.grouping(.never))
Text("×")
TextField("Height", value: $height, format: .number.grouping(.never))
Text("@")
TextField("Hz", value: $hz, format: .number.grouping(.never))
}
Button("Use this display's mode") { fillFromMainScreen() }
.buttonStyle(.link)
}
.frame(width: 340)
if let error = model.errorMessage {
Text(error)
.font(.caption)
.foregroundStyle(.red)
.frame(width: 340)
}
Button(model.connecting ? "Connecting…" : "Connect") {
model.connect(
host: host, port: UInt16(clamping: port),
width: UInt32(clamping: width), height: UInt32(clamping: height),
hz: UInt32(clamping: hz))
}
.keyboardShortcut(.defaultAction)
.disabled(model.connecting || host.isEmpty)
}
.padding(28)
.frame(minWidth: 420, minHeight: 320)
}
private func fillFromMainScreen() {
guard let screen = NSScreen.main else { return }
let scale = screen.backingScaleFactor
width = Int(screen.frame.width * scale)
height = Int(screen.frame.height * scale)
hz = screen.maximumFramesPerSecond
}
}
@@ -0,0 +1,29 @@
// LumenClient development app shell around LumenKit (swift run LumenClient).
// Connect form StreamView (AVSampleBufferDisplayLayer HEVC) + InputCapture.
import AppKit
import SwiftUI
@main
struct LumenClientApp: App {
@NSApplicationDelegateAdaptor(AppDelegate.self) private var appDelegate
var body: some Scene {
WindowGroup("lumen") {
ContentView()
}
}
}
final class AppDelegate: NSObject, NSApplicationDelegate {
func applicationDidFinishLaunching(_ notification: Notification) {
// `swift run` launches an unbundled binary; promote it to a regular app so the
// window fronts and receives keyboard/mouse focus (GameController needs focus).
NSApp.setActivationPolicy(.regular)
NSApp.activate(ignoringOtherApps: true)
}
func applicationShouldTerminateAfterLastWindowClosed(_ sender: NSApplication) -> Bool {
true
}
}
@@ -0,0 +1,115 @@
// Session state for the app shell: owns the connection, the input capture, and the
// pump-thread main-actor stats relay.
import Foundation
import LumenKit
import SwiftUI
/// Pump-thread-side frame counters; a 1 Hz main-actor timer drains them into @Published
/// values. NSLock instead of an actor the writer is the (non-async) pump thread.
final class FrameMeter: @unchecked Sendable {
private let lock = NSLock()
private var frames = 0
private var bytes = 0
private var totalFrames = 0
func note(byteCount: Int) {
lock.lock()
frames += 1
bytes += byteCount
totalFrames += 1
lock.unlock()
}
/// Returns and resets the per-interval counters (the running total stays).
func drain() -> (frames: Int, bytes: Int, total: Int) {
lock.lock()
defer {
frames = 0
bytes = 0
lock.unlock()
}
return (frames, bytes, totalFrames)
}
}
@MainActor
final class SessionModel: ObservableObject {
@Published var connection: LumenConnection?
@Published var connecting = false
@Published var errorMessage: String?
@Published var fps = 0
@Published var mbps = 0.0
@Published var totalFrames = 0
let meter = FrameMeter()
private var inputCapture: InputCapture?
private var statsTimer: Timer?
func connect(host: String, port: UInt16, width: UInt32, height: UInt32, hz: UInt32) {
guard !connecting else { return }
connecting = true
errorMessage = nil
Task.detached(priority: .userInitiated) {
// LumenConnection.init blocks on the QUIC handshake keep it off the main actor.
let result = Result { try LumenConnection(
host: host, port: port, width: width, height: height, refreshHz: hz) }
await MainActor.run { [weak self] in
guard let self else { return }
self.connecting = false
switch result {
case .success(let conn):
self.connection = conn
self.startInput(conn)
self.startStatsTimer()
case .failure:
self.errorMessage = "Connection failed — is the host running? " +
"(lumen-host m3-host on \(host):\(port))"
}
}
}
}
func disconnect() {
inputCapture?.stop()
inputCapture = nil
statsTimer?.invalidate()
statsTimer = nil
if let conn = connection {
// close() waits out an in-flight poll (100 ms) and joins the Rust worker
// threads keep that off the main actor.
Task.detached { conn.close() }
}
connection = nil
fps = 0
mbps = 0
}
/// Called (via the main actor) when the pump hits end-of-session.
func sessionEnded() {
guard connection != nil else { return }
disconnect()
errorMessage = "Session ended by host."
}
private func startInput(_ conn: LumenConnection) {
let capture = InputCapture(connection: conn)
capture.start()
inputCapture = capture
}
private func startStatsTimer() {
let timer = Timer(timeInterval: 1.0, repeats: true) { [weak self] _ in
guard let self else { return }
Task { @MainActor in
let (frames, bytes, total) = self.meter.drain()
self.fps = frames
self.mbps = Double(bytes) * 8 / 1_000_000
self.totalFrames = total
}
}
// .common so the HUD keeps updating during window drags / menu tracking.
RunLoop.main.add(timer, forMode: .common)
statsTimer = timer
}
}