feat: M4 stage 1 — the SwiftUI client is real: compiles, tested, first light on glass
ci / rust (push) Has been cancelled

The clients/apple scaffold is now a working macOS client, validated live against this
repo's host across the LAN: gamescope virtual output → NVENC HEVC → lumen/1 (GF(2¹⁶) FEC +
AES-GCM over UDP, QUIC control) → VideoToolbox → AVSampleBufferDisplayLayer at 720p60,
mouse/keyboard flowing back as QUIC datagrams into the host's gamescope EIS injector
(~3.7k events injected in one session).

LumenKit:
- LumenConnection: the predicted cbindgen compile fixes (C17 header spells the typedefs as
  integers while the enum constants import as a distinct Swift type — bridge by rawValue);
  close() is now safe from any thread (a close flag + pumpLock held across the blocking
  poll enforce the C contract "never close with a next_au in flight"; flag prevents
  lock-starvation by back-to-back polls).
- StreamView: per-pump cancellation token (reconnects can't double-pump), flush + re-gate
  on the next in-band parameter sets when the layer fails, no stale enqueue after restart.
- InputCapture: fractional-delta accumulation (sub-pixel motion isn't truncated away),
  pressed-state tracking with release-all on focus loss and stop() (nothing sticks down
  host-side), global-singleton ownership guard (GC has one handler slot per process),
  X1/X2 buttons, horizontal scroll, full keypad/CapsLock/ISO-102nd/PrintScreen/Menu VKs.
- LumenClient app shell (swift run LumenClient): connect form, fps/Mb-s HUD,
  LUMEN_AUTOCONNECT/LUMEN_MODE for scripted first-light runs.
- Tests: Annex-B byte-level units; real-codec round trip (VTCompressionSession-encoded
  HEVC rebuilt as the host's wire shape → AnnexB → VTDecompressionSession → pixels);
  test-loopback.sh (Swift client vs a real local m3-host over loopback — the Swift twin of
  c_abi_connection_roundtrip); RemoteFirstLightTests (full pipeline over the LAN).

Host/build fixes that fell out:
- The workspace builds on non-Linux again: gamestream audio (opus) and sendmmsg batching
  are now platform-gated with stubs/fallback, per the crate's "compiles everywhere" rule.
- Horizontal scroll was inverted end-to-end: the injectors negated BOTH axes onto the
  ei/wl axes, but GameStream's horizontal convention is positive = right
  (moonlight-qt/Sunshine pass it through unnegated) — only vertical flips now. This also
  un-inverts real Moonlight clients.
- AnnexB drops all zeros preceding a start code (trailing_zero_8bits padding), ffmpeg's
  policy, instead of leaking them into the preceding NAL.
- build-xcframework.sh: deployment targets pinned to the package floor + an otool guard —
  cargo does not fingerprint MACOSX_DEPLOYMENT_TARGET, so warm caches can silently ship
  too-new minos objects.

Adversarially reviewed (5-dimension multi-agent pass, every finding refutation-verified):
14 confirmed findings, all fixed above; the send-while-polling core-contract gap flagged
here is closed by the lumen/1 session-planes work (&self pulls + per-plane borrow slots).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
2026-06-10 14:38:01 +02:00
parent 520d7342dd
commit bf8a974e8b
23 changed files with 1212 additions and 180 deletions
+80 -20
View File
@@ -5,8 +5,8 @@
// zero-copy on Apple silicon. Stage 2 (explicit VTDecompressionSession + CAMetalLayer)
// replaces this when we start tuning frame pacing / measuring glass-to-glass.
//
// SCAFFOLD: written on the Linux host, not yet compiled against Xcode. macOS-first
// (NSViewRepresentable); the iOS variant is the same layer under UIViewRepresentable.
// macOS-first (NSViewRepresentable); the iOS variant is the same layer under
// UIViewRepresentable.
#if os(macOS)
import AVFoundation
@@ -14,70 +14,130 @@ import SwiftUI
public struct StreamView: NSViewRepresentable {
private let connection: LumenConnection
private let onFrame: (@Sendable (AccessUnit) -> Void)?
private let onSessionEnd: (@Sendable () -> Void)?
public init(connection: LumenConnection) {
/// `onFrame`/`onSessionEnd` fire on the pump thread hop to the main actor for UI.
public init(
connection: LumenConnection,
onFrame: (@Sendable (AccessUnit) -> Void)? = nil,
onSessionEnd: (@Sendable () -> Void)? = nil
) {
self.connection = connection
self.onFrame = onFrame
self.onSessionEnd = onSessionEnd
}
public func makeNSView(context: Context) -> StreamLayerView {
let view = StreamLayerView()
view.start(connection: connection)
view.start(connection: connection, onFrame: onFrame, onSessionEnd: onSessionEnd)
return view
}
public func updateNSView(_ view: StreamLayerView, context: Context) {}
public func updateNSView(_ view: StreamLayerView, context: Context) {
// SwiftUI reuses the NSView across state changes repoint the pump only when the
// connection identity actually changed.
if view.connection !== connection {
view.start(connection: connection, onFrame: onFrame, onSessionEnd: onSessionEnd)
}
}
public static func dismantleNSView(_ view: StreamLayerView, coordinator: ()) {
view.stop()
}
}
public final class StreamLayerView: NSView {
/// Cancellation handle owned by exactly one pump thread a restart hands the old pump
/// its own token, so it can never be revived by a newer start().
private final class PumpToken: @unchecked Sendable {
private let lock = NSLock()
private var live = true
var isLive: Bool {
lock.lock()
defer { lock.unlock() }
return live
}
func cancel() {
lock.lock()
live = false
lock.unlock()
}
}
private let displayLayer = AVSampleBufferDisplayLayer()
private var pump: Thread?
private var running = false
private var token: PumpToken?
public private(set) var connection: LumenConnection?
public override init(frame: NSRect) {
super.init(frame: frame)
wantsLayer = true
displayLayer.videoGravity = .resizeAspect
layer = displayLayer
layer = displayLayer // layer-hosting: assign before wantsLayer
wantsLayer = true
}
public required init?(coder: NSCoder) { fatalError("not used") }
/// Pump thread: pull AUs from the connection, wrap, enqueue. The first IDR yields the
/// format description; non-IDR AUs before it are dropped (the host opens with an IDR).
public func start(connection: LumenConnection) {
guard !running else { return }
running = true
public func start(
connection: LumenConnection,
onFrame: (@Sendable (AccessUnit) -> Void)? = nil,
onSessionEnd: (@Sendable () -> Void)? = nil
) {
stop()
let token = PumpToken()
self.token = token
self.connection = connection
let layer = displayLayer
let thread = Thread { [weak self] in
layer.flush() // drop any frames a previous connection left queued
let thread = Thread {
var format: CMVideoFormatDescription?
while self?.running == true {
while token.isLive {
do {
guard let au = try connection.nextAU(timeoutMs: 100) else { continue }
onFrame?(au)
if let f = AnnexB.formatDescription(fromIDR: au.data) {
format = f // refreshed on every IDR (mode changes included)
}
guard let f = format,
let sample = AnnexB.sampleBuffer(au: au, format: f)
else { continue }
if layer.status == .failed {
// Decode wedged: flush and re-gate on the next in-band parameter
// sets resuming with a delta frame can't recover. (A
// request-IDR channel on lumen/1 is a host-side TODO; with the
// host's infinite GOP this may otherwise stay black until the
// next recovery keyframe.)
layer.flush()
format = AnnexB.formatDescription(fromIDR: au.data)
}
guard let f = format,
let sample = AnnexB.sampleBuffer(au: au, format: f),
token.isLive // don't enqueue a stale frame after a restart
else { continue }
layer.enqueue(sample)
} catch {
if token.isLive {
onSessionEnd?()
}
break // session closed
}
}
}
thread.name = "lumen-pump"
thread.qualityOfService = .userInteractive
pump = thread
thread.start()
}
/// Stop pumping ( one poll timeout). Does not close the connection that stays with
/// whoever owns it (LumenConnection.close() is safe alongside a draining pump).
public func stop() {
running = false
token?.cancel()
token = nil
connection = nil
}
deinit { running = false }
deinit {
token?.cancel()
}
}
#endif