From 3ea096ace9d07b4e21e4af75c3b8413dc50c6cf4 Mon Sep 17 00:00:00 2001 From: enricobuehler Date: Wed, 10 Jun 2026 07:28:41 +0000 Subject: [PATCH] =?UTF-8?q?feat:=20M4=20groundwork=20=E2=80=94=20lumen/1?= =?UTF-8?q?=20client=20connector=20in=20the=20C=20ABI=20+=20SwiftUI=20clie?= =?UTF-8?q?nt=20scaffold?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The shared-core architecture pays off: platform clients now link ONE Rust library that does the entire lumen/1 protocol, and only add decode/present/input on top. lumen-core: - client.rs (quic feature): NativeClient — QUIC handshake + UDP data plane + input datagrams on internal threads; embedder surface = connect / next_frame / send_input. - abi.rs: lumen_connect / lumen_connection_next_au (borrow-until-next-call, matching lumen_client_poll_frame semantics) / lumen_connection_send_input / lumen_connection_mode / lumen_connection_close. Guarded in the generated header by LUMEN_FEATURE_QUIC (cbindgen [defines] mapping), so the checked-in header is stable across feature sets. - error.rs: append-only LumenStatus additions Timeout (-9) and Closed (-10). - TESTED end-to-end through the C ABI: in-process lumen/1 host, lumen_connect pulls 25 byte-verified frames, sends input, closes (m3.rs::c_abi_connection_roundtrip). Apple client (clients/apple — SCAFFOLD, written on Linux, first Xcode build pending): - scripts/build-xcframework.sh: cargo per Apple target → universal staticlib + header (LUMEN_FEATURE_QUIC pre-defined) + modulemap → LumenCore.xcframework. - Package.swift (LumenKit) + Swift sources: LumenConnection (ABI wrapper), AnnexB (in-band VPS/SPS/PPS → CMVideoFormatDescription, Annex-B → AVCC CMSampleBuffers with DisplayImmediately), StreamView (SwiftUI over AVSampleBufferDisplayLayer — stage-1 presenter that hardware-decodes compressed HEVC itself), InputCapture (GCMouse raw deltas + GCKeyboard HID→VK). - README.md is the full handoff for the next (Mac-side) agent: build steps, ABI contract, first-light test recipe against the Linux host, stage-2 (VT+Metal pacing) plan, and the known host-side gaps (single-session m3-host, no lumen/1 audio yet, gamepad kinds not yet routed in m3's injector, seed-stage trust). Co-Authored-By: Claude Opus 4.8 (1M context) --- CLAUDE.md | 8 +- clients/apple/Package.swift | 26 +++ clients/apple/README.md | 111 +++++++-- clients/apple/Sources/LumenKit/AnnexB.swift | 140 ++++++++++++ .../apple/Sources/LumenKit/InputCapture.swift | 103 +++++++++ .../Sources/LumenKit/LumenConnection.swift | 106 +++++++++ .../apple/Sources/LumenKit/StreamView.swift | 83 +++++++ clients/apple/module.modulemap | 7 - crates/lumen-core/build.rs | 2 + crates/lumen-core/cbindgen.toml | 3 + crates/lumen-core/src/abi.rs | 173 ++++++++++++++ crates/lumen-core/src/client.rs | 215 ++++++++++++++++++ crates/lumen-core/src/error.rs | 8 + crates/lumen-core/src/lib.rs | 2 + crates/lumen-host/src/m3.rs | 73 ++++++ include/lumen_core.h | 59 +++++ scripts/build-xcframework.sh | 54 +++++ 17 files changed, 1147 insertions(+), 26 deletions(-) create mode 100644 clients/apple/Package.swift create mode 100644 clients/apple/Sources/LumenKit/AnnexB.swift create mode 100644 clients/apple/Sources/LumenKit/InputCapture.swift create mode 100644 clients/apple/Sources/LumenKit/LumenConnection.swift create mode 100644 clients/apple/Sources/LumenKit/StreamView.swift delete mode 100644 clients/apple/module.modulemap create mode 100644 crates/lumen-core/src/client.rs create mode 100644 scripts/build-xcframework.sh diff --git a/CLAUDE.md b/CLAUDE.md index 9c6ae5d..55732dc 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -36,8 +36,12 @@ Low-latency desktop/game streaming stack, Linux-first, with a shared Rust protoc ## What's left -1. **M4 — client decode + present**: VAAPI/NVDEC + wgpu on `lumen-client-rs`'s skeleton; - then real glass-to-glass numbers via `tools/latency-probe` (scaffold). +1. **M4 — client decode + present**: the SwiftUI client is scaffolded and handed off — + the lumen/1 connector is in the C ABI (`lumen_connect` & co., ABI-roundtrip-tested) with + an xcframework build script + LumenKit Swift package; **see + [`clients/apple/README.md`](clients/apple/README.md) for the Mac-side pickup**. Then + glass-to-glass numbers via `tools/latency-probe` (scaffold). The Linux reference client + (`lumen-client-rs`) gets VAAPI + wgpu on the same connector later. 2. **Sub-frame pipelining**: overlap encode and transmit within a frame. Requires a direct NVENC SDK wrapper (libavcodec only emits whole AUs) — the next big latency lever (~2–4 ms at high res). diff --git a/clients/apple/Package.swift b/clients/apple/Package.swift new file mode 100644 index 0000000..b2beb43 --- /dev/null +++ b/clients/apple/Package.swift @@ -0,0 +1,26 @@ +// swift-tools-version: 5.9 +// LumenKit — Swift wrapper around the lumen-core C ABI (lumen/1 client connector) plus the +// SwiftUI/VideoToolbox presentation layer. Build LumenCore.xcframework first: +// bash ../../scripts/build-xcframework.sh (on a Mac; see README.md) +import PackageDescription + +let package = Package( + name: "LumenKit", + platforms: [.macOS(.v14), .iOS(.v17)], + products: [ + .library(name: "LumenKit", targets: ["LumenKit"]) + ], + targets: [ + .binaryTarget(name: "LumenCore", path: "LumenCore.xcframework"), + .target( + name: "LumenKit", + dependencies: ["LumenCore"], + linkerSettings: [ + // Rust staticlib system deps. + .linkedFramework("Security"), + .linkedFramework("SystemConfiguration"), + .linkedLibrary("resolv"), + ] + ), + ] +) diff --git a/clients/apple/README.md b/clients/apple/README.md index 12eaef5..a60a41e 100644 --- a/clients/apple/README.md +++ b/clients/apple/README.md @@ -1,22 +1,99 @@ -# lumen Apple client (M5) +# lumen Apple client (SwiftUI) — handoff -Swift + VideoToolbox (decode) + Metal (present) + SwiftUI, linking `lumen-core` through -the generated C ABI — **no glue layer**. Imports `include/lumen_core.h` via a module map. +The native macOS/iOS client for **`lumen/1`** (the post-GameStream protocol). All +networking/protocol work — QUIC control plane, UDP data plane, GF(2¹⁶) FEC, AES-GCM, +input datagrams — lives in the shared Rust core and is **done and tested**; this package +is the Swift shell: decode (VideoToolbox), present (SwiftUI), input capture. -## Wiring +## What exists (built + tested on the Linux host) -1. Build the core as a static or dynamic library for Apple targets: - ```sh - rustup target add aarch64-apple-ios aarch64-apple-darwin - cargo build -p lumen-core --release --target aarch64-apple-darwin # liblumen_core.a / .dylib - ``` -2. Expose the C ABI to Swift with a module map (`module.modulemap` here) that points at - the checked-in header `../../include/lumen_core.h`. -3. In Swift: create a client `LumenSession`, `lumen_client_poll_frame` on a display-link - thread, feed the access unit to a `VTDecompressionSession`, present the `CVImageBuffer` - with Metal aligned to the screen's refresh (frame pacing, plan §7). +- **The connector**: `lumen_core::client::NativeClient` (Rust) exposed over the C ABI as + `lumen_connect` / `lumen_connection_next_au` / `lumen_connection_send_input` / + `lumen_connection_mode` / `lumen_connection_close` (see `include/lumen_core.h`, guarded + by `LUMEN_FEATURE_QUIC`). **End-to-end tested through the C ABI** against an in-process + host (`crates/lumen-host/src/m3.rs::tests::c_abi_connection_roundtrip`). +- **The host to test against**: `lumen-host m3-host --source virtual --seconds 60` on the + Linux box (it creates a native virtual output at whatever mode the client requests and + streams HEVC; `LUMEN_COMPOSITOR=gamescope LUMEN_GAMESCOPE_APP=vkcube` for moving content). +- **This package (SCAFFOLD — written on Linux, never compiled in Xcode)**: + - `LumenConnection.swift` — Swift wrapper over the C ABI (AUs copied into `Data`). + - `AnnexB.swift` — in-band VPS/SPS/PPS → `CMVideoFormatDescription`; Annex-B → AVCC + `CMSampleBuffer` with `DisplayImmediately` set. + - `StreamView.swift` — SwiftUI `NSViewRepresentable` over `AVSampleBufferDisplayLayer` + (stage-1 presenter: the layer hardware-decodes compressed HEVC itself). + - `InputCapture.swift` — `GCMouse` raw deltas + `GCKeyboard` HID→VK mapping → + `lumen_connection_send_input`. -## Status +## Build steps (on the Mac) -Scaffold. The client half of `lumen_core` (`poll_frame`, FEC recovery, reassembly) is -complete and tested; this target adds the platform decode + present. +```sh +rustup target add aarch64-apple-darwin x86_64-apple-darwin +bash scripts/build-xcframework.sh # → clients/apple/LumenCore.xcframework +open clients/apple/Package.swift # or add the package to an Xcode app project +``` + +Minimal app around it: + +```swift +@main struct LumenApp: App { + var body: some Scene { WindowGroup { ContentView() } } +} +struct ContentView: View { + @State private var conn: LumenConnection? + var body: some View { + if let conn { + StreamView(connection: conn) + .onAppear { InputCapture(connection: conn).start() } + } else { + Button("Connect") { + conn = try? LumenConnection( + host: "192.168.1.70", width: 2560, height: 1440, refreshHz: 120) + } + } + } +} +``` + +## Handoff — what the next agent needs to know + +1. **Expect small compile fixes.** Every Swift file is flagged SCAFFOLD: API-checked from + documentation, never run through Xcode. Likely friction: the imported C enum spellings + (`LUMEN_STATUS_OK` etc. — cbindgen emits `QualifiedScreamingSnakeCase`), `LumenFrame()` + zero-init, `_pad` tuple shape on `LumenInputEvent`. +2. **ABI contract** (matches `lumen_core.h` docs): `next_au`'s pointer is valid only until + the *next* call on that handle (we copy to `Data` immediately); one pump thread per + connection; `send_input` is enqueue-only and thread-safe alongside it; `close` joins the + Rust threads — never call it with a `next_au` call in flight. +3. **Decode flow**: the host opens every stream with an IDR carrying VPS/SPS/PPS in-band, + and recovery keyframes re-send them — so "wait for the first format description, refresh + it on every IDR" (already what `StreamView` does) is sufficient; there is no out-of-band + extradata, ever. +4. **First-light test**: Linux box runs + `PATH=/tmp/gamescope-src/build/src:$PATH LUMEN_COMPOSITOR=gamescope \ + LUMEN_GAMESCOPE_APP=vkcube LUMEN_ZEROCOPY=1 cargo run -rp lumen-host -- m3-host + --source virtual --seconds 120`; Mac connects with the app. Success = the spinning + vkcube on glass. Then mouse/keys should appear inside the gamescope session (verify + with `LUMEN_GAMESCOPE_APP=xev` and the box-side log `/tmp/lumen-gamescope.log`). +5. **Stage 2 (after first light)**: replace `AVSampleBufferDisplayLayer` with explicit + `VTDecompressionSession` + `CAMetalLayer` for frame-pacing control (ProMotion/120 Hz), + and add glass-to-glass measurement (`tools/latency-probe` is the scaffold; the host + already stamps `pts_ns` with its capture wall clock — across machines you'll need a + clock-offset estimate from the QUIC RTT, or the probe's visual timestamp loop). +6. **Gamepads**: `GCController` → `GamepadButton`/`GamepadAxis` `LumenInputEvent`s. The + host does NOT yet route those kinds in `m3.rs`'s injector path (mouse/keys work; the + gamepad kinds need a `GamepadManager` hookup like the GameStream control stream has — + small host-side task). +7. **Trust model is seed-stage**: the client accepts any host certificate + (`endpoint::client_insecure`). Pairing + pinning is a planned lumen-core task; design it + alongside this client's "add host" UX. +8. **iOS**: same package (`BUILD_IOS=1` for the xcframework slice); `StreamView` needs the + `UIViewRepresentable` twin and touch→input mapping. + +## Known limitations of the current host (relevant to client UX) + +- `m3-host` serves **one session and exits** — fine for development; the persistent + lumen/1 listener (serve-style) is a small host-side task. +- No audio on lumen/1 yet (the GameStream path has it; porting the Opus stream onto a + second datagram flow is straightforward). +- Mid-stream renegotiation (resolution change without reconnect) is designed-for but not + implemented (the Welcome is one-shot today). diff --git a/clients/apple/Sources/LumenKit/AnnexB.swift b/clients/apple/Sources/LumenKit/AnnexB.swift new file mode 100644 index 0000000..49608b3 --- /dev/null +++ b/clients/apple/Sources/LumenKit/AnnexB.swift @@ -0,0 +1,140 @@ +// Annex-B HEVC → CoreMedia plumbing. +// +// The lumen host emits Annex-B access units with in-band VPS/SPS/PPS on every IDR +// (deliberately — the client needs no out-of-band extradata). VideoToolbox wants the AVCC +// flavor instead: a CMVideoFormatDescription built from the parameter sets, and sample +// buffers whose NALs are 4-byte-length-prefixed. This file converts between the two. +// +// SCAFFOLD: written on the Linux host, not yet compiled against Xcode. + +import CoreMedia +import Foundation + +public enum AnnexB { + /// Split an Annex-B stream into NAL units (start codes 00 00 01 / 00 00 00 01 stripped). + public static func nalUnits(in data: Data) -> [Data] { + var nals: [Data] = [] + let bytes = [UInt8](data) + var i = 0 + var start = -1 + while i + 2 < bytes.count { + if bytes[i] == 0, bytes[i + 1] == 0, bytes[i + 2] == 1 { + let codeStart = (i > 0 && bytes[i - 1] == 0) ? i - 1 : i + if start >= 0 { + nals.append(Data(bytes[start..= 0, start < bytes.count { + nals.append(Data(bytes[start...])) + } + return nals + } + + /// HEVC NAL unit type (bits 1..6 of the first byte). + public static func hevcNalType(_ nal: Data) -> UInt8 { + guard let first = nal.first else { return 0xFF } + return (first >> 1) & 0x3F + } + + /// Build a format description from an IDR AU's in-band VPS(32)/SPS(33)/PPS(34). + /// Returns nil when the AU carries no parameter sets (non-IDR). + public static func formatDescription(fromIDR au: Data) -> CMVideoFormatDescription? { + var vps: Data?, sps: Data?, pps: Data? + for nal in nalUnits(in: au) { + switch hevcNalType(nal) { + case 32: vps = nal + case 33: sps = nal + case 34: pps = nal + default: break + } + } + guard let vps, let sps, let pps else { return nil } + + var format: CMVideoFormatDescription? + let sets = [vps, sps, pps] + let status: OSStatus = sets[0].withUnsafeBytes { v in + sets[1].withUnsafeBytes { s in + sets[2].withUnsafeBytes { p in + let pointers: [UnsafePointer] = [ + v.bindMemory(to: UInt8.self).baseAddress!, + s.bindMemory(to: UInt8.self).baseAddress!, + p.bindMemory(to: UInt8.self).baseAddress!, + ] + let sizes = [vps.count, sps.count, pps.count] + return CMVideoFormatDescriptionCreateFromHEVCParameterSets( + allocator: kCFAllocatorDefault, + parameterSetCount: 3, + parameterSetPointers: pointers, + parameterSetSizes: sizes, + nalUnitHeaderLength: 4, + extensions: nil, + formatDescriptionOut: &format) + } + } + } + return status == noErr ? format : nil + } + + /// Re-pack an Annex-B AU as AVCC (4-byte big-endian length before each NAL), dropping + /// the parameter-set NALs (they live in the format description). + public static func avcc(from au: Data) -> Data { + var out = Data(capacity: au.count + 16) + for nal in nalUnits(in: au) { + let t = hevcNalType(nal) + if t == 32 || t == 33 || t == 34 { continue } // VPS/SPS/PPS + var len = UInt32(nal.count).bigEndian + withUnsafeBytes(of: &len) { out.append(contentsOf: $0) } + out.append(nal) + } + return out + } + + /// Wrap one AU as a decode-ready CMSampleBuffer. + public static func sampleBuffer( + au: AccessUnit, format: CMVideoFormatDescription + ) -> CMSampleBuffer? { + let avccData = avcc(from: au.data) + var blockBuffer: CMBlockBuffer? + guard CMBlockBufferCreateWithMemoryBlock( + allocator: kCFAllocatorDefault, memoryBlock: nil, + blockLength: avccData.count, blockAllocator: kCFAllocatorDefault, + customBlockSource: nil, offsetToData: 0, dataLength: avccData.count, + flags: 0, blockBufferOut: &blockBuffer) == noErr, + let block = blockBuffer + else { return nil } + let copied = avccData.withUnsafeBytes { raw in + CMBlockBufferReplaceDataBytes( + with: raw.baseAddress!, blockBuffer: block, + offsetIntoDestination: 0, dataLength: avccData.count) + } + guard copied == noErr else { return nil } + + var timing = CMSampleTimingInfo( + duration: .invalid, + presentationTimeStamp: CMTime(value: Int64(au.ptsNs), timescale: 1_000_000_000), + decodeTimeStamp: .invalid) + var sampleSize = avccData.count + var sample: CMSampleBuffer? + guard CMSampleBufferCreate( + allocator: kCFAllocatorDefault, dataBuffer: block, dataReady: true, + makeDataReadyCallback: nil, refcon: nil, formatDescription: format, + sampleCount: 1, sampleTimingEntryCount: 1, sampleTimingArray: &timing, + sampleSizeEntryCount: 1, sampleSizeArray: &sampleSize, + sampleBufferOut: &sample) == noErr + else { return nil } + // Low-latency display: render on arrival, don't wait for a clock. + if let attachments = CMSampleBufferGetSampleAttachmentsArray(sample!, createIfNecessary: true) { + let dict = unsafeBitCast(CFArrayGetValueAtIndex(attachments, 0), to: CFMutableDictionary.self) + CFDictionarySetValue( + dict, + Unmanaged.passUnretained(kCMSampleAttachmentKey_DisplayImmediately).toOpaque(), + Unmanaged.passUnretained(kCFBooleanTrue).toOpaque()) + } + return sample + } +} diff --git a/clients/apple/Sources/LumenKit/InputCapture.swift b/clients/apple/Sources/LumenKit/InputCapture.swift new file mode 100644 index 0000000..583173a --- /dev/null +++ b/clients/apple/Sources/LumenKit/InputCapture.swift @@ -0,0 +1,103 @@ +// Input capture → lumen/1 datagrams, via the GameController framework. +// +// GCMouse delivers RAW deltas (not the accelerated cursor) — exactly what the host-side +// injector expects for relative motion. GCKeyboard gives HID keycodes which we map to the +// Windows VK space the host's vk_to_evdev table consumes (same space Moonlight uses). +// Gamepads (GCController) come later — the host's uinput pads already speak the +// GamepadButton/GamepadAxis event kinds. +// +// SCAFFOLD: written on the Linux host, not yet compiled against Xcode. The VK map covers +// the common keys; extend alongside lumen-host/src/inject.rs::vk_to_evdev. + +#if os(macOS) +import Foundation +import GameController +import LumenCore + +public final class InputCapture { + private let connection: LumenConnection + private var observers: [NSObjectProtocol] = [] + + public init(connection: LumenConnection) { + self.connection = connection + } + + /// Begin forwarding the current (and future) mouse/keyboard to the host. + public func start() { + if let mouse = GCMouse.current { attach(mouse: mouse) } + if let keyboard = GCKeyboard.coalesced { attach(keyboard: keyboard) } + observers.append(NotificationCenter.default.addObserver( + forName: .GCMouseDidConnect, object: nil, queue: .main + ) { [weak self] n in + if let m = n.object as? GCMouse { self?.attach(mouse: m) } + }) + observers.append(NotificationCenter.default.addObserver( + forName: .GCKeyboardDidConnect, object: nil, queue: .main + ) { [weak self] n in + if let k = n.object as? GCKeyboard { self?.attach(keyboard: k) } + }) + } + + public func stop() { + observers.forEach(NotificationCenter.default.removeObserver(_:)) + observers.removeAll() + } + + private func attach(mouse: GCMouse) { + guard let input = mouse.mouseInput else { return } + let conn = connection + input.mouseMovedHandler = { _, dx, dy in + // GC gives +y up; the host expects screen-space (+y down). + conn.send(.mouseMove(dx: Int32(dx), dy: Int32(-dy))) + } + input.leftButton.pressedChangedHandler = { _, _, pressed in + conn.send(.mouseButton(1, down: pressed)) + } + input.rightButton?.pressedChangedHandler = { _, _, pressed in + conn.send(.mouseButton(3, down: pressed)) + } + input.middleButton?.pressedChangedHandler = { _, _, pressed in + conn.send(.mouseButton(2, down: pressed)) + } + input.scroll.valueChangedHandler = { _, _, dy in + if dy != 0 { conn.send(.scroll(Int32(dy * 120))) } + } + } + + private func attach(keyboard: GCKeyboard) { + let conn = connection + keyboard.keyboardInput?.keyChangedHandler = { _, _, keyCode, pressed in + if let vk = Self.hidToVK[keyCode.rawValue] { + conn.send(.key(vk, down: pressed)) + } + } + } + + /// HID usage (GCKeyCode raw) → Windows VK (the host maps VK → evdev). + static let hidToVK: [Int: UInt32] = { + var m: [Int: UInt32] = [:] + // a–z: HID 0x04..0x1D → VK 'A'..'Z'. + for i in 0..<26 { m[0x04 + i] = UInt32(0x41 + i) } + // 1–9, 0: HID 0x1E..0x27 → VK '1'..'9','0'. + for i in 0..<9 { m[0x1E + i] = UInt32(0x31 + i) } + m[0x27] = 0x30 + m[0x28] = 0x0D // return + m[0x29] = 0x1B // escape + m[0x2A] = 0x08 // backspace + m[0x2B] = 0x09 // tab + m[0x2C] = 0x20 // space + m[0x2D] = 0xBD; m[0x2E] = 0xBB // - = + m[0x2F] = 0xDB; m[0x30] = 0xDD; m[0x31] = 0xDC // [ ] backslash + m[0x33] = 0xBA; m[0x34] = 0xDE; m[0x35] = 0xC0 // ; ' ` + m[0x36] = 0xBC; m[0x37] = 0xBE; m[0x38] = 0xBF // , . / + // F1..F12: HID 0x3A..0x45 → VK 0x70..0x7B. + for i in 0..<12 { m[0x3A + i] = UInt32(0x70 + i) } + m[0x4F] = 0x27; m[0x50] = 0x25; m[0x51] = 0x28; m[0x52] = 0x26 // arrows R L D U + m[0x49] = 0x2D; m[0x4A] = 0x24; m[0x4B] = 0x21 // insert home pageup + m[0x4C] = 0x2E; m[0x4D] = 0x23; m[0x4E] = 0x22 // delete end pagedown + m[0xE0] = 0xA2; m[0xE1] = 0xA0; m[0xE2] = 0xA4; m[0xE3] = 0x5B // Lctrl Lshift Lalt Lcmd + m[0xE4] = 0xA3; m[0xE5] = 0xA1; m[0xE6] = 0xA5; m[0xE7] = 0x5C // Rctrl Rshift Ralt Rcmd + return m + }() +} +#endif diff --git a/clients/apple/Sources/LumenKit/LumenConnection.swift b/clients/apple/Sources/LumenKit/LumenConnection.swift new file mode 100644 index 0000000..ea597b5 --- /dev/null +++ b/clients/apple/Sources/LumenKit/LumenConnection.swift @@ -0,0 +1,106 @@ +// Swift wrapper around the lumen-core C ABI's lumen/1 connection API. +// +// Threading contract (mirrors the C header): one LumenConnection is used from a single +// pump thread for nextAU(); sendInput() is enqueue-only and safe alongside it. The pointer +// inside an AU is only valid until the next nextAU() call, so we copy into Data here — +// the copy is small (an encoded AU, tens of KB) and keeps the Swift side memory-safe. +// +// SCAFFOLD: written on the Linux host, not yet compiled against Xcode — expect to fix +// trivial issues on first build (see README.md "Handoff"). + +import Foundation +import LumenCore + +/// One reassembled, FEC-recovered, decrypted access unit (Annex-B HEVC from the host). +public struct AccessUnit: Sendable { + public let data: Data + public let ptsNs: UInt64 + public let frameIndex: UInt32 + public let flags: UInt32 +} + +public enum LumenClientError: Error { + case connectFailed + case closed +} + +public final class LumenConnection { + private var handle: OpaquePointer? + + /// Negotiated session mode (host-confirmed). + public private(set) var width: UInt32 = 0 + public private(set) var height: UInt32 = 0 + public private(set) var refreshHz: UInt32 = 0 + + /// Connect and start a session at the requested mode (the host creates a native virtual + /// output at exactly this size/refresh). Blocks up to `timeoutMs`. + public init( + host: String, port: UInt16 = 9777, + width: UInt32, height: UInt32, refreshHz: UInt32, + timeoutMs: UInt32 = 10_000 + ) throws { + handle = host.withCString { cs in + lumen_connect(cs, port, width, height, refreshHz, timeoutMs) + } + guard handle != nil else { throw LumenClientError.connectFailed } + var w: UInt32 = 0, h: UInt32 = 0, hz: UInt32 = 0 + _ = lumen_connection_mode(handle, &w, &h, &hz) + self.width = w + self.height = h + self.refreshHz = hz + } + + /// Pull the next access unit; nil on timeout, throws once the session is closed. + public func nextAU(timeoutMs: UInt32 = 100) throws -> AccessUnit? { + var frame = LumenFrame() + switch lumen_connection_next_au(handle, &frame, timeoutMs) { + case LUMEN_STATUS_OK: + let data = Data(bytes: frame.data, count: frame.len) // copy: ptr valid only until next call + return AccessUnit( + data: data, ptsNs: frame.pts_ns, + frameIndex: frame.frame_index, flags: frame.flags) + case LUMEN_STATUS_NO_FRAME: + return nil + case LUMEN_STATUS_CLOSED: + throw LumenClientError.closed + default: + throw LumenClientError.closed + } + } + + /// Send one input event (delivered to the host as a QUIC datagram). + public func send(_ event: LumenInputEvent) { + var ev = event + _ = lumen_connection_send_input(handle, &ev) + } + + public func close() { + if let h = handle { + lumen_connection_close(h) + handle = nil + } + } + + deinit { close() } +} + +// Convenience constructors for the wire input events (field semantics match +// lumen_core::input::InputEvent; see lumen_core.h). +public extension LumenInputEvent { + static func mouseMove(dx: Int32, dy: Int32) -> LumenInputEvent { + LumenInputEvent(kind: LUMEN_INPUT_KIND_MOUSE_MOVE, _pad: (0, 0, 0), code: 0, x: dx, y: dy, flags: 0) + } + static func mouseButton(_ button: UInt32, down: Bool) -> LumenInputEvent { + LumenInputEvent( + kind: down ? LUMEN_INPUT_KIND_MOUSE_BUTTON_DOWN : LUMEN_INPUT_KIND_MOUSE_BUTTON_UP, + _pad: (0, 0, 0), code: button, x: 0, y: 0, flags: 0) + } + static func key(_ vk: UInt32, down: Bool) -> LumenInputEvent { + LumenInputEvent( + kind: down ? LUMEN_INPUT_KIND_KEY_DOWN : LUMEN_INPUT_KIND_KEY_UP, + _pad: (0, 0, 0), code: vk, x: 0, y: 0, flags: 0) + } + static func scroll(_ delta: Int32) -> LumenInputEvent { + LumenInputEvent(kind: LUMEN_INPUT_KIND_MOUSE_SCROLL, _pad: (0, 0, 0), code: 0, x: delta, y: 0, flags: 0) + } +} diff --git a/clients/apple/Sources/LumenKit/StreamView.swift b/clients/apple/Sources/LumenKit/StreamView.swift new file mode 100644 index 0000000..db90193 --- /dev/null +++ b/clients/apple/Sources/LumenKit/StreamView.swift @@ -0,0 +1,83 @@ +// SwiftUI presentation: AVSampleBufferDisplayLayer fed straight from the lumen/1 connection. +// +// Stage-1 presenter (see README): the layer accepts *compressed* HEVC sample buffers and +// does hardware decode + display itself — fastest path to pixels, IOSurface-backed +// zero-copy on Apple silicon. Stage 2 (explicit VTDecompressionSession + CAMetalLayer) +// replaces this when we start tuning frame pacing / measuring glass-to-glass. +// +// SCAFFOLD: written on the Linux host, not yet compiled against Xcode. macOS-first +// (NSViewRepresentable); the iOS variant is the same layer under UIViewRepresentable. + +#if os(macOS) +import AVFoundation +import SwiftUI + +public struct StreamView: NSViewRepresentable { + private let connection: LumenConnection + + public init(connection: LumenConnection) { + self.connection = connection + } + + public func makeNSView(context: Context) -> StreamLayerView { + let view = StreamLayerView() + view.start(connection: connection) + return view + } + + public func updateNSView(_ view: StreamLayerView, context: Context) {} +} + +public final class StreamLayerView: NSView { + private let displayLayer = AVSampleBufferDisplayLayer() + private var pump: Thread? + private var running = false + + public override init(frame: NSRect) { + super.init(frame: frame) + wantsLayer = true + displayLayer.videoGravity = .resizeAspect + layer = displayLayer + } + + public required init?(coder: NSCoder) { fatalError("not used") } + + /// Pump thread: pull AUs from the connection, wrap, enqueue. The first IDR yields the + /// format description; non-IDR AUs before it are dropped (the host opens with an IDR). + public func start(connection: LumenConnection) { + guard !running else { return } + running = true + let layer = displayLayer + let thread = Thread { [weak self] in + var format: CMVideoFormatDescription? + while self?.running == true { + do { + guard let au = try connection.nextAU(timeoutMs: 100) else { continue } + if let f = AnnexB.formatDescription(fromIDR: au.data) { + format = f // refreshed on every IDR (mode changes included) + } + guard let f = format, + let sample = AnnexB.sampleBuffer(au: au, format: f) + else { continue } + if layer.status == .failed { + layer.flush() + } + layer.enqueue(sample) + } catch { + break // session closed + } + } + } + thread.name = "lumen-pump" + thread.qualityOfService = .userInteractive + pump = thread + thread.start() + } + + public func stop() { + running = false + } + + deinit { running = false } +} +#endif diff --git a/clients/apple/module.modulemap b/clients/apple/module.modulemap deleted file mode 100644 index 8bdacf2..0000000 --- a/clients/apple/module.modulemap +++ /dev/null @@ -1,7 +0,0 @@ -// Exposes the lumen-core C ABI to Swift as `import LumenCore`. -// Point Xcode's "Import Paths" (SWIFT_INCLUDE_PATHS) at this directory, and link -// liblumen_core.a (or .dylib) built via `cargo build -p lumen-core --target `. -module LumenCore { - header "../../include/lumen_core.h" - export * -} diff --git a/crates/lumen-core/build.rs b/crates/lumen-core/build.rs index 070a409..f0a8551 100644 --- a/crates/lumen-core/build.rs +++ b/crates/lumen-core/build.rs @@ -10,6 +10,8 @@ fn main() { println!("cargo:rerun-if-changed=src/abi.rs"); println!("cargo:rerun-if-changed=src/config.rs"); println!("cargo:rerun-if-changed=src/input.rs"); + println!("cargo:rerun-if-changed=src/client.rs"); + println!("cargo:rerun-if-changed=src/error.rs"); println!("cargo:rerun-if-changed=cbindgen.toml"); let crate_dir = env::var("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR"); diff --git a/crates/lumen-core/cbindgen.toml b/crates/lumen-core/cbindgen.toml index 7808474..25a9c53 100644 --- a/crates/lumen-core/cbindgen.toml +++ b/crates/lumen-core/cbindgen.toml @@ -26,3 +26,6 @@ sort_by = "None" [struct] derive_eq = false + +[defines] +"feature = quic" = "LUMEN_FEATURE_QUIC" diff --git a/crates/lumen-core/src/abi.rs b/crates/lumen-core/src/abi.rs index 8e981cd..6d81dac 100644 --- a/crates/lumen-core/src/abi.rs +++ b/crates/lumen-core/src/abi.rs @@ -441,3 +441,176 @@ pub unsafe extern "C" fn lumen_get_stats( LumenStatus::Ok }) } + +// --------------------------------------------------------------------------------------------- +// lumen/1 connection API (`quic` feature) — the embeddable client connector platform clients +// link (SwiftUI/VideoToolbox, Android, …). In the generated header these are guarded by +// `LUMEN_FEATURE_QUIC`; define it when linking a lumen-core built with `--features quic`. +// --------------------------------------------------------------------------------------------- + +/// Opaque handle to a live `lumen/1` connection (QUIC control plane + UDP data plane, all +/// pumped on internal threads). +#[cfg(feature = "quic")] +pub struct LumenConnection { + inner: crate::client::NativeClient, + /// Backs the pointer returned by the last `lumen_connection_next_au` (borrow-until-next-call). + last: Option, +} + +/// Connect to a `lumen/1` host and start a session at `width`x`height`@`refresh_hz`. +/// Blocks up to `timeout_ms` for the handshake. Returns NULL on failure. +/// +/// # Safety +/// `host` is a NUL-terminated UTF-8 string (IP or hostname resolvable by the platform). +#[cfg(feature = "quic")] +#[no_mangle] +pub unsafe extern "C" fn lumen_connect( + host: *const std::os::raw::c_char, + port: u16, + width: u32, + height: u32, + refresh_hz: u32, + timeout_ms: u32, +) -> *mut LumenConnection { + let r = std::panic::catch_unwind(AssertUnwindSafe(|| { + if host.is_null() { + return std::ptr::null_mut(); + } + let host = match unsafe { std::ffi::CStr::from_ptr(host) }.to_str() { + Ok(s) => s, + Err(_) => return std::ptr::null_mut(), + }; + let mode = crate::config::Mode { + width, + height, + refresh_hz, + }; + match crate::client::NativeClient::connect( + host, + port, + mode, + std::time::Duration::from_millis(timeout_ms as u64), + ) { + Ok(c) => Box::into_raw(Box::new(LumenConnection { + inner: c, + last: None, + })), + Err(_) => std::ptr::null_mut(), + } + })); + r.unwrap_or(std::ptr::null_mut()) +} + +/// Pull the next reassembled access unit, waiting up to `timeout_ms`. Returns +/// [`LumenStatus::NoFrame`] on timeout and [`LumenStatus::Closed`] once the session ended. +/// On `Ok`, `*out` borrows connection memory **until the next call** on this handle. +/// +/// # Safety +/// `c` is a valid connection handle used from a single thread; `out` is writable. +#[cfg(feature = "quic")] +#[no_mangle] +pub unsafe extern "C" fn lumen_connection_next_au( + c: *mut LumenConnection, + out: *mut LumenFrame, + timeout_ms: u32, +) -> LumenStatus { + guard(|| { + let c = match unsafe { c.as_mut() } { + Some(c) => c, + None => return LumenStatus::NullPointer, + }; + if out.is_null() { + return LumenStatus::NullPointer; + } + match c + .inner + .next_frame(std::time::Duration::from_millis(timeout_ms as u64)) + { + Ok(frame) => { + c.last = Some(frame); + let f = c.last.as_ref().unwrap(); + unsafe { + *out = LumenFrame { + data: f.data.as_ptr(), + len: f.data.len(), + frame_index: f.frame_index, + pts_ns: f.pts_ns, + flags: f.flags, + }; + } + LumenStatus::Ok + } + Err(e) => e.status(), + } + }) +} + +/// Send one input event to the host as a QUIC datagram (non-blocking enqueue). +/// +/// # Safety +/// `c` is a valid connection handle; `ev` points to a valid [`InputEvent`]. +#[cfg(feature = "quic")] +#[no_mangle] +pub unsafe extern "C" fn lumen_connection_send_input( + c: *mut LumenConnection, + ev: *const InputEvent, +) -> LumenStatus { + guard(|| { + let c = match unsafe { c.as_ref() } { + Some(c) => c, + None => return LumenStatus::NullPointer, + }; + let ev = match unsafe { ev.as_ref() } { + Some(e) => e, + None => return LumenStatus::NullPointer, + }; + match c.inner.send_input(ev) { + Ok(()) => LumenStatus::Ok, + Err(e) => e.status(), + } + }) +} + +/// The host-confirmed session mode (from the Welcome). Safe any time after connect. +/// +/// # Safety +/// `c` is a valid connection handle; out pointers are writable (NULLs are skipped). +#[cfg(feature = "quic")] +#[no_mangle] +pub unsafe extern "C" fn lumen_connection_mode( + c: *const LumenConnection, + width: *mut u32, + height: *mut u32, + refresh_hz: *mut u32, +) -> LumenStatus { + guard(|| { + let c = match unsafe { c.as_ref() } { + Some(c) => c, + None => return LumenStatus::NullPointer, + }; + unsafe { + if !width.is_null() { + *width = c.inner.mode.width; + } + if !height.is_null() { + *height = c.inner.mode.height; + } + if !refresh_hz.is_null() { + *refresh_hz = c.inner.mode.refresh_hz; + } + } + LumenStatus::Ok + }) +} + +/// Close the connection and free the handle (joins the internal threads). NULL is a no-op. +/// +/// # Safety +/// `c` was returned by [`lumen_connect`] and is not used after this call. +#[cfg(feature = "quic")] +#[no_mangle] +pub unsafe extern "C" fn lumen_connection_close(c: *mut LumenConnection) { + if !c.is_null() { + drop(unsafe { Box::from_raw(c) }); + } +} diff --git a/crates/lumen-core/src/client.rs b/crates/lumen-core/src/client.rs new file mode 100644 index 0000000..66fb9f5 --- /dev/null +++ b/crates/lumen-core/src/client.rs @@ -0,0 +1,215 @@ +//! The embeddable `lumen/1` client connector (M4 groundwork), behind the `quic` feature. +//! +//! [`NativeClient::connect`] runs the full client side of the protocol — QUIC handshake +//! ([`crate::quic`]), UDP data plane ([`crate::session::Session`] on a native thread), input +//! datagrams — and hands the embedder a dead-simple surface: *pull reassembled access units, +//! push input events*. This is what the platform clients (SwiftUI/VideoToolbox, Android, …) +//! link via the C ABI (`lumen_connect` & co. in [`crate::abi`]); `lumen-client-rs` is the +//! Rust-native consumer of the same flow. +//! +//! Threading: one worker thread owns a tokio runtime (QUIC control plane only — design +//! invariant) plus a blocking data-plane pump; frames cross to the embedder over a bounded +//! channel. All methods are safe to call from any single embedder thread. + +use crate::config::{Mode, Role}; +use crate::error::{LumenError, Result}; +use crate::input::InputEvent; +use crate::quic::{endpoint, io, Hello, Start, Welcome}; +use crate::session::{Frame, Session}; +use crate::transport::UdpTransport; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::mpsc::{Receiver, RecvTimeoutError, SyncSender}; +use std::sync::Arc; +use std::time::Duration; + +/// Frames buffered between the data-plane pump and the embedder. Small: the embedder +/// (decoder) should drain at frame rate; when it falls behind, the newest frame is dropped +/// (display freshness over completeness — FEC/keyframes recover). +const FRAME_QUEUE: usize = 16; + +pub struct NativeClient { + frames: Receiver, + input_tx: tokio::sync::mpsc::UnboundedSender, + shutdown: Arc, + worker: Option>, + /// The host-confirmed session mode (from the Welcome). + pub mode: Mode, +} + +impl NativeClient { + /// Connect to a `lumen/1` host and start the session at (up to) `mode`. Blocks until the + /// handshake completes or `timeout` elapses. + pub fn connect(host: &str, port: u16, mode: Mode, timeout: Duration) -> Result { + let (frame_tx, frame_rx) = std::sync::mpsc::sync_channel::(FRAME_QUEUE); + let (input_tx, input_rx) = tokio::sync::mpsc::unbounded_channel::(); + let (ready_tx, ready_rx) = std::sync::mpsc::channel::>(); + let shutdown = Arc::new(AtomicBool::new(false)); + + let host = host.to_string(); + let shutdown_w = shutdown.clone(); + let worker = std::thread::Builder::new() + .name("lumen-client".into()) + .spawn(move || { + let rt = match tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .enable_all() + .build() + { + Ok(rt) => rt, + Err(e) => { + let _ = ready_tx.send(Err(LumenError::Io(e))); + return; + } + }; + rt.block_on(worker_main( + host, port, mode, frame_tx, input_rx, ready_tx, shutdown_w, + )); + }) + .map_err(LumenError::Io)?; + + let negotiated = match ready_rx.recv_timeout(timeout) { + Ok(Ok(m)) => m, + Ok(Err(e)) => return Err(e), + Err(_) => { + shutdown.store(true, Ordering::SeqCst); + return Err(LumenError::Timeout); + } + }; + Ok(NativeClient { + frames: frame_rx, + input_tx, + shutdown, + worker: Some(worker), + mode: negotiated, + }) + } + + /// Pull the next reassembled, FEC-recovered access unit; [`LumenError::NoFrame`] on + /// timeout, [`LumenError::Closed`]-class errors once the session ended. + pub fn next_frame(&mut self, timeout: Duration) -> Result { + match self.frames.recv_timeout(timeout) { + Ok(f) => Ok(f), + Err(RecvTimeoutError::Timeout) => Err(LumenError::NoFrame), + Err(RecvTimeoutError::Disconnected) => Err(LumenError::Closed), + } + } + + /// Queue one input event for delivery as a QUIC datagram. + pub fn send_input(&self, ev: &InputEvent) -> Result<()> { + self.input_tx.send(*ev).map_err(|_| LumenError::Closed) + } +} + +impl Drop for NativeClient { + fn drop(&mut self) { + self.shutdown.store(true, Ordering::SeqCst); + if let Some(w) = self.worker.take() { + let _ = w.join(); + } + } +} + +/// The worker: QUIC handshake, then the input task + the blocking data-plane pump. +async fn worker_main( + host: String, + port: u16, + mode: Mode, + frame_tx: SyncSender, + mut input_rx: tokio::sync::mpsc::UnboundedReceiver, + ready_tx: std::sync::mpsc::Sender>, + shutdown: Arc, +) { + let setup = async { + let remote: std::net::SocketAddr = format!("{host}:{port}") + .parse() + .map_err(|_| LumenError::InvalidArg("host:port"))?; + let ep = endpoint::client_insecure() + .map_err(|e| LumenError::Io(std::io::Error::other(e.to_string())))?; + let conn = ep + .connect(remote, "lumen") + .map_err(|_| LumenError::InvalidArg("connect"))? + .await + .map_err(|e| LumenError::Io(std::io::Error::other(e.to_string())))?; + let (mut send, mut recv) = conn + .open_bi() + .await + .map_err(|e| LumenError::Io(std::io::Error::other(e.to_string())))?; + + io::write_msg( + &mut send, + &Hello { + abi_version: crate::ABI_VERSION, + mode, + } + .encode(), + ) + .await?; + let welcome = Welcome::decode(&io::read_msg(&mut recv).await?)?; + + // Reserve our data-plane port, then start the host. + let probe = std::net::UdpSocket::bind("0.0.0.0:0")?; + let udp_port = probe.local_addr()?.port(); + drop(probe); + io::write_msg( + &mut send, + &Start { + client_udp_port: udp_port, + } + .encode(), + ) + .await?; + + let host_udp = std::net::SocketAddr::new(remote.ip(), welcome.udp_port); + let transport = + UdpTransport::connect(&format!("0.0.0.0:{udp_port}"), &host_udp.to_string())?; + let session = Session::new(welcome.session_config(Role::Client), Box::new(transport))?; + Ok::<_, LumenError>((conn, session, welcome.mode)) + }; + + let (conn, mut session, negotiated) = match setup.await { + Ok(t) => t, + Err(e) => { + let _ = ready_tx.send(Err(e)); + return; + } + }; + let _ = ready_tx.send(Ok(negotiated)); + + // Input task: embedder events → QUIC datagrams. + let input_conn = conn.clone(); + tokio::spawn(async move { + while let Some(ev) = input_rx.recv().await { + let _ = input_conn.send_datagram(ev.encode().to_vec().into()); + } + }); + + // Watch for connection close → stop the pump. + { + let shutdown = shutdown.clone(); + let conn = conn.clone(); + tokio::spawn(async move { + conn.closed().await; + shutdown.store(true, Ordering::SeqCst); + }); + } + + // Data-plane pump on a blocking thread: poll the session, hand frames to the embedder. + // try_send drops the newest frame when the embedder lags (freshness over completeness). + let pump_shutdown = shutdown.clone(); + let _ = tokio::task::spawn_blocking(move || { + while !pump_shutdown.load(Ordering::SeqCst) { + match session.poll_frame() { + Ok(frame) => { + let _ = frame_tx.try_send(frame); + } + Err(LumenError::NoFrame) => { + std::thread::sleep(Duration::from_micros(300)); + } + Err(_) => break, + } + } + }) + .await; + + conn.close(0u32.into(), b"client closed"); +} diff --git a/crates/lumen-core/src/error.rs b/crates/lumen-core/src/error.rs index 3d260bc..f383c10 100644 --- a/crates/lumen-core/src/error.rs +++ b/crates/lumen-core/src/error.rs @@ -19,6 +19,10 @@ pub enum LumenError { Unsupported(&'static str), #[error("io error: {0}")] Io(#[from] std::io::Error), + #[error("timed out")] + Timeout, + #[error("session closed")] + Closed, } pub type Result = core::result::Result; @@ -37,6 +41,8 @@ pub enum LumenStatus { Unsupported = -6, Io = -7, NullPointer = -8, + Timeout = -9, + Closed = -10, Panic = -99, } @@ -51,6 +57,8 @@ impl LumenError { LumenError::NoFrame => LumenStatus::NoFrame, LumenError::Unsupported(_) => LumenStatus::Unsupported, LumenError::Io(_) => LumenStatus::Io, + LumenError::Timeout => LumenStatus::Timeout, + LumenError::Closed => LumenStatus::Closed, } } } diff --git a/crates/lumen-core/src/lib.rs b/crates/lumen-core/src/lib.rs index b092953..2ba9240 100644 --- a/crates/lumen-core/src/lib.rs +++ b/crates/lumen-core/src/lib.rs @@ -25,6 +25,8 @@ #![forbid(unsafe_op_in_unsafe_fn)] pub mod abi; +#[cfg(feature = "quic")] +pub mod client; pub mod config; pub mod crypto; pub mod error; diff --git a/crates/lumen-host/src/m3.rs b/crates/lumen-host/src/m3.rs index cc8d567..f6f34b1 100644 --- a/crates/lumen-host/src/m3.rs +++ b/crates/lumen-host/src/m3.rs @@ -285,3 +285,76 @@ fn virtual_stream( tracing::info!(sent, "lumen/1 virtual stream complete"); Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + + /// End-to-end through the C ABI — the exact contract platform clients (Swift) link: + /// in-process lumen/1 host, `lumen_connect` → `lumen_connection_next_au` pulls verified + /// frames → `lumen_connection_send_input` enqueues → `lumen_connection_close`. + #[test] + fn c_abi_connection_roundtrip() { + use lumen_core::abi::{ + lumen_connect, lumen_connection_close, lumen_connection_mode, lumen_connection_next_au, + lumen_connection_send_input, + }; + use lumen_core::error::LumenStatus; + + let host = std::thread::spawn(|| { + run(M3Options { + port: 19777, + source: M3Source::Synthetic, + seconds: 0, + frames: 25, + }) + }); + std::thread::sleep(std::time::Duration::from_millis(500)); + + let addr = std::ffi::CString::new("127.0.0.1").unwrap(); + let conn = unsafe { lumen_connect(addr.as_ptr(), 19777, 1280, 720, 60, 10_000) }; + assert!(!conn.is_null(), "lumen_connect failed"); + + let (mut w, mut h, mut hz) = (0u32, 0u32, 0u32); + assert_eq!( + unsafe { lumen_connection_mode(conn, &mut w, &mut h, &mut hz) }, + LumenStatus::Ok + ); + assert_eq!((w, h, hz), (1280, 720, 60)); + + let mut got = 0u32; + let mut frame = unsafe { std::mem::zeroed() }; + while got < 25 { + match unsafe { lumen_connection_next_au(conn, &mut frame, 2000) } { + LumenStatus::Ok => { + let data = unsafe { std::slice::from_raw_parts(frame.data, frame.len) }; + let idx = u32::from_le_bytes(data[0..4].try_into().unwrap()); + assert_eq!( + data, + &test_frame(idx, data.len())[..], + "frame {idx} content" + ); + got += 1; + } + LumenStatus::NoFrame => continue, + other => panic!("next_au: {other:?}"), + } + } + + let ev = lumen_core::input::InputEvent { + kind: lumen_core::input::InputKind::MouseMove, + _pad: [0; 3], + code: 0, + x: 1, + y: 2, + flags: 0, + }; + assert_eq!( + unsafe { lumen_connection_send_input(conn, &ev) }, + LumenStatus::Ok + ); + + unsafe { lumen_connection_close(conn) }; + host.join().unwrap().unwrap(); + } +} diff --git a/include/lumen_core.h b/include/lumen_core.h index 04b08f7..75628fc 100644 --- a/include/lumen_core.h +++ b/include/lumen_core.h @@ -54,6 +54,8 @@ enum LumenStatus LUMEN_STATUS_UNSUPPORTED = -6, LUMEN_STATUS_IO = -7, LUMEN_STATUS_NULL_POINTER = -8, + LUMEN_STATUS_TIMEOUT = -9, + LUMEN_STATUS_CLOSED = -10, LUMEN_STATUS_PANIC = -99, }; #ifndef __cplusplus @@ -92,6 +94,12 @@ typedef uint8_t LumenInputKind; #endif // __STDC_VERSION__ >= 202311L #endif // __cplusplus +#if defined(LUMEN_FEATURE_QUIC) +// Opaque handle to a live `lumen/1` connection (QUIC control plane + UDP data plane, all +// pumped on internal threads). +typedef struct LumenConnection LumenConnection; +#endif + // Opaque session handle. Pointer-only from C. typedef struct LumenSession LumenSession; @@ -230,6 +238,57 @@ int32_t lumen_host_poll_input(LumenSession *s); // `s` is a valid handle; `out` points to a writable `LumenStats`. LumenStatus lumen_get_stats(LumenSession *s, LumenStats *out); +#if defined(LUMEN_FEATURE_QUIC) +// Connect to a `lumen/1` host and start a session at `width`x`height`@`refresh_hz`. +// Blocks up to `timeout_ms` for the handshake. Returns NULL on failure. +// +// # Safety +// `host` is a NUL-terminated UTF-8 string (IP or hostname resolvable by the platform). +LumenConnection *lumen_connect(const char *host, + uint16_t port, + uint32_t width, + uint32_t height, + uint32_t refresh_hz, + uint32_t timeout_ms); +#endif + +#if defined(LUMEN_FEATURE_QUIC) +// Pull the next reassembled access unit, waiting up to `timeout_ms`. Returns +// [`LumenStatus::NoFrame`] on timeout and [`LumenStatus::Closed`] once the session ended. +// On `Ok`, `*out` borrows connection memory **until the next call** on this handle. +// +// # Safety +// `c` is a valid connection handle used from a single thread; `out` is writable. +LumenStatus lumen_connection_next_au(LumenConnection *c, LumenFrame *out, uint32_t timeout_ms); +#endif + +#if defined(LUMEN_FEATURE_QUIC) +// Send one input event to the host as a QUIC datagram (non-blocking enqueue). +// +// # Safety +// `c` is a valid connection handle; `ev` points to a valid [`InputEvent`]. +LumenStatus lumen_connection_send_input(LumenConnection *c, const LumenInputEvent *ev); +#endif + +#if defined(LUMEN_FEATURE_QUIC) +// The host-confirmed session mode (from the Welcome). Safe any time after connect. +// +// # Safety +// `c` is a valid connection handle; out pointers are writable (NULLs are skipped). +LumenStatus lumen_connection_mode(const LumenConnection *c, + uint32_t *width, + uint32_t *height, + uint32_t *refresh_hz); +#endif + +#if defined(LUMEN_FEATURE_QUIC) +// Close the connection and free the handle (joins the internal threads). NULL is a no-op. +// +// # Safety +// `c` was returned by [`lumen_connect`] and is not used after this call. +void lumen_connection_close(LumenConnection *c); +#endif + #ifdef __cplusplus } // extern "C" #endif // __cplusplus diff --git a/scripts/build-xcframework.sh b/scripts/build-xcframework.sh new file mode 100644 index 0000000..9330daf --- /dev/null +++ b/scripts/build-xcframework.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# Build LumenCore.xcframework for the Apple clients — run ON A MAC with Xcode + rustup. +# +# rustup target add aarch64-apple-darwin x86_64-apple-darwin # + aarch64-apple-ios for iOS +# bash scripts/build-xcframework.sh +# +# Output: clients/apple/LumenCore.xcframework (consumed by clients/apple/Package.swift). +# The library is built WITH the `quic` feature (the lumen/1 connection API), so the bundled +# header gets LUMEN_FEATURE_QUIC pre-defined — Swift sees lumen_connect & co. unconditionally. +set -euo pipefail +cd "$(dirname "$0")/.." + +TARGETS_MAC=(aarch64-apple-darwin x86_64-apple-darwin) +BUILD_IOS="${BUILD_IOS:-0}" # BUILD_IOS=1 adds an iOS slice (requires the ios target installed) + +for t in "${TARGETS_MAC[@]}"; do + cargo build --release -p lumen-core --features quic --target "$t" +done +if [[ "$BUILD_IOS" == "1" ]]; then + cargo build --release -p lumen-core --features quic --target aarch64-apple-ios +fi + +STAGE="$(mktemp -d)" +trap 'rm -rf "$STAGE"' EXIT + +# Universal macOS static lib. +mkdir -p "$STAGE/macos" +lipo -create \ + target/aarch64-apple-darwin/release/liblumen_core.a \ + target/x86_64-apple-darwin/release/liblumen_core.a \ + -output "$STAGE/macos/liblumen_core.a" + +# Headers dir: the generated C header (with the quic API force-enabled) + a modulemap so +# Swift can `import LumenCore`. +mkdir -p "$STAGE/include" +{ + echo "#define LUMEN_FEATURE_QUIC 1" + cat include/lumen_core.h +} > "$STAGE/include/lumen_core.h" +cat > "$STAGE/include/module.modulemap" <<'EOF' +module LumenCore { + header "lumen_core.h" + export * +} +EOF + +ARGS=(-library "$STAGE/macos/liblumen_core.a" -headers "$STAGE/include") +if [[ "$BUILD_IOS" == "1" ]]; then + ARGS+=(-library target/aarch64-apple-ios/release/liblumen_core.a -headers "$STAGE/include") +fi + +rm -rf clients/apple/LumenCore.xcframework +xcodebuild -create-xcframework "${ARGS[@]}" -output clients/apple/LumenCore.xcframework +echo "OK: clients/apple/LumenCore.xcframework"