feat(apple): iOS/iPadOS client — touch, pointer lock, shared SwiftUI shell
ci / rust (push) Has been cancelled
ci / rust (push) Has been cancelled
The whole client now runs on iPadOS/iOS from the same sources, first-lit live in the
iPad simulator against the real host at 1280x720@60 (60 fps on the HUD, capture state
machine active, mic permission flow shown).
- PunktfunkCore.xcframework grows iOS device + universal-simulator slices
(BUILD_IOS=1; rustup targets aarch64-apple-ios{,-sim} + x86_64-apple-ios).
- The decode pump is extracted into a shared StreamPump (identical IDR re-gate logic on
both platforms); the iOS StreamView (StreamViewIOS.swift) has the same name/signature
as the macOS one, so ContentView & co. are byte-identical across platforms — hosted
in a UIViewController for prefersPointerLocked (the iPadOS cursor capture; see README
note 9 for the UIHostingController forwarding caveat).
- Touch is always forwarded: per-finger wire ids, coordinates mapped through the
aspect-fit letterbox into LIVE host-mode pixels (surface == host mode, identity
rescale host-side; follows mid-stream requestMode switches).
- InputCapture is cross-platform: GC works the same on iPadOS, ⌘⎋ is detected from the
HID stream there; stale-⌘ tracking after focus loss fixed on both platforms
(releaseAll now drops the modifier/latch state — a ⌘ released in another app
otherwise hijacked Esc forever).
- SessionAudio: AVAudioSession on iOS (.playAndRecord + .defaultToSpeaker — without it
iPhones route host audio to the EARPIECE; deactivated with
notifyOthersOnDeactivation on stop so interrupted background audio resumes); HAL
device pinning + the Settings pickers stay macOS-only.
- New Punktfunk-iOS app target (shared synchronized sources, generated Info.plist with
mic + local-network usage descriptions — QUIC to a LAN host trips local network
privacy on real devices — scene manifest + indirect input events for Stage Manager /
external displays), shared scheme, macOS min-window frames gated off iOS.
For the iPad-on-an-external-screen idea: with multiple scenes + indirect input enabled,
Stage Manager iPads can drag the punktfunk window onto the external display and drive
the PC with keyboard/mouse/touch. Known gaps (README note 9): the pointer-lock
preference isn't consulted through UIHostingController (relative mouse works, the local
cursor just stays visible) and AVAudioSession interruptions don't auto-restart audio.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -25,6 +25,10 @@
|
||||
|
||||
#if os(macOS)
|
||||
import AppKit
|
||||
#endif
|
||||
#if os(iOS)
|
||||
import UIKit
|
||||
#endif
|
||||
import Foundation
|
||||
import GameController
|
||||
import PunktfunkCore
|
||||
@@ -36,7 +40,9 @@ public final class InputCapture {
|
||||
private var observers: [NSObjectProtocol] = []
|
||||
private var mice: [GCMouse] = []
|
||||
private var keyboards: [GCKeyboard] = []
|
||||
#if os(macOS)
|
||||
private var keyEventMonitor: Any?
|
||||
#endif
|
||||
|
||||
// Main-queue-only state (see header comment).
|
||||
private var residualX: Float = 0
|
||||
@@ -53,6 +59,9 @@ public final class InputCapture {
|
||||
/// reaches GCKeyboard, racing the NSEvent monitor — latched here so it can't type
|
||||
/// an Escape into the host in either toggle direction.
|
||||
private var suppressedVK: UInt32?
|
||||
/// Physical ⌘ keys currently held (tracked even while released — the ⌘⎋ toggle and
|
||||
/// its Esc suppression need it in both states).
|
||||
private var cmdKeysDown: Set<UInt32> = []
|
||||
|
||||
/// While true, mouse/keyboard flow to the host and key NSEvents are swallowed
|
||||
/// locally; while false the user is interacting with the local UI (dragging the
|
||||
@@ -119,8 +128,13 @@ public final class InputCapture {
|
||||
if let k = n.object as? GCKeyboard { self?.attach(keyboard: k) }
|
||||
})
|
||||
// Focus loss: GC stops delivering, so release everything still held host-side.
|
||||
#if os(macOS)
|
||||
let resignActive = NSApplication.didResignActiveNotification
|
||||
#else
|
||||
let resignActive = UIApplication.willResignActiveNotification
|
||||
#endif
|
||||
observers.append(NotificationCenter.default.addObserver(
|
||||
forName: NSApplication.didResignActiveNotification, object: nil, queue: .main
|
||||
forName: resignActive, object: nil, queue: .main
|
||||
) { [weak self] _ in
|
||||
self?.releaseAll()
|
||||
})
|
||||
@@ -128,6 +142,8 @@ public final class InputCapture {
|
||||
// that one combo is intercepted: swallowing keys wholesale at the monitor level
|
||||
// risks starving GC's own delivery, so the no-beep behavior lives in
|
||||
// StreamLayerView (first responder consumes keyDown/keyUp while captured).
|
||||
// (On iOS there is no NSEvent monitor — the GC key handler detects the combo.)
|
||||
#if os(macOS)
|
||||
keyEventMonitor = NSEvent.addLocalMonitorForEvents(
|
||||
matching: [.keyDown]
|
||||
) { [weak self] event in
|
||||
@@ -140,16 +156,19 @@ public final class InputCapture {
|
||||
}
|
||||
return event
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
public func stop() {
|
||||
releaseAll()
|
||||
observers.forEach(NotificationCenter.default.removeObserver(_:))
|
||||
observers.removeAll()
|
||||
#if os(macOS)
|
||||
if let monitor = keyEventMonitor {
|
||||
NSEvent.removeMonitor(monitor)
|
||||
keyEventMonitor = nil
|
||||
}
|
||||
#endif
|
||||
// Don't clobber the handlers if a newer capture has taken the global devices.
|
||||
if Self.activeCapture === self || Self.activeCapture == nil {
|
||||
for mouse in mice {
|
||||
@@ -172,8 +191,12 @@ public final class InputCapture {
|
||||
|
||||
deinit { stop() }
|
||||
|
||||
/// Send release events for everything currently held, and drop the motion residuals.
|
||||
/// Send release events for everything currently held, and drop the motion residuals
|
||||
/// and modifier/latch tracking (GC delivers nothing while inactive, so a ⌘ released
|
||||
/// in another app would otherwise stay "held" here forever — hijacking Esc).
|
||||
private func releaseAll() {
|
||||
cmdKeysDown.removeAll()
|
||||
suppressedVK = nil
|
||||
for vk in pressedVKs {
|
||||
connection.send(.key(vk, down: false))
|
||||
}
|
||||
@@ -264,16 +287,32 @@ public final class InputCapture {
|
||||
keyboards.append(keyboard)
|
||||
keyboard.keyboardInput?.keyChangedHandler = { [weak self] _, _, keyCode, pressed in
|
||||
guard let self, let vk = Self.hidToVK[keyCode.rawValue] else { return }
|
||||
if vk == 0x5B || vk == 0x5C { // physical ⌘ state, tracked in both states
|
||||
if pressed {
|
||||
self.cmdKeysDown.insert(vk)
|
||||
} else {
|
||||
self.cmdKeysDown.remove(vk)
|
||||
}
|
||||
}
|
||||
// The ⌘⎋ toggle's Esc — checked before the forwarding gate, because in the
|
||||
// engage direction forwarding is already true when this fires.
|
||||
if vk == self.suppressedVK {
|
||||
if !pressed { self.suppressedVK = nil }
|
||||
return
|
||||
}
|
||||
#if os(iOS)
|
||||
// No NSEvent monitor here — the toggle combo is detected from the HID
|
||||
// stream itself.
|
||||
if pressed, vk == 0x1B, !self.cmdKeysDown.isEmpty {
|
||||
self.suppressedVK = 0x1B
|
||||
self.onToggleCapture?()
|
||||
return
|
||||
}
|
||||
#endif
|
||||
guard self.forwarding else { return }
|
||||
// Release direction of the toggle: GC's Esc-down can beat the NSEvent
|
||||
// monitor — never type Esc into the host while ⌘ is held (⌘⎋ is reserved).
|
||||
if vk == 0x1B, self.pressedVKs.contains(0x5B) || self.pressedVKs.contains(0x5C) {
|
||||
if vk == 0x1B, !self.cmdKeysDown.isEmpty {
|
||||
return
|
||||
}
|
||||
if pressed {
|
||||
@@ -325,4 +364,3 @@ public final class InputCapture {
|
||||
return m
|
||||
}()
|
||||
}
|
||||
#endif
|
||||
|
||||
@@ -14,7 +14,6 @@
|
||||
// AVAudioEngine ties input+output to one aggregate clock, separate engines keep
|
||||
// arbitrary mic/speaker combinations trivial.
|
||||
|
||||
#if os(macOS)
|
||||
import AVFoundation
|
||||
import os
|
||||
|
||||
@@ -140,9 +139,29 @@ public final class SessionAudio {
|
||||
}
|
||||
|
||||
/// Start playback (and, if enabled+authorized, the mic uplink). Empty UIDs = system
|
||||
/// default device. Main thread (engine setup); returns after the engines start —
|
||||
/// the mic may start slightly later if the permission prompt is pending.
|
||||
/// default device; on iOS the UIDs are ignored entirely (routes are
|
||||
/// AVAudioSession-managed). Main thread (engine setup); returns after the engines
|
||||
/// start — the mic may start slightly later if the permission prompt is pending.
|
||||
public func start(speakerUID: String, micUID: String, micEnabled: Bool) {
|
||||
#if os(iOS)
|
||||
// Route + policy live in the session, not per-engine: stereo playback, mic
|
||||
// capture when enabled, Bluetooth allowed. Failure is non-fatal (defaults).
|
||||
let session = AVAudioSession.sharedInstance()
|
||||
do {
|
||||
if micEnabled {
|
||||
// .defaultToSpeaker: .playAndRecord otherwise routes to the iPhone
|
||||
// EARPIECE; only affects the built-in route (headphones/BT still win).
|
||||
try session.setCategory(
|
||||
.playAndRecord, mode: .default,
|
||||
options: [.allowBluetoothA2DP, .defaultToSpeaker])
|
||||
} else {
|
||||
try session.setCategory(.playback, mode: .default)
|
||||
}
|
||||
try session.setActive(true)
|
||||
} catch {
|
||||
log.warning("AVAudioSession setup failed: \(error.localizedDescription)")
|
||||
}
|
||||
#endif
|
||||
startPlayback(speakerUID: speakerUID)
|
||||
guard micEnabled else { return }
|
||||
switch AVCaptureDevice.authorizationStatus(for: .audio) {
|
||||
@@ -180,6 +199,16 @@ public final class SessionAudio {
|
||||
if wasDraining {
|
||||
_ = drainDone.wait(timeout: .now() + .milliseconds(400))
|
||||
}
|
||||
#if os(iOS)
|
||||
// Release the session so audio we interrupted (Music, podcasts) gets its
|
||||
// resume cue.
|
||||
do {
|
||||
try AVAudioSession.sharedInstance().setActive(
|
||||
false, options: .notifyOthersOnDeactivation)
|
||||
} catch {
|
||||
log.warning("AVAudioSession deactivation failed: \(error.localizedDescription)")
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
// MARK: - Playback (host → speaker)
|
||||
@@ -190,6 +219,7 @@ public final class SessionAudio {
|
||||
let ring = AudioRing(capacity: 96_000, prefill: 1920)
|
||||
|
||||
let engine = AVAudioEngine()
|
||||
#if os(macOS)
|
||||
if !speakerUID.isEmpty {
|
||||
if let dev = AudioDevices.deviceID(forUID: speakerUID),
|
||||
let unit = engine.outputNode.audioUnit {
|
||||
@@ -200,6 +230,7 @@ public final class SessionAudio {
|
||||
log.warning("speaker \(speakerUID) not present — using default")
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
// Engine-native deinterleaved float; the render block deinterleaves from the ring.
|
||||
guard let format = AVAudioFormat(standardFormatWithSampleRate: 48_000, channels: 2)
|
||||
@@ -282,6 +313,7 @@ public final class SessionAudio {
|
||||
private func startCapture(micUID: String) {
|
||||
let engine = AVAudioEngine()
|
||||
let input = engine.inputNode
|
||||
#if os(macOS)
|
||||
if !micUID.isEmpty {
|
||||
if let dev = AudioDevices.deviceID(forUID: micUID), let unit = input.audioUnit {
|
||||
if !Self.setDevice(dev, on: unit) {
|
||||
@@ -291,6 +323,7 @@ public final class SessionAudio {
|
||||
log.warning("microphone \(micUID) not present — using default")
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
let inFormat = input.outputFormat(forBus: 0)
|
||||
guard inFormat.sampleRate > 0, inFormat.channelCount > 0 else {
|
||||
@@ -376,11 +409,12 @@ public final class SessionAudio {
|
||||
log.info("mic uplink started (\(micUID.isEmpty ? "default input" : micUID))")
|
||||
}
|
||||
|
||||
#if os(macOS)
|
||||
private static func setDevice(_ id: AudioDeviceID, on unit: AudioUnit) -> Bool {
|
||||
var dev = id
|
||||
return AudioUnitSetProperty(
|
||||
unit, kAudioOutputUnitProperty_CurrentDevice, kAudioUnitScope_Global, 0,
|
||||
&dev, UInt32(MemoryLayout<AudioDeviceID>.size)) == noErr
|
||||
}
|
||||
#endif
|
||||
}
|
||||
#endif
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
// The platform-independent heart of the presenters: one thread pulling AUs from the
|
||||
// connection into an AVSampleBufferDisplayLayer, with the format description refreshed
|
||||
// on every IDR (the host opens with an IDR carrying in-band parameter sets; recovery
|
||||
// keyframes re-send them — there is no out-of-band extradata, ever). Shared by the
|
||||
// macOS StreamLayerView and the iOS/iPadOS stream view.
|
||||
|
||||
import AVFoundation
|
||||
import Foundation
|
||||
|
||||
/// Cancellation handle owned by exactly one pump thread — a restart hands the old pump
|
||||
/// its own token, so it can never be revived by a newer start().
|
||||
private final class PumpToken: @unchecked Sendable {
|
||||
private let lock = NSLock()
|
||||
private var live = true
|
||||
var isLive: Bool {
|
||||
lock.lock()
|
||||
defer { lock.unlock() }
|
||||
return live
|
||||
}
|
||||
func cancel() {
|
||||
lock.lock()
|
||||
live = false
|
||||
lock.unlock()
|
||||
}
|
||||
}
|
||||
|
||||
/// One pump per instance; create a fresh StreamPump per start (cancel is permanent).
|
||||
final class StreamPump {
|
||||
private let token = PumpToken()
|
||||
|
||||
/// Pump thread: pull AUs, wrap, enqueue. Non-IDR AUs before the first format
|
||||
/// description are dropped. `onFrame`/`onSessionEnd` fire on the pump thread.
|
||||
func start(
|
||||
connection: PunktfunkConnection,
|
||||
layer: AVSampleBufferDisplayLayer,
|
||||
onFrame: (@Sendable (AccessUnit) -> Void)?,
|
||||
onSessionEnd: (@Sendable () -> Void)?
|
||||
) {
|
||||
let token = token
|
||||
layer.flush() // drop any frames a previous connection left queued
|
||||
|
||||
let thread = Thread {
|
||||
var format: CMVideoFormatDescription?
|
||||
while token.isLive {
|
||||
do {
|
||||
guard let au = try connection.nextAU(timeoutMs: 100) else { continue }
|
||||
onFrame?(au)
|
||||
if let f = AnnexB.formatDescription(fromIDR: au.data) {
|
||||
format = f // refreshed on every IDR (mode changes included)
|
||||
}
|
||||
if layer.status == .failed {
|
||||
// Decode wedged: flush and re-gate on the next in-band parameter
|
||||
// sets — resuming with a delta frame can't recover. (A
|
||||
// request-IDR channel on punktfunk/1 is a host-side TODO; with the
|
||||
// host's infinite GOP this may otherwise stay black until the
|
||||
// next recovery keyframe.)
|
||||
layer.flush()
|
||||
format = AnnexB.formatDescription(fromIDR: au.data)
|
||||
}
|
||||
guard let f = format,
|
||||
let sample = AnnexB.sampleBuffer(au: au, format: f),
|
||||
token.isLive // don't enqueue a stale frame after a restart
|
||||
else { continue }
|
||||
layer.enqueue(sample)
|
||||
} catch {
|
||||
if token.isLive {
|
||||
onSessionEnd?()
|
||||
}
|
||||
break // session closed
|
||||
}
|
||||
}
|
||||
}
|
||||
thread.name = "punktfunk-pump"
|
||||
thread.qualityOfService = .userInteractive
|
||||
thread.start()
|
||||
}
|
||||
|
||||
/// Stop pumping (≤ one poll timeout). Does not close the connection.
|
||||
func stop() {
|
||||
token.cancel()
|
||||
}
|
||||
|
||||
deinit { token.cancel() }
|
||||
}
|
||||
@@ -99,25 +99,8 @@ public struct StreamView: NSViewRepresentable {
|
||||
}
|
||||
|
||||
public final class StreamLayerView: NSView {
|
||||
/// Cancellation handle owned by exactly one pump thread — a restart hands the old pump
|
||||
/// its own token, so it can never be revived by a newer start().
|
||||
private final class PumpToken: @unchecked Sendable {
|
||||
private let lock = NSLock()
|
||||
private var live = true
|
||||
var isLive: Bool {
|
||||
lock.lock()
|
||||
defer { lock.unlock() }
|
||||
return live
|
||||
}
|
||||
func cancel() {
|
||||
lock.lock()
|
||||
live = false
|
||||
lock.unlock()
|
||||
}
|
||||
}
|
||||
|
||||
private let displayLayer = AVSampleBufferDisplayLayer()
|
||||
private var token: PumpToken?
|
||||
private var pump: StreamPump?
|
||||
public private(set) var connection: PunktfunkConnection?
|
||||
private let cursorCapture = CursorCapture()
|
||||
private var inputCapture: InputCapture?
|
||||
@@ -261,7 +244,7 @@ public final class StreamLayerView: NSView {
|
||||
// A click is explicit intent AND may arrive mid-activation (acceptsFirstMouse:
|
||||
// NSApp.isActive / isKeyWindow are still false for the click coming in from
|
||||
// another app) — only the auto-engage paths require already-held key status.
|
||||
guard captureEnabled, !captured, token != nil, window != nil,
|
||||
guard captureEnabled, !captured, pump != nil, window != nil,
|
||||
fromClick || (NSApp.isActive && window?.isKeyWindow == true)
|
||||
else { return }
|
||||
cursorCapture.capture(in: self)
|
||||
@@ -297,11 +280,7 @@ public final class StreamLayerView: NSView {
|
||||
onSessionEnd: (@Sendable () -> Void)? = nil
|
||||
) {
|
||||
stop()
|
||||
let token = PumpToken()
|
||||
self.token = token
|
||||
self.connection = connection
|
||||
let layer = displayLayer
|
||||
layer.flush() // drop any frames a previous connection left queued
|
||||
|
||||
// The view owns the session's input capture: handlers attach now, but nothing is
|
||||
// forwarded until capture engages (captureEnabled + auto-engage or a click).
|
||||
@@ -324,40 +303,11 @@ public final class StreamLayerView: NSView {
|
||||
capture.start()
|
||||
inputCapture = capture
|
||||
|
||||
let thread = Thread {
|
||||
var format: CMVideoFormatDescription?
|
||||
while token.isLive {
|
||||
do {
|
||||
guard let au = try connection.nextAU(timeoutMs: 100) else { continue }
|
||||
onFrame?(au)
|
||||
if let f = AnnexB.formatDescription(fromIDR: au.data) {
|
||||
format = f // refreshed on every IDR (mode changes included)
|
||||
}
|
||||
if layer.status == .failed {
|
||||
// Decode wedged: flush and re-gate on the next in-band parameter
|
||||
// sets — resuming with a delta frame can't recover. (A
|
||||
// request-IDR channel on punktfunk/1 is a host-side TODO; with the
|
||||
// host's infinite GOP this may otherwise stay black until the
|
||||
// next recovery keyframe.)
|
||||
layer.flush()
|
||||
format = AnnexB.formatDescription(fromIDR: au.data)
|
||||
}
|
||||
guard let f = format,
|
||||
let sample = AnnexB.sampleBuffer(au: au, format: f),
|
||||
token.isLive // don't enqueue a stale frame after a restart
|
||||
else { continue }
|
||||
layer.enqueue(sample)
|
||||
} catch {
|
||||
if token.isLive {
|
||||
onSessionEnd?()
|
||||
}
|
||||
break // session closed
|
||||
}
|
||||
}
|
||||
}
|
||||
thread.name = "punktfunk-pump"
|
||||
thread.qualityOfService = .userInteractive
|
||||
thread.start()
|
||||
let pump = StreamPump()
|
||||
pump.start(
|
||||
connection: connection, layer: displayLayer,
|
||||
onFrame: onFrame, onSessionEnd: onSessionEnd)
|
||||
self.pump = pump
|
||||
requestAutoCapture() // entering a session is the deliberate "capture me" moment
|
||||
}
|
||||
|
||||
@@ -367,15 +317,15 @@ public final class StreamLayerView: NSView {
|
||||
releaseCapture()
|
||||
inputCapture?.stop()
|
||||
inputCapture = nil
|
||||
token?.cancel()
|
||||
token = nil
|
||||
pump?.stop()
|
||||
pump = nil
|
||||
connection = nil
|
||||
}
|
||||
|
||||
deinit {
|
||||
appObservers.forEach(NotificationCenter.default.removeObserver(_:))
|
||||
windowObservers.forEach(NotificationCenter.default.removeObserver(_:))
|
||||
token?.cancel()
|
||||
pump?.stop()
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
@@ -0,0 +1,272 @@
|
||||
// iOS/iPadOS presenter: the same AVSampleBufferDisplayLayer + StreamPump as macOS,
|
||||
// hosted in a UIViewController so the scene can pointer-lock (the iPadOS equivalent of
|
||||
// the Mac's cursor capture — with a hardware mouse/trackpad the system cursor is hidden
|
||||
// and GCMouse's raw deltas drive the host cursor alone; the system only honors the lock
|
||||
// fullscreen-and-frontmost, so in Stage Manager it degrades to Mac-style "both cursors
|
||||
// visible" forwarding).
|
||||
//
|
||||
// Touch is the primary input and is always forwarded (touching the video IS explicit
|
||||
// intent): every finger maps to a wire touch id, coordinates are mapped through the
|
||||
// aspect-fit letterbox into host-mode pixels, so surface == host mode and the host's
|
||||
// rescale is the identity. Hardware keyboard/mouse forwarding shares InputCapture with
|
||||
// macOS — auto-engaged when streaming starts, ⌘⎋ toggles (detected from the HID stream;
|
||||
// there is no NSEvent monitor here).
|
||||
//
|
||||
// The public type is named StreamView like its macOS twin (each is platform-gated), so
|
||||
// the SwiftUI app layer is identical on both platforms.
|
||||
|
||||
#if os(iOS)
|
||||
import AVFoundation
|
||||
import GameController
|
||||
import PunktfunkCore
|
||||
import SwiftUI
|
||||
import UIKit
|
||||
|
||||
public struct StreamView: UIViewControllerRepresentable {
|
||||
private let connection: PunktfunkConnection
|
||||
private let captureEnabled: Bool
|
||||
private let onCaptureChange: ((Bool) -> Void)?
|
||||
private let onFrame: (@Sendable (AccessUnit) -> Void)?
|
||||
private let onSessionEnd: (@Sendable () -> Void)?
|
||||
|
||||
public init(
|
||||
connection: PunktfunkConnection,
|
||||
captureEnabled: Bool = true,
|
||||
onCaptureChange: ((Bool) -> Void)? = nil,
|
||||
onFrame: (@Sendable (AccessUnit) -> Void)? = nil,
|
||||
onSessionEnd: (@Sendable () -> Void)? = nil
|
||||
) {
|
||||
self.connection = connection
|
||||
self.captureEnabled = captureEnabled
|
||||
self.onCaptureChange = onCaptureChange
|
||||
self.onFrame = onFrame
|
||||
self.onSessionEnd = onSessionEnd
|
||||
}
|
||||
|
||||
public func makeUIViewController(context: Context) -> StreamViewController {
|
||||
let controller = StreamViewController()
|
||||
controller.onCaptureChange = onCaptureChange
|
||||
controller.captureEnabled = captureEnabled
|
||||
controller.start(connection: connection, onFrame: onFrame, onSessionEnd: onSessionEnd)
|
||||
return controller
|
||||
}
|
||||
|
||||
public func updateUIViewController(_ controller: StreamViewController, context: Context) {
|
||||
controller.onCaptureChange = onCaptureChange
|
||||
controller.captureEnabled = captureEnabled
|
||||
if controller.connection !== connection {
|
||||
controller.start(connection: connection, onFrame: onFrame, onSessionEnd: onSessionEnd)
|
||||
}
|
||||
}
|
||||
|
||||
public static func dismantleUIViewController(
|
||||
_ controller: StreamViewController, coordinator: ()
|
||||
) {
|
||||
controller.stop()
|
||||
}
|
||||
}
|
||||
|
||||
public final class StreamViewController: UIViewController {
|
||||
public private(set) var connection: PunktfunkConnection?
|
||||
private var pump: StreamPump?
|
||||
private var inputCapture: InputCapture?
|
||||
private var captured = false
|
||||
private var observers: [NSObjectProtocol] = []
|
||||
|
||||
var onCaptureChange: ((Bool) -> Void)?
|
||||
|
||||
var captureEnabled = true {
|
||||
didSet {
|
||||
guard captureEnabled != oldValue else { return }
|
||||
setCaptured(captureEnabled)
|
||||
}
|
||||
}
|
||||
|
||||
private var streamView: StreamLayerUIView {
|
||||
// swiftlint:disable:next force_cast
|
||||
view as! StreamLayerUIView
|
||||
}
|
||||
|
||||
public override func loadView() {
|
||||
view = StreamLayerUIView()
|
||||
}
|
||||
|
||||
public override var prefersPointerLocked: Bool { captured }
|
||||
public override var prefersHomeIndicatorAutoHidden: Bool { true }
|
||||
|
||||
func start(
|
||||
connection: PunktfunkConnection,
|
||||
onFrame: (@Sendable (AccessUnit) -> Void)?,
|
||||
onSessionEnd: (@Sendable () -> Void)?
|
||||
) {
|
||||
stop()
|
||||
self.connection = connection
|
||||
loadViewIfNeeded()
|
||||
// Read the LIVE mode per touch batch — an accepted requestMode() mid-stream
|
||||
// changes the letterbox, and touches must follow it.
|
||||
streamView.currentHostMode = { [weak connection] in
|
||||
guard let connection else { return .zero }
|
||||
let mode = connection.currentMode()
|
||||
return CGSize(width: Double(mode.width), height: Double(mode.height))
|
||||
}
|
||||
streamView.onTouchEvent = { [weak connection] event in
|
||||
connection?.send(event)
|
||||
}
|
||||
|
||||
let capture = InputCapture(connection: connection)
|
||||
capture.onToggleCapture = { [weak self] in
|
||||
guard let self else { return }
|
||||
self.setCaptured(!self.captured)
|
||||
}
|
||||
capture.onPreempted = { [weak self] in
|
||||
self?.setCaptured(false)
|
||||
}
|
||||
capture.start()
|
||||
inputCapture = capture
|
||||
|
||||
let pump = StreamPump()
|
||||
pump.start(
|
||||
connection: connection, layer: streamView.displayLayer,
|
||||
onFrame: onFrame, onSessionEnd: onSessionEnd)
|
||||
self.pump = pump
|
||||
|
||||
// GC only delivers while active; everything held is flushed by InputCapture's
|
||||
// own resign observer — here we just mirror the capture state for the HUD and
|
||||
// the pointer lock.
|
||||
observers.append(NotificationCenter.default.addObserver(
|
||||
forName: UIApplication.willResignActiveNotification, object: nil, queue: .main
|
||||
) { [weak self] _ in
|
||||
self?.setCaptured(false)
|
||||
})
|
||||
|
||||
if captureEnabled {
|
||||
setCaptured(true) // entering a session is the deliberate "capture me" moment
|
||||
}
|
||||
}
|
||||
|
||||
func stop() {
|
||||
setCaptured(false)
|
||||
observers.forEach(NotificationCenter.default.removeObserver(_:))
|
||||
observers.removeAll()
|
||||
inputCapture?.stop()
|
||||
inputCapture = nil
|
||||
pump?.stop()
|
||||
pump = nil
|
||||
connection = nil
|
||||
streamView.onTouchEvent = nil
|
||||
streamView.currentHostMode = nil
|
||||
}
|
||||
|
||||
private func setCaptured(_ on: Bool) {
|
||||
if on {
|
||||
guard captureEnabled, !captured, pump != nil else { return }
|
||||
inputCapture?.setForwarding(true)
|
||||
captured = true
|
||||
} else {
|
||||
guard captured else { return }
|
||||
inputCapture?.setForwarding(false)
|
||||
captured = false
|
||||
}
|
||||
setNeedsUpdateOfPrefersPointerLocked()
|
||||
let onCaptureChange = onCaptureChange
|
||||
let captured = captured
|
||||
DispatchQueue.main.async { onCaptureChange?(captured) }
|
||||
}
|
||||
|
||||
deinit {
|
||||
observers.forEach(NotificationCenter.default.removeObserver(_:))
|
||||
pump?.stop()
|
||||
}
|
||||
}
|
||||
|
||||
/// The layer-backed video surface + touch source. Touches are mapped through the
|
||||
/// aspect-fit letterbox into host-mode pixels (surface == host mode, so the host-side
|
||||
/// rescale is the identity); touches outside the video area are clamped onto its edge.
|
||||
final class StreamLayerUIView: UIView {
|
||||
override class var layerClass: AnyClass { AVSampleBufferDisplayLayer.self }
|
||||
var displayLayer: AVSampleBufferDisplayLayer {
|
||||
// swiftlint:disable:next force_cast
|
||||
layer as! AVSampleBufferDisplayLayer
|
||||
}
|
||||
|
||||
/// Reads the LIVE negotiated mode in pixels (the touch coordinate space).
|
||||
var currentHostMode: (() -> CGSize)?
|
||||
var onTouchEvent: ((PunktfunkInputEvent) -> Void)?
|
||||
|
||||
/// Wire touch ids per active UITouch; ids are reused after the touch ends.
|
||||
private var touchIDs: [ObjectIdentifier: UInt32] = [:]
|
||||
|
||||
override init(frame: CGRect) {
|
||||
super.init(frame: frame)
|
||||
displayLayer.videoGravity = .resizeAspect
|
||||
isMultipleTouchEnabled = true
|
||||
backgroundColor = .black
|
||||
}
|
||||
|
||||
@available(*, unavailable)
|
||||
required init?(coder: NSCoder) { fatalError("not used") }
|
||||
|
||||
override func touchesBegan(_ touches: Set<UITouch>, with event: UIEvent?) {
|
||||
forward(touches, kind: .down)
|
||||
}
|
||||
override func touchesMoved(_ touches: Set<UITouch>, with event: UIEvent?) {
|
||||
forward(touches, kind: .move)
|
||||
}
|
||||
override func touchesEnded(_ touches: Set<UITouch>, with event: UIEvent?) {
|
||||
forward(touches, kind: .up)
|
||||
}
|
||||
override func touchesCancelled(_ touches: Set<UITouch>, with event: UIEvent?) {
|
||||
forward(touches, kind: .up)
|
||||
}
|
||||
|
||||
private enum TouchKind { case down, move, up }
|
||||
|
||||
private func forward(_ touches: Set<UITouch>, kind: TouchKind) {
|
||||
guard let hostMode = currentHostMode?(),
|
||||
hostMode.width > 0, hostMode.height > 0, onTouchEvent != nil
|
||||
else { return }
|
||||
let video = AVMakeRect(aspectRatio: hostMode, insideRect: bounds)
|
||||
guard video.width > 0, video.height > 0 else { return }
|
||||
for touch in touches {
|
||||
let key = ObjectIdentifier(touch)
|
||||
let id: UInt32
|
||||
switch kind {
|
||||
case .down:
|
||||
id = nextFreeID()
|
||||
touchIDs[key] = id
|
||||
case .move, .up:
|
||||
guard let known = touchIDs[key] else { continue }
|
||||
id = known
|
||||
}
|
||||
if kind == .up {
|
||||
touchIDs.removeValue(forKey: key)
|
||||
onTouchEvent?(.touchUp(id: id))
|
||||
continue
|
||||
}
|
||||
let p = touch.location(in: self)
|
||||
let x = Int32(((p.x - video.minX) / video.width * hostMode.width)
|
||||
.rounded().clamped(to: 0...(hostMode.width - 1)))
|
||||
let y = Int32(((p.y - video.minY) / video.height * hostMode.height)
|
||||
.rounded().clamped(to: 0...(hostMode.height - 1)))
|
||||
let w = UInt32(hostMode.width)
|
||||
let h = UInt32(hostMode.height)
|
||||
onTouchEvent?(
|
||||
kind == .down
|
||||
? .touchDown(id: id, x: x, y: y, surfaceWidth: w, surfaceHeight: h)
|
||||
: .touchMove(id: id, x: x, y: y, surfaceWidth: w, surfaceHeight: h))
|
||||
}
|
||||
}
|
||||
|
||||
private func nextFreeID() -> UInt32 {
|
||||
var id: UInt32 = 0
|
||||
while touchIDs.values.contains(id) { id += 1 }
|
||||
return id
|
||||
}
|
||||
}
|
||||
|
||||
extension CGFloat {
|
||||
fileprivate func clamped(to range: ClosedRange<CGFloat>) -> CGFloat {
|
||||
Swift.min(Swift.max(self, range.lowerBound), range.upperBound)
|
||||
}
|
||||
}
|
||||
#endif
|
||||
Reference in New Issue
Block a user