punktfunk/clients/apple/Sources/PunktfunkKit/StreamViewIOS.swift

// iOS/iPadOS presenter: the same AVSampleBufferDisplayLayer + StreamPump as macOS,
// hosted in a UIViewController so the scene can pointer-lock (the iPadOS equivalent of
// the Mac's cursor capture — with a hardware mouse/trackpad the system cursor is hidden
// and GCMouse's raw deltas drive the host cursor alone; the system only honors the lock
// fullscreen-and-frontmost, so in Stage Manager it degrades to Mac-style "both cursors
// visible" forwarding).
//
// Touch is the primary input and is always forwarded (touching the video IS explicit
// intent): every finger maps to a wire touch id, coordinates are mapped through the
// aspect-fit letterbox into host-mode pixels, so surface == host mode and the host's
// rescale is the identity. Hardware keyboard/mouse forwarding shares InputCapture with
// macOS — auto-engaged when streaming starts, ⌘⎋ toggles (detected from the HID stream;
// there is no NSEvent monitor here).
//
// The public type is named StreamView like its macOS twin (each is platform-gated), so
// the SwiftUI app layer is identical on both platforms.

#if os(iOS) || os(tvOS)
import AVFoundation
import GameController
import PunktfunkCore
import SwiftUI
import UIKit
import os

/// Same diagnostic switch as InputCapture (PUNKTFUNK_INPUT_DEBUG=1): on iOS we log the
/// resolved pointer-lock state each time capture engages, so the user can see whether the
/// scene actually locked (GCMouse only delivers deltas while it did) or whether we're on
/// the touch fallback.
private let iosInputLog = Logger(subsystem: "io.unom.punktfunk", category: "input")
private let iosInputDebug = ProcessInfo.processInfo.environment["PUNKTFUNK_INPUT_DEBUG"] == "1"

public struct StreamView: UIViewControllerRepresentable {
    private let connection: PunktfunkConnection
    private let captureEnabled: Bool
    private let onCaptureChange: ((Bool) -> Void)?
    private let onFrame: (@Sendable (AccessUnit) -> Void)?
    private let onSessionEnd: (@Sendable () -> Void)?

    public init(
        connection: PunktfunkConnection,
        captureEnabled: Bool = true,
        onCaptureChange: ((Bool) -> Void)? = nil,
        onFrame: (@Sendable (AccessUnit) -> Void)? = nil,
        onSessionEnd: (@Sendable () -> Void)? = nil
    ) {
        self.connection = connection
        self.captureEnabled = captureEnabled
        self.onCaptureChange = onCaptureChange
        self.onFrame = onFrame
        self.onSessionEnd = onSessionEnd
    }

    public func makeUIViewController(context: Context) -> StreamViewController {
        let controller = StreamViewController()
        controller.onCaptureChange = onCaptureChange
        controller.captureEnabled = captureEnabled
        controller.start(connection: connection, onFrame: onFrame, onSessionEnd: onSessionEnd)
        return controller
    }

    public func updateUIViewController(_ controller: StreamViewController, context: Context) {
        controller.onCaptureChange = onCaptureChange
        controller.captureEnabled = captureEnabled
        if controller.connection !== connection {
            controller.start(connection: connection, onFrame: onFrame, onSessionEnd: onSessionEnd)
        }
    }

    public static func dismantleUIViewController(
        _ controller: StreamViewController, coordinator: ()
    ) {
        controller.stop()
    }
}

public final class StreamViewController: UIViewController {
    public private(set) var connection: PunktfunkConnection?
    private var pump: StreamPump?
    private var observers: [NSObjectProtocol] = []
    #if os(iOS)
    private var inputCapture: InputCapture?
    fileprivate var captured = false
    private var pointerInteraction: UIPointerInteraction?
    #endif

    /// Reads whether the scene's pointer is actually locked right now; nil = state
    /// unavailable (no scene yet, or pre-availability). Only while this is true does GCMouse
    /// deliver relative deltas — otherwise the touch path carries input.
    private func pointerLockEngaged() -> Bool? {
        #if os(iOS)
        return view.window?.windowScene?.pointerLockState?.isLocked
        #else
        return nil
        #endif
    }

    var onCaptureChange: ((Bool) -> Void)?

    var captureEnabled = true {
        didSet {
            guard captureEnabled != oldValue else { return }
            #if os(iOS)
            setCaptured(captureEnabled)
            #endif
        }
    }

    private var streamView: StreamLayerUIView {
        // swiftlint:disable:next force_cast
        view as! StreamLayerUIView
    }

    public override func loadView() {
        view = StreamLayerUIView()
        #if os(iOS)
        // Hide the iPadOS cursor while it hovers the video: the host renders its own
        // cursor from our deltas, so the local one only diverges from it. This hides the
        // pointer; true pointer LOCK (below) is what makes GCMouse deliver relative deltas
        // — and the system only grants it on a full-screen, frontmost iPad scene.
        let interaction = UIPointerInteraction(delegate: self)
        view.addInteraction(interaction)
        pointerInteraction = interaction
        #endif
    }

    #if os(iOS)
    // Pointer lock is only meaningful on iPad (iPhone has no hardware-pointer lock) and
    // only when capture is engaged. The system additionally requires full-screen + frontmost
    // and may drop it (Slide Over/Stage Manager/backgrounding) — verified in setCaptured().
    public override var prefersPointerLocked: Bool {
        captured && UIDevice.current.userInterfaceIdiom == .pad
    }
    public override var prefersHomeIndicatorAutoHidden: Bool { true }

    // If SwiftUI's UIHostingController reparents us, a plain container parent that forwards
    // its pointer-lock decision to its children will then reach this VC. (UIHostingController
    // itself does not consult children, which is why GCMouse deltas can never arrive there —
    // the touch path, always forwarded, is the unconditional fallback.)
    public override var childViewControllerForPointerLock: UIViewController? { self }
    #endif

    func start(
        connection: PunktfunkConnection,
        onFrame: (@Sendable (AccessUnit) -> Void)?,
        onSessionEnd: (@Sendable () -> Void)?
    ) {
        stop()
        self.connection = connection
        loadViewIfNeeded()
        #if os(iOS)
        // Read the LIVE mode per touch batch — an accepted requestMode() mid-stream
        // changes the letterbox, and touches must follow it.
        streamView.currentHostMode = { [weak connection] in
            guard let connection else { return .zero }
            let mode = connection.currentMode()
            return CGSize(width: Double(mode.width), height: Double(mode.height))
        }
        streamView.onTouchEvent = { [weak connection] event in
            connection?.send(event)
        }

        let capture = InputCapture(connection: connection)
        capture.onToggleCapture = { [weak self] in
            guard let self else { return }
            self.setCaptured(!self.captured)
        }
        capture.onPreempted = { [weak self] in
            self?.setCaptured(false)
        }
        capture.start()
        inputCapture = capture
        #endif

        let pump = StreamPump()
        pump.start(
            connection: connection, layer: streamView.displayLayer,
            onFrame: onFrame, onSessionEnd: onSessionEnd)
        self.pump = pump

        #if os(iOS)
        // GC only delivers while active; everything held is flushed by InputCapture's
        // own resign observer — here we just mirror the capture state for the HUD and
        // the pointer lock.
        observers.append(NotificationCenter.default.addObserver(
            forName: UIApplication.willResignActiveNotification, object: nil, queue: .main
        ) { [weak self] _ in
            self?.setCaptured(false)
        })
        // The system can drop the lock without us asking (Slide Over, Stage Manager, leaving
        // foregroundActive). Surface it so the user sees, in PUNKTFUNK_INPUT_DEBUG, when
        // GCMouse delivery has silently stopped and we've fallen back to touch.
        observers.append(NotificationCenter.default.addObserver(
            forName: UIPointerLockState.didChangeNotification, object: nil, queue: .main
        ) { [weak self] _ in
            guard let self, iosInputDebug else { return }
            let locked = self.pointerLockEngaged().map(String.init(describing:)) ?? "unavailable"
            iosInputLog.debug("pointer lock changed: isLocked=\(locked, privacy: .public)")
        })

        if captureEnabled {
            setCaptured(true) // entering a session is the deliberate "capture me" moment
        }
        #endif
    }

    func stop() {
        observers.forEach(NotificationCenter.default.removeObserver(_:))
        observers.removeAll()
        #if os(iOS)
        setCaptured(false)
        inputCapture?.stop()
        inputCapture = nil
        streamView.onTouchEvent = nil
        streamView.currentHostMode = nil
        #endif
        pump?.stop()
        pump = nil
        connection = nil
    }

    #if os(iOS)
    private func setCaptured(_ on: Bool) {
        if on {
            guard captureEnabled, !captured, pump != nil else { return }
            inputCapture?.setForwarding(true)
            captured = true
        } else {
            guard captured else { return }
            inputCapture?.setForwarding(false)
            captured = false
        }
        setNeedsUpdateOfPrefersPointerLocked()
        pointerInteraction?.invalidate() // re-resolve the hidden/visible pointer style
        let onCaptureChange = onCaptureChange
        let captured = captured
        DispatchQueue.main.async { [weak self] in
            onCaptureChange?(captured)
            // The lock request is async — read the resolved state next turn. If it didn't
            // engage, GCMouse won't deliver and the always-on touch path carries input.
            if iosInputDebug, let self {
                let locked = self.pointerLockEngaged().map(String.init(describing:)) ?? "unavailable"
                iosInputLog.debug(
                    "setCaptured(\(captured, privacy: .public)) → pointer lock isLocked=\(locked, privacy: .public)")
            }
        }
    }
    #endif

    deinit {
        observers.forEach(NotificationCenter.default.removeObserver(_:))
        pump?.stop()
    }
}

#if os(iOS)
extension StreamViewController: UIPointerInteractionDelegate {
    public func pointerInteraction(
        _ interaction: UIPointerInteraction, styleFor region: UIPointerRegion
    ) -> UIPointerStyle? {
        captured ? .hidden() : nil
    }
}
#endif

/// The layer-backed video surface + touch source. Touches are mapped through the
/// aspect-fit letterbox into host-mode pixels (surface == host mode, so the host-side
/// rescale is the identity); touches outside the video area are clamped onto its edge.
final class StreamLayerUIView: UIView {
    override class var layerClass: AnyClass { AVSampleBufferDisplayLayer.self }
    var displayLayer: AVSampleBufferDisplayLayer {
        // swiftlint:disable:next force_cast
        layer as! AVSampleBufferDisplayLayer
    }

    #if os(iOS)
    /// Reads the LIVE negotiated mode in pixels (the touch coordinate space).
    var currentHostMode: (() -> CGSize)?
    var onTouchEvent: ((PunktfunkInputEvent) -> Void)?

    /// Wire touch ids per active UITouch; ids are reused after the touch ends.
    private var touchIDs: [ObjectIdentifier: UInt32] = [:]
    #endif

    override init(frame: CGRect) {
        super.init(frame: frame)
        displayLayer.videoGravity = .resizeAspect
        #if os(iOS)
        isMultipleTouchEnabled = true
        #endif
        backgroundColor = .black
    }

    @available(*, unavailable)
    required init?(coder: NSCoder) { fatalError("not used") }

    #if os(iOS)
    override func touchesBegan(_ touches: Set<UITouch>, with event: UIEvent?) {
        forward(touches, kind: .down)
    }
    override func touchesMoved(_ touches: Set<UITouch>, with event: UIEvent?) {
        forward(touches, kind: .move)
    }
    override func touchesEnded(_ touches: Set<UITouch>, with event: UIEvent?) {
        forward(touches, kind: .up)
    }
    override func touchesCancelled(_ touches: Set<UITouch>, with event: UIEvent?) {
        forward(touches, kind: .up)
    }

    private enum TouchKind { case down, move, up }

    private func forward(_ touches: Set<UITouch>, kind: TouchKind) {
        guard let hostMode = currentHostMode?(),
              hostMode.width > 0, hostMode.height > 0, onTouchEvent != nil
        else { return }
        let video = AVMakeRect(aspectRatio: hostMode, insideRect: bounds)
        guard video.width > 0, video.height > 0 else { return }
        for touch in touches {
            let key = ObjectIdentifier(touch)
            let id: UInt32
            switch kind {
            case .down:
                id = nextFreeID()
                touchIDs[key] = id
            case .move, .up:
                guard let known = touchIDs[key] else { continue }
                id = known
            }
            if kind == .up {
                touchIDs.removeValue(forKey: key)
                onTouchEvent?(.touchUp(id: id))
                continue
            }
            let p = touch.location(in: self)
            let x = Int32(((p.x - video.minX) / video.width * hostMode.width)
                .rounded().clamped(to: 0...(hostMode.width - 1)))
            let y = Int32(((p.y - video.minY) / video.height * hostMode.height)
                .rounded().clamped(to: 0...(hostMode.height - 1)))
            let w = UInt32(hostMode.width)
            let h = UInt32(hostMode.height)
            onTouchEvent?(
                kind == .down
                    ? .touchDown(id: id, x: x, y: y, surfaceWidth: w, surfaceHeight: h)
                    : .touchMove(id: id, x: x, y: y, surfaceWidth: w, surfaceHeight: h))
        }
    }

    private func nextFreeID() -> UInt32 {
        var id: UInt32 = 0
        while touchIDs.values.contains(id) { id += 1 }
        return id
    }
    #endif
}

#if os(iOS)
extension CGFloat {
    fileprivate func clamped(to range: ClosedRange<CGFloat>) -> CGFloat {
        Swift.min(Swift.max(self, range.lowerBound), range.upperBound)
    }
}
#endif
#endif