From e1af4d57c6e2cd58cbe0d6f74b5741b648f80d45 Mon Sep 17 00:00:00 2001 From: enricobuehler Date: Thu, 11 Jun 2026 11:18:18 +0200 Subject: [PATCH] =?UTF-8?q?feat(apple):=20iOS/iPadOS=20client=20=E2=80=94?= =?UTF-8?q?=20touch,=20pointer=20lock,=20shared=20SwiftUI=20shell?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The whole client now runs on iPadOS/iOS from the same sources, first-lit live in the iPad simulator against the real host at 1280x720@60 (60 fps on the HUD, capture state machine active, mic permission flow shown). - PunktfunkCore.xcframework grows iOS device + universal-simulator slices (BUILD_IOS=1; rustup targets aarch64-apple-ios{,-sim} + x86_64-apple-ios). - The decode pump is extracted into a shared StreamPump (identical IDR re-gate logic on both platforms); the iOS StreamView (StreamViewIOS.swift) has the same name/signature as the macOS one, so ContentView & co. are byte-identical across platforms — hosted in a UIViewController for prefersPointerLocked (the iPadOS cursor capture; see README note 9 for the UIHostingController forwarding caveat). - Touch is always forwarded: per-finger wire ids, coordinates mapped through the aspect-fit letterbox into LIVE host-mode pixels (surface == host mode, identity rescale host-side; follows mid-stream requestMode switches). - InputCapture is cross-platform: GC works the same on iPadOS, ⌘⎋ is detected from the HID stream there; stale-⌘ tracking after focus loss fixed on both platforms (releaseAll now drops the modifier/latch state — a ⌘ released in another app otherwise hijacked Esc forever). - SessionAudio: AVAudioSession on iOS (.playAndRecord + .defaultToSpeaker — without it iPhones route host audio to the EARPIECE; deactivated with notifyOthersOnDeactivation on stop so interrupted background audio resumes); HAL device pinning + the Settings pickers stay macOS-only. - New Punktfunk-iOS app target (shared synchronized sources, generated Info.plist with mic + local-network usage descriptions — QUIC to a LAN host trips local network privacy on real devices — scene manifest + indirect input events for Stage Manager / external displays), shared scheme, macOS min-window frames gated off iOS. For the iPad-on-an-external-screen idea: with multiple scenes + indirect input enabled, Stage Manager iPads can drag the punktfunk window onto the external display and drive the PC with keyboard/mouse/touch. Known gaps (README note 9): the pointer-lock preference isn't consulted through UIHostingController (relative mouse works, the local cursor just stays visible) and AVAudioSession interruptions don't auto-restart audio. Co-Authored-By: Claude Fable 5 --- .../apple/Punktfunk.xcodeproj/project.pbxproj | 131 +++++++++ .../xcschemes/Punktfunk-iOS.xcscheme | 77 +++++ clients/apple/README.md | 29 +- .../PunktfunkClient/AddHostSheet.swift | 2 + .../Sources/PunktfunkClient/ContentView.swift | 41 +++ .../Sources/PunktfunkClient/PairSheet.swift | 6 + .../PunktfunkClient/PunktfunkClientApp.swift | 8 + .../PunktfunkClient/SettingsView.swift | 20 +- .../Sources/PunktfunkKit/InputCapture.swift | 46 ++- .../Sources/PunktfunkKit/SessionAudio.swift | 42 ++- .../Sources/PunktfunkKit/StreamPump.swift | 84 ++++++ .../Sources/PunktfunkKit/StreamView.swift | 70 +---- .../Sources/PunktfunkKit/StreamViewIOS.swift | 272 ++++++++++++++++++ scripts/build-xcframework.sh | 11 +- 14 files changed, 766 insertions(+), 73 deletions(-) create mode 100644 clients/apple/Punktfunk.xcodeproj/xcshareddata/xcschemes/Punktfunk-iOS.xcscheme create mode 100644 clients/apple/Sources/PunktfunkKit/StreamPump.swift create mode 100644 clients/apple/Sources/PunktfunkKit/StreamViewIOS.swift diff --git a/clients/apple/Punktfunk.xcodeproj/project.pbxproj b/clients/apple/Punktfunk.xcodeproj/project.pbxproj index d7addd1..ef5f371 100644 --- a/clients/apple/Punktfunk.xcodeproj/project.pbxproj +++ b/clients/apple/Punktfunk.xcodeproj/project.pbxproj @@ -8,10 +8,12 @@ /* Begin PBXBuildFile section */ AA0000000000000000000005 /* PunktfunkKit in Frameworks */ = {isa = PBXBuildFile; productRef = AA0000000000000000000006 /* PunktfunkKit */; }; + BB0000000000000000000005 /* PunktfunkKit in Frameworks */ = {isa = PBXBuildFile; productRef = BB0000000000000000000006 /* PunktfunkKit */; }; /* End PBXBuildFile section */ /* Begin PBXFileReference section */ AA0000000000000000000001 /* Punktfunk.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = Punktfunk.app; sourceTree = BUILT_PRODUCTS_DIR; }; + BB0000000000000000000001 /* Punktfunk-iOS.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = "Punktfunk-iOS.app"; sourceTree = BUILT_PRODUCTS_DIR; }; /* End PBXFileReference section */ /* Begin PBXFileSystemSynchronizedRootGroup section */ @@ -36,6 +38,14 @@ ); runOnlyForDeploymentPostprocessing = 0; }; + BB0000000000000000000004 /* Frameworks */ = { + isa = PBXFrameworksBuildPhase; + buildActionMask = 2147483647; + files = ( + BB0000000000000000000005 /* PunktfunkKit in Frameworks */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; /* End PBXFrameworksBuildPhase section */ /* Begin PBXGroup section */ @@ -52,6 +62,7 @@ isa = PBXGroup; children = ( AA0000000000000000000001 /* Punktfunk.app */, + BB0000000000000000000001 /* Punktfunk-iOS.app */, ); name = Products; sourceTree = ""; @@ -83,6 +94,30 @@ productReference = AA0000000000000000000001 /* Punktfunk.app */; productType = "com.apple.product-type.application"; }; + BB0000000000000000000009 /* Punktfunk-iOS */ = { + isa = PBXNativeTarget; + buildConfigurationList = BB000000000000000000000A /* Build configuration list for PBXNativeTarget "Punktfunk-iOS" */; + buildPhases = ( + BB000000000000000000000B /* Sources */, + BB0000000000000000000004 /* Frameworks */, + BB000000000000000000000C /* Resources */, + ); + buildRules = ( + ); + dependencies = ( + ); + fileSystemSynchronizedGroups = ( + AA0000000000000000000002 /* App */, + AA0000000000000000000003 /* Sources/PunktfunkClient */, + ); + name = "Punktfunk-iOS"; + packageProductDependencies = ( + BB0000000000000000000006 /* PunktfunkKit */, + ); + productName = "Punktfunk-iOS"; + productReference = BB0000000000000000000001 /* Punktfunk-iOS.app */; + productType = "com.apple.product-type.application"; + }; /* End PBXNativeTarget section */ /* Begin PBXProject section */ @@ -114,6 +149,7 @@ projectRoot = ""; targets = ( AA0000000000000000000009 /* Punktfunk */, + BB0000000000000000000009 /* Punktfunk-iOS */, ); }; /* End PBXProject section */ @@ -126,6 +162,13 @@ ); runOnlyForDeploymentPostprocessing = 0; }; + BB000000000000000000000C /* Resources */ = { + isa = PBXResourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; /* End PBXResourcesBuildPhase section */ /* Begin PBXSourcesBuildPhase section */ @@ -136,6 +179,13 @@ ); runOnlyForDeploymentPostprocessing = 0; }; + BB000000000000000000000B /* Sources */ = { + isa = PBXSourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; /* End PBXSourcesBuildPhase section */ /* Begin XCBuildConfiguration section */ @@ -257,6 +307,7 @@ INFOPLIST_KEY_CFBundleDisplayName = "Punktfunkempfänger"; INFOPLIST_KEY_LSApplicationCategoryType = "public.app-category.utilities"; INFOPLIST_KEY_NSHumanReadableCopyright = ""; + INFOPLIST_KEY_NSLocalNetworkUsageDescription = "Punktfunk connects directly to your punktfunk host on the local network to stream video, audio, and input."; INFOPLIST_KEY_NSMicrophoneUsageDescription = "Your microphone is streamed to the connected punktfunk host, where it appears as a virtual microphone."; INFOPLIST_KEY_NSPrincipalClass = NSApplication; LD_RUNPATH_SEARCH_PATHS = ( @@ -284,6 +335,7 @@ INFOPLIST_KEY_CFBundleDisplayName = "Punktfunkempfänger"; INFOPLIST_KEY_LSApplicationCategoryType = "public.app-category.utilities"; INFOPLIST_KEY_NSHumanReadableCopyright = ""; + INFOPLIST_KEY_NSLocalNetworkUsageDescription = "Punktfunk connects directly to your punktfunk host on the local network to stream video, audio, and input."; INFOPLIST_KEY_NSMicrophoneUsageDescription = "Your microphone is streamed to the connected punktfunk host, where it appears as a virtual microphone."; INFOPLIST_KEY_NSPrincipalClass = NSApplication; LD_RUNPATH_SEARCH_PATHS = ( @@ -298,9 +350,84 @@ }; name = Release; }; + BB0000000000000000000012 /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + ASSETCATALOG_COMPILER_APPICON_NAME = punktfunk_Logo; + CODE_SIGN_STYLE = Automatic; + CURRENT_PROJECT_VERSION = 1; + GENERATE_INFOPLIST_FILE = YES; + INFOPLIST_KEY_CFBundleDisplayName = "Punktfunkempfänger"; + INFOPLIST_KEY_LSApplicationCategoryType = "public.app-category.utilities"; + INFOPLIST_KEY_NSLocalNetworkUsageDescription = "Punktfunk connects directly to your punktfunk host on the local network to stream video, audio, and input."; + INFOPLIST_KEY_NSMicrophoneUsageDescription = "Your microphone is streamed to the connected punktfunk host, where it appears as a virtual microphone."; + INFOPLIST_KEY_UIApplicationSceneManifest_Generation = YES; + INFOPLIST_KEY_UIApplicationSupportsIndirectInputEvents = YES; + INFOPLIST_KEY_UILaunchScreen_Generation = YES; + INFOPLIST_KEY_UISupportedInterfaceOrientations_iPad = "UIInterfaceOrientationPortrait UIInterfaceOrientationPortraitUpsideDown UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight"; + INFOPLIST_KEY_UISupportedInterfaceOrientations_iPhone = "UIInterfaceOrientationPortrait UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight"; + IPHONEOS_DEPLOYMENT_TARGET = 17.0; + LD_RUNPATH_SEARCH_PATHS = ( + "$(inherited)", + "@executable_path/Frameworks", + ); + MARKETING_VERSION = 0.1; + PRODUCT_BUNDLE_IDENTIFIER = io.unom.punktfunk.ios; + PRODUCT_NAME = "$(TARGET_NAME)"; + SDKROOT = iphoneos; + SUPPORTED_PLATFORMS = "iphoneos iphonesimulator"; + SUPPORTS_MACCATALYST = NO; + SWIFT_EMIT_LOC_STRINGS = YES; + SWIFT_VERSION = 5.0; + TARGETED_DEVICE_FAMILY = "1,2"; + }; + name = Debug; + }; + BB0000000000000000000013 /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + ASSETCATALOG_COMPILER_APPICON_NAME = punktfunk_Logo; + CODE_SIGN_STYLE = Automatic; + CURRENT_PROJECT_VERSION = 1; + GENERATE_INFOPLIST_FILE = YES; + INFOPLIST_KEY_CFBundleDisplayName = "Punktfunkempfänger"; + INFOPLIST_KEY_LSApplicationCategoryType = "public.app-category.utilities"; + INFOPLIST_KEY_NSLocalNetworkUsageDescription = "Punktfunk connects directly to your punktfunk host on the local network to stream video, audio, and input."; + INFOPLIST_KEY_NSMicrophoneUsageDescription = "Your microphone is streamed to the connected punktfunk host, where it appears as a virtual microphone."; + INFOPLIST_KEY_UIApplicationSceneManifest_Generation = YES; + INFOPLIST_KEY_UIApplicationSupportsIndirectInputEvents = YES; + INFOPLIST_KEY_UILaunchScreen_Generation = YES; + INFOPLIST_KEY_UISupportedInterfaceOrientations_iPad = "UIInterfaceOrientationPortrait UIInterfaceOrientationPortraitUpsideDown UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight"; + INFOPLIST_KEY_UISupportedInterfaceOrientations_iPhone = "UIInterfaceOrientationPortrait UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight"; + IPHONEOS_DEPLOYMENT_TARGET = 17.0; + LD_RUNPATH_SEARCH_PATHS = ( + "$(inherited)", + "@executable_path/Frameworks", + ); + MARKETING_VERSION = 0.1; + PRODUCT_BUNDLE_IDENTIFIER = io.unom.punktfunk.ios; + PRODUCT_NAME = "$(TARGET_NAME)"; + SDKROOT = iphoneos; + SUPPORTED_PLATFORMS = "iphoneos iphonesimulator"; + SUPPORTS_MACCATALYST = NO; + SWIFT_EMIT_LOC_STRINGS = YES; + SWIFT_VERSION = 5.0; + TARGETED_DEVICE_FAMILY = "1,2"; + }; + name = Release; + }; /* End XCBuildConfiguration section */ /* Begin XCConfigurationList section */ + BB000000000000000000000A /* Build configuration list for PBXNativeTarget "Punktfunk-iOS" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + BB0000000000000000000012 /* Debug */, + BB0000000000000000000013 /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; AA000000000000000000000A /* Build configuration list for PBXNativeTarget "Punktfunk" */ = { isa = XCConfigurationList; buildConfigurations = ( @@ -333,6 +460,10 @@ isa = XCSwiftPackageProductDependency; productName = PunktfunkKit; }; + BB0000000000000000000006 /* PunktfunkKit */ = { + isa = XCSwiftPackageProductDependency; + productName = PunktfunkKit; + }; /* End XCSwiftPackageProductDependency section */ }; rootObject = AA000000000000000000000D /* Project object */; diff --git a/clients/apple/Punktfunk.xcodeproj/xcshareddata/xcschemes/Punktfunk-iOS.xcscheme b/clients/apple/Punktfunk.xcodeproj/xcshareddata/xcschemes/Punktfunk-iOS.xcscheme new file mode 100644 index 0000000..78c9f68 --- /dev/null +++ b/clients/apple/Punktfunk.xcodeproj/xcshareddata/xcschemes/Punktfunk-iOS.xcscheme @@ -0,0 +1,77 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/clients/apple/README.md b/clients/apple/README.md index ec43e7e..029b9f2 100644 --- a/clients/apple/README.md +++ b/clients/apple/README.md @@ -97,7 +97,10 @@ signing, bundle id `io.unom.punktfunk`. Notes: - **Tests from Xcode**: the package tests run with `swift test`; to get them on ⌘U, add `PunktfunkKitTests` once via Edit Scheme → Test → + (Xcode persists it into the shared scheme — a hand-written package-test reference doesn't resolve headlessly). -- `xcodebuild -project Punktfunk.xcodeproj -scheme Punktfunk build` works headlessly. +- `xcodebuild -project Punktfunk.xcodeproj -scheme Punktfunk build` works headlessly; + same for `-scheme Punktfunk-iOS -destination 'generic/platform=iOS Simulator'` (run it + in a simulator via `xcrun simctl install/launch` — `SIMCTL_CHILD_PUNKTFUNK_AUTOCONNECT=…` + passes the dev autoconnect env through). ## Notes for whoever picks this up next @@ -170,8 +173,28 @@ signing, bundle id `io.unom.punktfunk`. Notes: while the app has focus, and focus loss also auto-releases everything held. One live capture per process (the GC mouse/keyboard singletons have a single handler slot — ownership is tracked so a stale capture's stop() can't clobber a newer one). -9. **iOS**: same package (`BUILD_IOS=1` for the xcframework slice); `StreamView` needs the - `UIViewRepresentable` twin and touch→input mapping. +9. **iOS/iPadOS — ported and first-lit** (iPad simulator ↔ the real host, 60 fps). + `BUILD_IOS=1 bash scripts/build-xcframework.sh` builds device + universal-simulator + slices; the Xcode project has a second target, **Punktfunk-iOS**, sharing the same + synchronized sources. The iOS `StreamView` (StreamViewIOS.swift — same name/signature + as the macOS one, so the SwiftUI shell is identical) hosts the shared `StreamPump` in + a view controller for `prefersPointerLocked`: with a hardware mouse/trackpad that is + the iPadOS cursor capture (system honors it fullscreen-and-frontmost; in Stage + Manager it degrades to both-cursors forwarding). Touch is always forwarded — every + finger gets a wire touch id and coordinates map through the aspect-fit letterbox + into host-mode pixels (surface == host mode, so the host rescale is the identity). + `InputCapture` is cross-platform (GC works the same on iPadOS; ⌘⎋ is detected from + the HID stream there); audio routes via `AVAudioSession` (the Settings device + pickers are macOS-only). For the iPad-with-external-display setup: the target + enables multiple scenes + indirect input events — on Stage Manager iPads, drag the + punktfunk window onto the external screen and the stream runs there with full + keyboard/mouse/touch. Known gaps: `prefersPointerLocked` is declared on the stream + view controller but UIHostingController doesn't forward the preference from + representable children, so the system cursor stays visible (relative-mouse + forwarding works regardless — fixing it means putting the controller into the UIKit + presentation chain, e.g. a full-screen UIKit presentation on session start); and + AVAudioSession interruptions (calls, Siri) don't auto-restart the audio engines yet + (reconnect recovers). ## Known limitations of the current host (relevant to client UX) diff --git a/clients/apple/Sources/PunktfunkClient/AddHostSheet.swift b/clients/apple/Sources/PunktfunkClient/AddHostSheet.swift index 1593c86..988f6fe 100644 --- a/clients/apple/Sources/PunktfunkClient/AddHostSheet.swift +++ b/clients/apple/Sources/PunktfunkClient/AddHostSheet.swift @@ -36,7 +36,9 @@ struct AddHostSheet: View { } .padding(16) } + #if os(macOS) .frame(width: 380) .fixedSize(horizontal: false, vertical: true) + #endif } } diff --git a/clients/apple/Sources/PunktfunkClient/ContentView.swift b/clients/apple/Sources/PunktfunkClient/ContentView.swift index 932a50b..1543366 100644 --- a/clients/apple/Sources/PunktfunkClient/ContentView.swift +++ b/clients/apple/Sources/PunktfunkClient/ContentView.swift @@ -8,7 +8,9 @@ // the only way into hosts running --require-pairing. Once pinned, reconnects are silent // and a changed host identity refuses to connect. +#if os(macOS) import AppKit +#endif import PunktfunkKit import SwiftUI @@ -21,6 +23,9 @@ struct ContentView: View { @AppStorage("punktfunk.compositor") private var compositor = 0 @State private var showAddHost = false @State private var pairingTarget: StoredHost? + #if os(iOS) + @State private var showSettings = false + #endif var body: some View { Group { @@ -70,7 +75,9 @@ struct ContentView: View { trustCard(fp) } } + #if os(macOS) .frame(minWidth: 640, minHeight: 360) + #endif .background(Color.black) } @@ -106,17 +113,38 @@ struct ContentView: View { .help("Add a host") } ToolbarItem { + #if os(macOS) SettingsLink { Label("Settings", systemImage: "gearshape") } .help("Stream mode and settings") + #else + Button { + showSettings = true + } label: { + Label("Settings", systemImage: "gearshape") + } + #endif } } } + #if os(macOS) .frame(minWidth: 480, minHeight: 360) + #endif .sheet(isPresented: $showAddHost) { AddHostSheet { store.add($0) } } + #if os(iOS) + .sheet(isPresented: $showSettings) { + NavigationStack { + SettingsView() + .navigationTitle("Settings") + .toolbar { + Button("Done") { showSettings = false } + } + } + } + #endif .alert( "Connection failed", isPresented: Binding( @@ -239,7 +267,11 @@ struct ContentView: View { model.rejectTrust() pairingTarget = host } + #if os(macOS) .buttonStyle(.link) + #else + .buttonStyle(.borderless) + #endif .font(.callout) } .padding(28) @@ -287,11 +319,20 @@ struct ContentView: View { .font(.system(.caption, design: .monospaced)) // While captured the cursor is hidden+frozen, so the button is keyboard-only // (⌘⎋ or Cmd+Tab release the cursor; released, it's clickable again). + #if os(macOS) Text(model.mouseCaptured ? "⌘⎋ releases the mouse" : "Click the stream to capture input") .font(.caption2) .opacity(0.8) + #else + // Touch always plays directly; ⌘⎋ (hardware keyboard) toggles kb/mouse. + Text(model.mouseCaptured + ? "⌘⎋ releases keyboard & mouse" + : "⌘⎋ captures keyboard & mouse") + .font(.caption2) + .opacity(0.8) + #endif Button("Disconnect (⌘D)") { model.disconnect() } .font(.caption) .keyboardShortcut("d", modifiers: .command) diff --git a/clients/apple/Sources/PunktfunkClient/PairSheet.swift b/clients/apple/Sources/PunktfunkClient/PairSheet.swift index d392007..068c9d9 100644 --- a/clients/apple/Sources/PunktfunkClient/PairSheet.swift +++ b/clients/apple/Sources/PunktfunkClient/PairSheet.swift @@ -25,7 +25,11 @@ struct PairSheet: View { let onPaired: (Data) -> Void @State private var pin = "" + #if os(macOS) @State private var clientName = Host.current().localizedName ?? "Mac" + #else + @State private var clientName = UIDevice.current.name + #endif @State private var busy = false @State private var errorText: String? @State private var token = CeremonyToken() @@ -77,8 +81,10 @@ struct PairSheet: View { } .padding(16) } + #if os(macOS) .frame(width: 400) .fixedSize(horizontal: false, vertical: true) + #endif .interactiveDismissDisabled(busy) .onDisappear { token.cancelled = true } // any other dismissal path } diff --git a/clients/apple/Sources/PunktfunkClient/PunktfunkClientApp.swift b/clients/apple/Sources/PunktfunkClient/PunktfunkClientApp.swift index 64282e9..692e197 100644 --- a/clients/apple/Sources/PunktfunkClient/PunktfunkClientApp.swift +++ b/clients/apple/Sources/PunktfunkClient/PunktfunkClientApp.swift @@ -1,23 +1,30 @@ // PunktfunkClient — the macOS client app (also runs unbundled via swift run). // Hosts grid → trust-on-first-use → StreamView (AVSampleBufferDisplayLayer HEVC) + input. +#if os(macOS) import AppKit +#endif import SwiftUI @main struct PunktfunkClientApp: App { + #if os(macOS) @NSApplicationDelegateAdaptor(AppDelegate.self) private var appDelegate + #endif var body: some Scene { WindowGroup("punktfunk") { ContentView() } + #if os(macOS) Settings { SettingsView() } + #endif } } +#if os(macOS) final class AppDelegate: NSObject, NSApplicationDelegate { func applicationDidFinishLaunching(_ notification: Notification) { // `swift run` launches an unbundled binary; promote it to a regular app so the @@ -30,3 +37,4 @@ final class AppDelegate: NSObject, NSApplicationDelegate { true } } +#endif diff --git a/clients/apple/Sources/PunktfunkClient/SettingsView.swift b/clients/apple/Sources/PunktfunkClient/SettingsView.swift index 2891390..16aed9a 100644 --- a/clients/apple/Sources/PunktfunkClient/SettingsView.swift +++ b/clients/apple/Sources/PunktfunkClient/SettingsView.swift @@ -2,7 +2,9 @@ // virtual output at exactly this size/refresh — there is no scaling anywhere in the // pipeline. +#if os(macOS) import AppKit +#endif import PunktfunkKit import SwiftUI @@ -11,11 +13,13 @@ struct SettingsView: View { @AppStorage("punktfunk.height") private var height = 1080 @AppStorage("punktfunk.hz") private var hz = 60 @AppStorage("punktfunk.compositor") private var compositor = 0 + @AppStorage("punktfunk.micEnabled") private var micEnabled = true + #if os(macOS) @AppStorage("punktfunk.speakerUID") private var speakerUID = "" @AppStorage("punktfunk.micUID") private var micUID = "" - @AppStorage("punktfunk.micEnabled") private var micEnabled = true @State private var outputDevices: [AudioDevice] = [] @State private var inputDevices: [AudioDevice] = [] + #endif var body: some View { Form { @@ -39,6 +43,7 @@ struct SettingsView: View { .foregroundStyle(.secondary) } Section { + #if os(macOS) Picker("Speaker", selection: $speakerUID) { Text("System default").tag("") ForEach(outputDevices) { device in @@ -49,7 +54,9 @@ struct SettingsView: View { Text("Unavailable device").tag(speakerUID) } } + #endif Toggle("Send microphone to the host", isOn: $micEnabled) + #if os(macOS) Picker("Microphone", selection: $micUID) { Text("System default").tag("") ForEach(inputDevices) { device in @@ -61,6 +68,7 @@ struct SettingsView: View { } } .disabled(!micEnabled) + #endif } header: { Text("Audio") } footer: { @@ -89,19 +97,29 @@ struct SettingsView: View { } } .formStyle(.grouped) + #if os(macOS) .frame(width: 380) .fixedSize() .onAppear { outputDevices = AudioDevices.outputs() inputDevices = AudioDevices.inputs() } + #endif } private func fillFromMainScreen() { + #if os(macOS) guard let screen = NSScreen.main else { return } let scale = screen.backingScaleFactor width = Int(screen.frame.width * scale) height = Int(screen.frame.height * scale) hz = screen.maximumFramesPerSecond + #else + // nativeBounds is portrait-oriented pixels — streams are landscape. + let bounds = UIScreen.main.nativeBounds + width = Int(max(bounds.width, bounds.height)) + height = Int(min(bounds.width, bounds.height)) + hz = UIScreen.main.maximumFramesPerSecond + #endif } } diff --git a/clients/apple/Sources/PunktfunkKit/InputCapture.swift b/clients/apple/Sources/PunktfunkKit/InputCapture.swift index 6461c0e..0c77103 100644 --- a/clients/apple/Sources/PunktfunkKit/InputCapture.swift +++ b/clients/apple/Sources/PunktfunkKit/InputCapture.swift @@ -25,6 +25,10 @@ #if os(macOS) import AppKit +#endif +#if os(iOS) +import UIKit +#endif import Foundation import GameController import PunktfunkCore @@ -36,7 +40,9 @@ public final class InputCapture { private var observers: [NSObjectProtocol] = [] private var mice: [GCMouse] = [] private var keyboards: [GCKeyboard] = [] + #if os(macOS) private var keyEventMonitor: Any? + #endif // Main-queue-only state (see header comment). private var residualX: Float = 0 @@ -53,6 +59,9 @@ public final class InputCapture { /// reaches GCKeyboard, racing the NSEvent monitor — latched here so it can't type /// an Escape into the host in either toggle direction. private var suppressedVK: UInt32? + /// Physical ⌘ keys currently held (tracked even while released — the ⌘⎋ toggle and + /// its Esc suppression need it in both states). + private var cmdKeysDown: Set = [] /// While true, mouse/keyboard flow to the host and key NSEvents are swallowed /// locally; while false the user is interacting with the local UI (dragging the @@ -119,8 +128,13 @@ public final class InputCapture { if let k = n.object as? GCKeyboard { self?.attach(keyboard: k) } }) // Focus loss: GC stops delivering, so release everything still held host-side. + #if os(macOS) + let resignActive = NSApplication.didResignActiveNotification + #else + let resignActive = UIApplication.willResignActiveNotification + #endif observers.append(NotificationCenter.default.addObserver( - forName: NSApplication.didResignActiveNotification, object: nil, queue: .main + forName: resignActive, object: nil, queue: .main ) { [weak self] _ in self?.releaseAll() }) @@ -128,6 +142,8 @@ public final class InputCapture { // that one combo is intercepted: swallowing keys wholesale at the monitor level // risks starving GC's own delivery, so the no-beep behavior lives in // StreamLayerView (first responder consumes keyDown/keyUp while captured). + // (On iOS there is no NSEvent monitor — the GC key handler detects the combo.) + #if os(macOS) keyEventMonitor = NSEvent.addLocalMonitorForEvents( matching: [.keyDown] ) { [weak self] event in @@ -140,16 +156,19 @@ public final class InputCapture { } return event } + #endif } public func stop() { releaseAll() observers.forEach(NotificationCenter.default.removeObserver(_:)) observers.removeAll() + #if os(macOS) if let monitor = keyEventMonitor { NSEvent.removeMonitor(monitor) keyEventMonitor = nil } + #endif // Don't clobber the handlers if a newer capture has taken the global devices. if Self.activeCapture === self || Self.activeCapture == nil { for mouse in mice { @@ -172,8 +191,12 @@ public final class InputCapture { deinit { stop() } - /// Send release events for everything currently held, and drop the motion residuals. + /// Send release events for everything currently held, and drop the motion residuals + /// and modifier/latch tracking (GC delivers nothing while inactive, so a ⌘ released + /// in another app would otherwise stay "held" here forever — hijacking Esc). private func releaseAll() { + cmdKeysDown.removeAll() + suppressedVK = nil for vk in pressedVKs { connection.send(.key(vk, down: false)) } @@ -264,16 +287,32 @@ public final class InputCapture { keyboards.append(keyboard) keyboard.keyboardInput?.keyChangedHandler = { [weak self] _, _, keyCode, pressed in guard let self, let vk = Self.hidToVK[keyCode.rawValue] else { return } + if vk == 0x5B || vk == 0x5C { // physical ⌘ state, tracked in both states + if pressed { + self.cmdKeysDown.insert(vk) + } else { + self.cmdKeysDown.remove(vk) + } + } // The ⌘⎋ toggle's Esc — checked before the forwarding gate, because in the // engage direction forwarding is already true when this fires. if vk == self.suppressedVK { if !pressed { self.suppressedVK = nil } return } + #if os(iOS) + // No NSEvent monitor here — the toggle combo is detected from the HID + // stream itself. + if pressed, vk == 0x1B, !self.cmdKeysDown.isEmpty { + self.suppressedVK = 0x1B + self.onToggleCapture?() + return + } + #endif guard self.forwarding else { return } // Release direction of the toggle: GC's Esc-down can beat the NSEvent // monitor — never type Esc into the host while ⌘ is held (⌘⎋ is reserved). - if vk == 0x1B, self.pressedVKs.contains(0x5B) || self.pressedVKs.contains(0x5C) { + if vk == 0x1B, !self.cmdKeysDown.isEmpty { return } if pressed { @@ -325,4 +364,3 @@ public final class InputCapture { return m }() } -#endif diff --git a/clients/apple/Sources/PunktfunkKit/SessionAudio.swift b/clients/apple/Sources/PunktfunkKit/SessionAudio.swift index 3fcb982..c870e19 100644 --- a/clients/apple/Sources/PunktfunkKit/SessionAudio.swift +++ b/clients/apple/Sources/PunktfunkKit/SessionAudio.swift @@ -14,7 +14,6 @@ // AVAudioEngine ties input+output to one aggregate clock, separate engines keep // arbitrary mic/speaker combinations trivial. -#if os(macOS) import AVFoundation import os @@ -140,9 +139,29 @@ public final class SessionAudio { } /// Start playback (and, if enabled+authorized, the mic uplink). Empty UIDs = system - /// default device. Main thread (engine setup); returns after the engines start — - /// the mic may start slightly later if the permission prompt is pending. + /// default device; on iOS the UIDs are ignored entirely (routes are + /// AVAudioSession-managed). Main thread (engine setup); returns after the engines + /// start — the mic may start slightly later if the permission prompt is pending. public func start(speakerUID: String, micUID: String, micEnabled: Bool) { + #if os(iOS) + // Route + policy live in the session, not per-engine: stereo playback, mic + // capture when enabled, Bluetooth allowed. Failure is non-fatal (defaults). + let session = AVAudioSession.sharedInstance() + do { + if micEnabled { + // .defaultToSpeaker: .playAndRecord otherwise routes to the iPhone + // EARPIECE; only affects the built-in route (headphones/BT still win). + try session.setCategory( + .playAndRecord, mode: .default, + options: [.allowBluetoothA2DP, .defaultToSpeaker]) + } else { + try session.setCategory(.playback, mode: .default) + } + try session.setActive(true) + } catch { + log.warning("AVAudioSession setup failed: \(error.localizedDescription)") + } + #endif startPlayback(speakerUID: speakerUID) guard micEnabled else { return } switch AVCaptureDevice.authorizationStatus(for: .audio) { @@ -180,6 +199,16 @@ public final class SessionAudio { if wasDraining { _ = drainDone.wait(timeout: .now() + .milliseconds(400)) } + #if os(iOS) + // Release the session so audio we interrupted (Music, podcasts) gets its + // resume cue. + do { + try AVAudioSession.sharedInstance().setActive( + false, options: .notifyOthersOnDeactivation) + } catch { + log.warning("AVAudioSession deactivation failed: \(error.localizedDescription)") + } + #endif } // MARK: - Playback (host → speaker) @@ -190,6 +219,7 @@ public final class SessionAudio { let ring = AudioRing(capacity: 96_000, prefill: 1920) let engine = AVAudioEngine() + #if os(macOS) if !speakerUID.isEmpty { if let dev = AudioDevices.deviceID(forUID: speakerUID), let unit = engine.outputNode.audioUnit { @@ -200,6 +230,7 @@ public final class SessionAudio { log.warning("speaker \(speakerUID) not present — using default") } } + #endif // Engine-native deinterleaved float; the render block deinterleaves from the ring. guard let format = AVAudioFormat(standardFormatWithSampleRate: 48_000, channels: 2) @@ -282,6 +313,7 @@ public final class SessionAudio { private func startCapture(micUID: String) { let engine = AVAudioEngine() let input = engine.inputNode + #if os(macOS) if !micUID.isEmpty { if let dev = AudioDevices.deviceID(forUID: micUID), let unit = input.audioUnit { if !Self.setDevice(dev, on: unit) { @@ -291,6 +323,7 @@ public final class SessionAudio { log.warning("microphone \(micUID) not present — using default") } } + #endif let inFormat = input.outputFormat(forBus: 0) guard inFormat.sampleRate > 0, inFormat.channelCount > 0 else { @@ -376,11 +409,12 @@ public final class SessionAudio { log.info("mic uplink started (\(micUID.isEmpty ? "default input" : micUID))") } + #if os(macOS) private static func setDevice(_ id: AudioDeviceID, on unit: AudioUnit) -> Bool { var dev = id return AudioUnitSetProperty( unit, kAudioOutputUnitProperty_CurrentDevice, kAudioUnitScope_Global, 0, &dev, UInt32(MemoryLayout.size)) == noErr } + #endif } -#endif diff --git a/clients/apple/Sources/PunktfunkKit/StreamPump.swift b/clients/apple/Sources/PunktfunkKit/StreamPump.swift new file mode 100644 index 0000000..fbcb7fa --- /dev/null +++ b/clients/apple/Sources/PunktfunkKit/StreamPump.swift @@ -0,0 +1,84 @@ +// The platform-independent heart of the presenters: one thread pulling AUs from the +// connection into an AVSampleBufferDisplayLayer, with the format description refreshed +// on every IDR (the host opens with an IDR carrying in-band parameter sets; recovery +// keyframes re-send them — there is no out-of-band extradata, ever). Shared by the +// macOS StreamLayerView and the iOS/iPadOS stream view. + +import AVFoundation +import Foundation + +/// Cancellation handle owned by exactly one pump thread — a restart hands the old pump +/// its own token, so it can never be revived by a newer start(). +private final class PumpToken: @unchecked Sendable { + private let lock = NSLock() + private var live = true + var isLive: Bool { + lock.lock() + defer { lock.unlock() } + return live + } + func cancel() { + lock.lock() + live = false + lock.unlock() + } +} + +/// One pump per instance; create a fresh StreamPump per start (cancel is permanent). +final class StreamPump { + private let token = PumpToken() + + /// Pump thread: pull AUs, wrap, enqueue. Non-IDR AUs before the first format + /// description are dropped. `onFrame`/`onSessionEnd` fire on the pump thread. + func start( + connection: PunktfunkConnection, + layer: AVSampleBufferDisplayLayer, + onFrame: (@Sendable (AccessUnit) -> Void)?, + onSessionEnd: (@Sendable () -> Void)? + ) { + let token = token + layer.flush() // drop any frames a previous connection left queued + + let thread = Thread { + var format: CMVideoFormatDescription? + while token.isLive { + do { + guard let au = try connection.nextAU(timeoutMs: 100) else { continue } + onFrame?(au) + if let f = AnnexB.formatDescription(fromIDR: au.data) { + format = f // refreshed on every IDR (mode changes included) + } + if layer.status == .failed { + // Decode wedged: flush and re-gate on the next in-band parameter + // sets — resuming with a delta frame can't recover. (A + // request-IDR channel on punktfunk/1 is a host-side TODO; with the + // host's infinite GOP this may otherwise stay black until the + // next recovery keyframe.) + layer.flush() + format = AnnexB.formatDescription(fromIDR: au.data) + } + guard let f = format, + let sample = AnnexB.sampleBuffer(au: au, format: f), + token.isLive // don't enqueue a stale frame after a restart + else { continue } + layer.enqueue(sample) + } catch { + if token.isLive { + onSessionEnd?() + } + break // session closed + } + } + } + thread.name = "punktfunk-pump" + thread.qualityOfService = .userInteractive + thread.start() + } + + /// Stop pumping (≤ one poll timeout). Does not close the connection. + func stop() { + token.cancel() + } + + deinit { token.cancel() } +} diff --git a/clients/apple/Sources/PunktfunkKit/StreamView.swift b/clients/apple/Sources/PunktfunkKit/StreamView.swift index 7902b0d..6ef1324 100644 --- a/clients/apple/Sources/PunktfunkKit/StreamView.swift +++ b/clients/apple/Sources/PunktfunkKit/StreamView.swift @@ -99,25 +99,8 @@ public struct StreamView: NSViewRepresentable { } public final class StreamLayerView: NSView { - /// Cancellation handle owned by exactly one pump thread — a restart hands the old pump - /// its own token, so it can never be revived by a newer start(). - private final class PumpToken: @unchecked Sendable { - private let lock = NSLock() - private var live = true - var isLive: Bool { - lock.lock() - defer { lock.unlock() } - return live - } - func cancel() { - lock.lock() - live = false - lock.unlock() - } - } - private let displayLayer = AVSampleBufferDisplayLayer() - private var token: PumpToken? + private var pump: StreamPump? public private(set) var connection: PunktfunkConnection? private let cursorCapture = CursorCapture() private var inputCapture: InputCapture? @@ -261,7 +244,7 @@ public final class StreamLayerView: NSView { // A click is explicit intent AND may arrive mid-activation (acceptsFirstMouse: // NSApp.isActive / isKeyWindow are still false for the click coming in from // another app) — only the auto-engage paths require already-held key status. - guard captureEnabled, !captured, token != nil, window != nil, + guard captureEnabled, !captured, pump != nil, window != nil, fromClick || (NSApp.isActive && window?.isKeyWindow == true) else { return } cursorCapture.capture(in: self) @@ -297,11 +280,7 @@ public final class StreamLayerView: NSView { onSessionEnd: (@Sendable () -> Void)? = nil ) { stop() - let token = PumpToken() - self.token = token self.connection = connection - let layer = displayLayer - layer.flush() // drop any frames a previous connection left queued // The view owns the session's input capture: handlers attach now, but nothing is // forwarded until capture engages (captureEnabled + auto-engage or a click). @@ -324,40 +303,11 @@ public final class StreamLayerView: NSView { capture.start() inputCapture = capture - let thread = Thread { - var format: CMVideoFormatDescription? - while token.isLive { - do { - guard let au = try connection.nextAU(timeoutMs: 100) else { continue } - onFrame?(au) - if let f = AnnexB.formatDescription(fromIDR: au.data) { - format = f // refreshed on every IDR (mode changes included) - } - if layer.status == .failed { - // Decode wedged: flush and re-gate on the next in-band parameter - // sets — resuming with a delta frame can't recover. (A - // request-IDR channel on punktfunk/1 is a host-side TODO; with the - // host's infinite GOP this may otherwise stay black until the - // next recovery keyframe.) - layer.flush() - format = AnnexB.formatDescription(fromIDR: au.data) - } - guard let f = format, - let sample = AnnexB.sampleBuffer(au: au, format: f), - token.isLive // don't enqueue a stale frame after a restart - else { continue } - layer.enqueue(sample) - } catch { - if token.isLive { - onSessionEnd?() - } - break // session closed - } - } - } - thread.name = "punktfunk-pump" - thread.qualityOfService = .userInteractive - thread.start() + let pump = StreamPump() + pump.start( + connection: connection, layer: displayLayer, + onFrame: onFrame, onSessionEnd: onSessionEnd) + self.pump = pump requestAutoCapture() // entering a session is the deliberate "capture me" moment } @@ -367,15 +317,15 @@ public final class StreamLayerView: NSView { releaseCapture() inputCapture?.stop() inputCapture = nil - token?.cancel() - token = nil + pump?.stop() + pump = nil connection = nil } deinit { appObservers.forEach(NotificationCenter.default.removeObserver(_:)) windowObservers.forEach(NotificationCenter.default.removeObserver(_:)) - token?.cancel() + pump?.stop() } } #endif diff --git a/clients/apple/Sources/PunktfunkKit/StreamViewIOS.swift b/clients/apple/Sources/PunktfunkKit/StreamViewIOS.swift new file mode 100644 index 0000000..b7d9a70 --- /dev/null +++ b/clients/apple/Sources/PunktfunkKit/StreamViewIOS.swift @@ -0,0 +1,272 @@ +// iOS/iPadOS presenter: the same AVSampleBufferDisplayLayer + StreamPump as macOS, +// hosted in a UIViewController so the scene can pointer-lock (the iPadOS equivalent of +// the Mac's cursor capture — with a hardware mouse/trackpad the system cursor is hidden +// and GCMouse's raw deltas drive the host cursor alone; the system only honors the lock +// fullscreen-and-frontmost, so in Stage Manager it degrades to Mac-style "both cursors +// visible" forwarding). +// +// Touch is the primary input and is always forwarded (touching the video IS explicit +// intent): every finger maps to a wire touch id, coordinates are mapped through the +// aspect-fit letterbox into host-mode pixels, so surface == host mode and the host's +// rescale is the identity. Hardware keyboard/mouse forwarding shares InputCapture with +// macOS — auto-engaged when streaming starts, ⌘⎋ toggles (detected from the HID stream; +// there is no NSEvent monitor here). +// +// The public type is named StreamView like its macOS twin (each is platform-gated), so +// the SwiftUI app layer is identical on both platforms. + +#if os(iOS) +import AVFoundation +import GameController +import PunktfunkCore +import SwiftUI +import UIKit + +public struct StreamView: UIViewControllerRepresentable { + private let connection: PunktfunkConnection + private let captureEnabled: Bool + private let onCaptureChange: ((Bool) -> Void)? + private let onFrame: (@Sendable (AccessUnit) -> Void)? + private let onSessionEnd: (@Sendable () -> Void)? + + public init( + connection: PunktfunkConnection, + captureEnabled: Bool = true, + onCaptureChange: ((Bool) -> Void)? = nil, + onFrame: (@Sendable (AccessUnit) -> Void)? = nil, + onSessionEnd: (@Sendable () -> Void)? = nil + ) { + self.connection = connection + self.captureEnabled = captureEnabled + self.onCaptureChange = onCaptureChange + self.onFrame = onFrame + self.onSessionEnd = onSessionEnd + } + + public func makeUIViewController(context: Context) -> StreamViewController { + let controller = StreamViewController() + controller.onCaptureChange = onCaptureChange + controller.captureEnabled = captureEnabled + controller.start(connection: connection, onFrame: onFrame, onSessionEnd: onSessionEnd) + return controller + } + + public func updateUIViewController(_ controller: StreamViewController, context: Context) { + controller.onCaptureChange = onCaptureChange + controller.captureEnabled = captureEnabled + if controller.connection !== connection { + controller.start(connection: connection, onFrame: onFrame, onSessionEnd: onSessionEnd) + } + } + + public static func dismantleUIViewController( + _ controller: StreamViewController, coordinator: () + ) { + controller.stop() + } +} + +public final class StreamViewController: UIViewController { + public private(set) var connection: PunktfunkConnection? + private var pump: StreamPump? + private var inputCapture: InputCapture? + private var captured = false + private var observers: [NSObjectProtocol] = [] + + var onCaptureChange: ((Bool) -> Void)? + + var captureEnabled = true { + didSet { + guard captureEnabled != oldValue else { return } + setCaptured(captureEnabled) + } + } + + private var streamView: StreamLayerUIView { + // swiftlint:disable:next force_cast + view as! StreamLayerUIView + } + + public override func loadView() { + view = StreamLayerUIView() + } + + public override var prefersPointerLocked: Bool { captured } + public override var prefersHomeIndicatorAutoHidden: Bool { true } + + func start( + connection: PunktfunkConnection, + onFrame: (@Sendable (AccessUnit) -> Void)?, + onSessionEnd: (@Sendable () -> Void)? + ) { + stop() + self.connection = connection + loadViewIfNeeded() + // Read the LIVE mode per touch batch — an accepted requestMode() mid-stream + // changes the letterbox, and touches must follow it. + streamView.currentHostMode = { [weak connection] in + guard let connection else { return .zero } + let mode = connection.currentMode() + return CGSize(width: Double(mode.width), height: Double(mode.height)) + } + streamView.onTouchEvent = { [weak connection] event in + connection?.send(event) + } + + let capture = InputCapture(connection: connection) + capture.onToggleCapture = { [weak self] in + guard let self else { return } + self.setCaptured(!self.captured) + } + capture.onPreempted = { [weak self] in + self?.setCaptured(false) + } + capture.start() + inputCapture = capture + + let pump = StreamPump() + pump.start( + connection: connection, layer: streamView.displayLayer, + onFrame: onFrame, onSessionEnd: onSessionEnd) + self.pump = pump + + // GC only delivers while active; everything held is flushed by InputCapture's + // own resign observer — here we just mirror the capture state for the HUD and + // the pointer lock. + observers.append(NotificationCenter.default.addObserver( + forName: UIApplication.willResignActiveNotification, object: nil, queue: .main + ) { [weak self] _ in + self?.setCaptured(false) + }) + + if captureEnabled { + setCaptured(true) // entering a session is the deliberate "capture me" moment + } + } + + func stop() { + setCaptured(false) + observers.forEach(NotificationCenter.default.removeObserver(_:)) + observers.removeAll() + inputCapture?.stop() + inputCapture = nil + pump?.stop() + pump = nil + connection = nil + streamView.onTouchEvent = nil + streamView.currentHostMode = nil + } + + private func setCaptured(_ on: Bool) { + if on { + guard captureEnabled, !captured, pump != nil else { return } + inputCapture?.setForwarding(true) + captured = true + } else { + guard captured else { return } + inputCapture?.setForwarding(false) + captured = false + } + setNeedsUpdateOfPrefersPointerLocked() + let onCaptureChange = onCaptureChange + let captured = captured + DispatchQueue.main.async { onCaptureChange?(captured) } + } + + deinit { + observers.forEach(NotificationCenter.default.removeObserver(_:)) + pump?.stop() + } +} + +/// The layer-backed video surface + touch source. Touches are mapped through the +/// aspect-fit letterbox into host-mode pixels (surface == host mode, so the host-side +/// rescale is the identity); touches outside the video area are clamped onto its edge. +final class StreamLayerUIView: UIView { + override class var layerClass: AnyClass { AVSampleBufferDisplayLayer.self } + var displayLayer: AVSampleBufferDisplayLayer { + // swiftlint:disable:next force_cast + layer as! AVSampleBufferDisplayLayer + } + + /// Reads the LIVE negotiated mode in pixels (the touch coordinate space). + var currentHostMode: (() -> CGSize)? + var onTouchEvent: ((PunktfunkInputEvent) -> Void)? + + /// Wire touch ids per active UITouch; ids are reused after the touch ends. + private var touchIDs: [ObjectIdentifier: UInt32] = [:] + + override init(frame: CGRect) { + super.init(frame: frame) + displayLayer.videoGravity = .resizeAspect + isMultipleTouchEnabled = true + backgroundColor = .black + } + + @available(*, unavailable) + required init?(coder: NSCoder) { fatalError("not used") } + + override func touchesBegan(_ touches: Set, with event: UIEvent?) { + forward(touches, kind: .down) + } + override func touchesMoved(_ touches: Set, with event: UIEvent?) { + forward(touches, kind: .move) + } + override func touchesEnded(_ touches: Set, with event: UIEvent?) { + forward(touches, kind: .up) + } + override func touchesCancelled(_ touches: Set, with event: UIEvent?) { + forward(touches, kind: .up) + } + + private enum TouchKind { case down, move, up } + + private func forward(_ touches: Set, kind: TouchKind) { + guard let hostMode = currentHostMode?(), + hostMode.width > 0, hostMode.height > 0, onTouchEvent != nil + else { return } + let video = AVMakeRect(aspectRatio: hostMode, insideRect: bounds) + guard video.width > 0, video.height > 0 else { return } + for touch in touches { + let key = ObjectIdentifier(touch) + let id: UInt32 + switch kind { + case .down: + id = nextFreeID() + touchIDs[key] = id + case .move, .up: + guard let known = touchIDs[key] else { continue } + id = known + } + if kind == .up { + touchIDs.removeValue(forKey: key) + onTouchEvent?(.touchUp(id: id)) + continue + } + let p = touch.location(in: self) + let x = Int32(((p.x - video.minX) / video.width * hostMode.width) + .rounded().clamped(to: 0...(hostMode.width - 1))) + let y = Int32(((p.y - video.minY) / video.height * hostMode.height) + .rounded().clamped(to: 0...(hostMode.height - 1))) + let w = UInt32(hostMode.width) + let h = UInt32(hostMode.height) + onTouchEvent?( + kind == .down + ? .touchDown(id: id, x: x, y: y, surfaceWidth: w, surfaceHeight: h) + : .touchMove(id: id, x: x, y: y, surfaceWidth: w, surfaceHeight: h)) + } + } + + private func nextFreeID() -> UInt32 { + var id: UInt32 = 0 + while touchIDs.values.contains(id) { id += 1 } + return id + } +} + +extension CGFloat { + fileprivate func clamped(to range: ClosedRange) -> CGFloat { + Swift.min(Swift.max(self, range.lowerBound), range.upperBound) + } +} +#endif diff --git a/scripts/build-xcframework.sh b/scripts/build-xcframework.sh index 8dff1af..ecf4837 100644 --- a/scripts/build-xcframework.sh +++ b/scripts/build-xcframework.sh @@ -11,7 +11,7 @@ set -euo pipefail cd "$(dirname "$0")/.." TARGETS_MAC=(aarch64-apple-darwin x86_64-apple-darwin) -BUILD_IOS="${BUILD_IOS:-0}" # BUILD_IOS=1 adds an iOS slice (requires the ios target installed) +BUILD_IOS="${BUILD_IOS:-0}" # BUILD_IOS=1 adds iOS device + simulator slices (rustup targets aarch64-apple-ios{,-sim}) # Deployment targets must match Package.swift's platforms, or every consumer link emits # "object file was built for newer macOS version" warnings. @@ -20,6 +20,8 @@ for t in "${TARGETS_MAC[@]}"; do done if [[ "$BUILD_IOS" == "1" ]]; then IPHONEOS_DEPLOYMENT_TARGET=17.0 cargo build --release -p punktfunk-core --features quic --target aarch64-apple-ios + IPHONEOS_DEPLOYMENT_TARGET=17.0 cargo build --release -p punktfunk-core --features quic --target aarch64-apple-ios-sim + IPHONEOS_DEPLOYMENT_TARGET=17.0 cargo build --release -p punktfunk-core --features quic --target x86_64-apple-ios fi STAGE="$(mktemp -d)" @@ -48,7 +50,14 @@ EOF ARGS=(-library "$STAGE/macos/libpunktfunk_core.a" -headers "$STAGE/include") if [[ "$BUILD_IOS" == "1" ]]; then + # Universal simulator lib (arm64 Macs run arm64 sims, but generic builds link x86_64 too). + mkdir -p "$STAGE/iossim" + lipo -create \ + target/aarch64-apple-ios-sim/release/libpunktfunk_core.a \ + target/x86_64-apple-ios/release/libpunktfunk_core.a \ + -output "$STAGE/iossim/libpunktfunk_core.a" ARGS+=(-library target/aarch64-apple-ios/release/libpunktfunk_core.a -headers "$STAGE/include") + ARGS+=(-library "$STAGE/iossim/libpunktfunk_core.a" -headers "$STAGE/include") fi # Cargo does NOT fingerprint MACOSX_DEPLOYMENT_TARGET — units cached from a build without