feat(apple): iOS/iPadOS client — touch, pointer lock, shared SwiftUI shell
ci / rust (push) Has been cancelled
ci / rust (push) Has been cancelled
The whole client now runs on iPadOS/iOS from the same sources, first-lit live in the
iPad simulator against the real host at 1280x720@60 (60 fps on the HUD, capture state
machine active, mic permission flow shown).
- PunktfunkCore.xcframework grows iOS device + universal-simulator slices
(BUILD_IOS=1; rustup targets aarch64-apple-ios{,-sim} + x86_64-apple-ios).
- The decode pump is extracted into a shared StreamPump (identical IDR re-gate logic on
both platforms); the iOS StreamView (StreamViewIOS.swift) has the same name/signature
as the macOS one, so ContentView & co. are byte-identical across platforms — hosted
in a UIViewController for prefersPointerLocked (the iPadOS cursor capture; see README
note 9 for the UIHostingController forwarding caveat).
- Touch is always forwarded: per-finger wire ids, coordinates mapped through the
aspect-fit letterbox into LIVE host-mode pixels (surface == host mode, identity
rescale host-side; follows mid-stream requestMode switches).
- InputCapture is cross-platform: GC works the same on iPadOS, ⌘⎋ is detected from the
HID stream there; stale-⌘ tracking after focus loss fixed on both platforms
(releaseAll now drops the modifier/latch state — a ⌘ released in another app
otherwise hijacked Esc forever).
- SessionAudio: AVAudioSession on iOS (.playAndRecord + .defaultToSpeaker — without it
iPhones route host audio to the EARPIECE; deactivated with
notifyOthersOnDeactivation on stop so interrupted background audio resumes); HAL
device pinning + the Settings pickers stay macOS-only.
- New Punktfunk-iOS app target (shared synchronized sources, generated Info.plist with
mic + local-network usage descriptions — QUIC to a LAN host trips local network
privacy on real devices — scene manifest + indirect input events for Stage Manager /
external displays), shared scheme, macOS min-window frames gated off iOS.
For the iPad-on-an-external-screen idea: with multiple scenes + indirect input enabled,
Stage Manager iPads can drag the punktfunk window onto the external display and drive
the PC with keyboard/mouse/touch. Known gaps (README note 9): the pointer-lock
preference isn't consulted through UIHostingController (relative mouse works, the local
cursor just stays visible) and AVAudioSession interruptions don't auto-restart audio.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -8,10 +8,12 @@
|
||||
|
||||
/* Begin PBXBuildFile section */
|
||||
AA0000000000000000000005 /* PunktfunkKit in Frameworks */ = {isa = PBXBuildFile; productRef = AA0000000000000000000006 /* PunktfunkKit */; };
|
||||
BB0000000000000000000005 /* PunktfunkKit in Frameworks */ = {isa = PBXBuildFile; productRef = BB0000000000000000000006 /* PunktfunkKit */; };
|
||||
/* End PBXBuildFile section */
|
||||
|
||||
/* Begin PBXFileReference section */
|
||||
AA0000000000000000000001 /* Punktfunk.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = Punktfunk.app; sourceTree = BUILT_PRODUCTS_DIR; };
|
||||
BB0000000000000000000001 /* Punktfunk-iOS.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = "Punktfunk-iOS.app"; sourceTree = BUILT_PRODUCTS_DIR; };
|
||||
/* End PBXFileReference section */
|
||||
|
||||
/* Begin PBXFileSystemSynchronizedRootGroup section */
|
||||
@@ -36,6 +38,14 @@
|
||||
);
|
||||
runOnlyForDeploymentPostprocessing = 0;
|
||||
};
|
||||
BB0000000000000000000004 /* Frameworks */ = {
|
||||
isa = PBXFrameworksBuildPhase;
|
||||
buildActionMask = 2147483647;
|
||||
files = (
|
||||
BB0000000000000000000005 /* PunktfunkKit in Frameworks */,
|
||||
);
|
||||
runOnlyForDeploymentPostprocessing = 0;
|
||||
};
|
||||
/* End PBXFrameworksBuildPhase section */
|
||||
|
||||
/* Begin PBXGroup section */
|
||||
@@ -52,6 +62,7 @@
|
||||
isa = PBXGroup;
|
||||
children = (
|
||||
AA0000000000000000000001 /* Punktfunk.app */,
|
||||
BB0000000000000000000001 /* Punktfunk-iOS.app */,
|
||||
);
|
||||
name = Products;
|
||||
sourceTree = "<group>";
|
||||
@@ -83,6 +94,30 @@
|
||||
productReference = AA0000000000000000000001 /* Punktfunk.app */;
|
||||
productType = "com.apple.product-type.application";
|
||||
};
|
||||
BB0000000000000000000009 /* Punktfunk-iOS */ = {
|
||||
isa = PBXNativeTarget;
|
||||
buildConfigurationList = BB000000000000000000000A /* Build configuration list for PBXNativeTarget "Punktfunk-iOS" */;
|
||||
buildPhases = (
|
||||
BB000000000000000000000B /* Sources */,
|
||||
BB0000000000000000000004 /* Frameworks */,
|
||||
BB000000000000000000000C /* Resources */,
|
||||
);
|
||||
buildRules = (
|
||||
);
|
||||
dependencies = (
|
||||
);
|
||||
fileSystemSynchronizedGroups = (
|
||||
AA0000000000000000000002 /* App */,
|
||||
AA0000000000000000000003 /* Sources/PunktfunkClient */,
|
||||
);
|
||||
name = "Punktfunk-iOS";
|
||||
packageProductDependencies = (
|
||||
BB0000000000000000000006 /* PunktfunkKit */,
|
||||
);
|
||||
productName = "Punktfunk-iOS";
|
||||
productReference = BB0000000000000000000001 /* Punktfunk-iOS.app */;
|
||||
productType = "com.apple.product-type.application";
|
||||
};
|
||||
/* End PBXNativeTarget section */
|
||||
|
||||
/* Begin PBXProject section */
|
||||
@@ -114,6 +149,7 @@
|
||||
projectRoot = "";
|
||||
targets = (
|
||||
AA0000000000000000000009 /* Punktfunk */,
|
||||
BB0000000000000000000009 /* Punktfunk-iOS */,
|
||||
);
|
||||
};
|
||||
/* End PBXProject section */
|
||||
@@ -126,6 +162,13 @@
|
||||
);
|
||||
runOnlyForDeploymentPostprocessing = 0;
|
||||
};
|
||||
BB000000000000000000000C /* Resources */ = {
|
||||
isa = PBXResourcesBuildPhase;
|
||||
buildActionMask = 2147483647;
|
||||
files = (
|
||||
);
|
||||
runOnlyForDeploymentPostprocessing = 0;
|
||||
};
|
||||
/* End PBXResourcesBuildPhase section */
|
||||
|
||||
/* Begin PBXSourcesBuildPhase section */
|
||||
@@ -136,6 +179,13 @@
|
||||
);
|
||||
runOnlyForDeploymentPostprocessing = 0;
|
||||
};
|
||||
BB000000000000000000000B /* Sources */ = {
|
||||
isa = PBXSourcesBuildPhase;
|
||||
buildActionMask = 2147483647;
|
||||
files = (
|
||||
);
|
||||
runOnlyForDeploymentPostprocessing = 0;
|
||||
};
|
||||
/* End PBXSourcesBuildPhase section */
|
||||
|
||||
/* Begin XCBuildConfiguration section */
|
||||
@@ -257,6 +307,7 @@
|
||||
INFOPLIST_KEY_CFBundleDisplayName = "Punktfunkempfänger";
|
||||
INFOPLIST_KEY_LSApplicationCategoryType = "public.app-category.utilities";
|
||||
INFOPLIST_KEY_NSHumanReadableCopyright = "";
|
||||
INFOPLIST_KEY_NSLocalNetworkUsageDescription = "Punktfunk connects directly to your punktfunk host on the local network to stream video, audio, and input.";
|
||||
INFOPLIST_KEY_NSMicrophoneUsageDescription = "Your microphone is streamed to the connected punktfunk host, where it appears as a virtual microphone.";
|
||||
INFOPLIST_KEY_NSPrincipalClass = NSApplication;
|
||||
LD_RUNPATH_SEARCH_PATHS = (
|
||||
@@ -284,6 +335,7 @@
|
||||
INFOPLIST_KEY_CFBundleDisplayName = "Punktfunkempfänger";
|
||||
INFOPLIST_KEY_LSApplicationCategoryType = "public.app-category.utilities";
|
||||
INFOPLIST_KEY_NSHumanReadableCopyright = "";
|
||||
INFOPLIST_KEY_NSLocalNetworkUsageDescription = "Punktfunk connects directly to your punktfunk host on the local network to stream video, audio, and input.";
|
||||
INFOPLIST_KEY_NSMicrophoneUsageDescription = "Your microphone is streamed to the connected punktfunk host, where it appears as a virtual microphone.";
|
||||
INFOPLIST_KEY_NSPrincipalClass = NSApplication;
|
||||
LD_RUNPATH_SEARCH_PATHS = (
|
||||
@@ -298,9 +350,84 @@
|
||||
};
|
||||
name = Release;
|
||||
};
|
||||
BB0000000000000000000012 /* Debug */ = {
|
||||
isa = XCBuildConfiguration;
|
||||
buildSettings = {
|
||||
ASSETCATALOG_COMPILER_APPICON_NAME = punktfunk_Logo;
|
||||
CODE_SIGN_STYLE = Automatic;
|
||||
CURRENT_PROJECT_VERSION = 1;
|
||||
GENERATE_INFOPLIST_FILE = YES;
|
||||
INFOPLIST_KEY_CFBundleDisplayName = "Punktfunkempfänger";
|
||||
INFOPLIST_KEY_LSApplicationCategoryType = "public.app-category.utilities";
|
||||
INFOPLIST_KEY_NSLocalNetworkUsageDescription = "Punktfunk connects directly to your punktfunk host on the local network to stream video, audio, and input.";
|
||||
INFOPLIST_KEY_NSMicrophoneUsageDescription = "Your microphone is streamed to the connected punktfunk host, where it appears as a virtual microphone.";
|
||||
INFOPLIST_KEY_UIApplicationSceneManifest_Generation = YES;
|
||||
INFOPLIST_KEY_UIApplicationSupportsIndirectInputEvents = YES;
|
||||
INFOPLIST_KEY_UILaunchScreen_Generation = YES;
|
||||
INFOPLIST_KEY_UISupportedInterfaceOrientations_iPad = "UIInterfaceOrientationPortrait UIInterfaceOrientationPortraitUpsideDown UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight";
|
||||
INFOPLIST_KEY_UISupportedInterfaceOrientations_iPhone = "UIInterfaceOrientationPortrait UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight";
|
||||
IPHONEOS_DEPLOYMENT_TARGET = 17.0;
|
||||
LD_RUNPATH_SEARCH_PATHS = (
|
||||
"$(inherited)",
|
||||
"@executable_path/Frameworks",
|
||||
);
|
||||
MARKETING_VERSION = 0.1;
|
||||
PRODUCT_BUNDLE_IDENTIFIER = io.unom.punktfunk.ios;
|
||||
PRODUCT_NAME = "$(TARGET_NAME)";
|
||||
SDKROOT = iphoneos;
|
||||
SUPPORTED_PLATFORMS = "iphoneos iphonesimulator";
|
||||
SUPPORTS_MACCATALYST = NO;
|
||||
SWIFT_EMIT_LOC_STRINGS = YES;
|
||||
SWIFT_VERSION = 5.0;
|
||||
TARGETED_DEVICE_FAMILY = "1,2";
|
||||
};
|
||||
name = Debug;
|
||||
};
|
||||
BB0000000000000000000013 /* Release */ = {
|
||||
isa = XCBuildConfiguration;
|
||||
buildSettings = {
|
||||
ASSETCATALOG_COMPILER_APPICON_NAME = punktfunk_Logo;
|
||||
CODE_SIGN_STYLE = Automatic;
|
||||
CURRENT_PROJECT_VERSION = 1;
|
||||
GENERATE_INFOPLIST_FILE = YES;
|
||||
INFOPLIST_KEY_CFBundleDisplayName = "Punktfunkempfänger";
|
||||
INFOPLIST_KEY_LSApplicationCategoryType = "public.app-category.utilities";
|
||||
INFOPLIST_KEY_NSLocalNetworkUsageDescription = "Punktfunk connects directly to your punktfunk host on the local network to stream video, audio, and input.";
|
||||
INFOPLIST_KEY_NSMicrophoneUsageDescription = "Your microphone is streamed to the connected punktfunk host, where it appears as a virtual microphone.";
|
||||
INFOPLIST_KEY_UIApplicationSceneManifest_Generation = YES;
|
||||
INFOPLIST_KEY_UIApplicationSupportsIndirectInputEvents = YES;
|
||||
INFOPLIST_KEY_UILaunchScreen_Generation = YES;
|
||||
INFOPLIST_KEY_UISupportedInterfaceOrientations_iPad = "UIInterfaceOrientationPortrait UIInterfaceOrientationPortraitUpsideDown UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight";
|
||||
INFOPLIST_KEY_UISupportedInterfaceOrientations_iPhone = "UIInterfaceOrientationPortrait UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight";
|
||||
IPHONEOS_DEPLOYMENT_TARGET = 17.0;
|
||||
LD_RUNPATH_SEARCH_PATHS = (
|
||||
"$(inherited)",
|
||||
"@executable_path/Frameworks",
|
||||
);
|
||||
MARKETING_VERSION = 0.1;
|
||||
PRODUCT_BUNDLE_IDENTIFIER = io.unom.punktfunk.ios;
|
||||
PRODUCT_NAME = "$(TARGET_NAME)";
|
||||
SDKROOT = iphoneos;
|
||||
SUPPORTED_PLATFORMS = "iphoneos iphonesimulator";
|
||||
SUPPORTS_MACCATALYST = NO;
|
||||
SWIFT_EMIT_LOC_STRINGS = YES;
|
||||
SWIFT_VERSION = 5.0;
|
||||
TARGETED_DEVICE_FAMILY = "1,2";
|
||||
};
|
||||
name = Release;
|
||||
};
|
||||
/* End XCBuildConfiguration section */
|
||||
|
||||
/* Begin XCConfigurationList section */
|
||||
BB000000000000000000000A /* Build configuration list for PBXNativeTarget "Punktfunk-iOS" */ = {
|
||||
isa = XCConfigurationList;
|
||||
buildConfigurations = (
|
||||
BB0000000000000000000012 /* Debug */,
|
||||
BB0000000000000000000013 /* Release */,
|
||||
);
|
||||
defaultConfigurationIsVisible = 0;
|
||||
defaultConfigurationName = Release;
|
||||
};
|
||||
AA000000000000000000000A /* Build configuration list for PBXNativeTarget "Punktfunk" */ = {
|
||||
isa = XCConfigurationList;
|
||||
buildConfigurations = (
|
||||
@@ -333,6 +460,10 @@
|
||||
isa = XCSwiftPackageProductDependency;
|
||||
productName = PunktfunkKit;
|
||||
};
|
||||
BB0000000000000000000006 /* PunktfunkKit */ = {
|
||||
isa = XCSwiftPackageProductDependency;
|
||||
productName = PunktfunkKit;
|
||||
};
|
||||
/* End XCSwiftPackageProductDependency section */
|
||||
};
|
||||
rootObject = AA000000000000000000000D /* Project object */;
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<Scheme
|
||||
LastUpgradeVersion = "2650"
|
||||
version = "1.7">
|
||||
<BuildAction
|
||||
parallelizeBuildables = "YES"
|
||||
buildImplicitDependencies = "YES">
|
||||
<BuildActionEntries>
|
||||
<BuildActionEntry
|
||||
buildForTesting = "YES"
|
||||
buildForRunning = "YES"
|
||||
buildForProfiling = "YES"
|
||||
buildForArchiving = "YES"
|
||||
buildForAnalyzing = "YES">
|
||||
<BuildableReference
|
||||
BuildableIdentifier = "primary"
|
||||
BlueprintIdentifier = "BB0000000000000000000009"
|
||||
BuildableName = "Punktfunk-iOS.app"
|
||||
BlueprintName = "Punktfunk-iOS"
|
||||
ReferencedContainer = "container:Punktfunk.xcodeproj">
|
||||
</BuildableReference>
|
||||
</BuildActionEntry>
|
||||
</BuildActionEntries>
|
||||
</BuildAction>
|
||||
<TestAction
|
||||
buildConfiguration = "Debug"
|
||||
selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB"
|
||||
selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB"
|
||||
shouldUseLaunchSchemeArgsEnv = "YES"
|
||||
shouldAutocreateTestPlan = "YES">
|
||||
</TestAction>
|
||||
<LaunchAction
|
||||
buildConfiguration = "Debug"
|
||||
selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB"
|
||||
selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB"
|
||||
launchStyle = "0"
|
||||
useCustomWorkingDirectory = "NO"
|
||||
ignoresPersistentStateOnLaunch = "NO"
|
||||
debugDocumentVersioning = "YES"
|
||||
debugServiceExtension = "internal"
|
||||
allowLocationSimulation = "YES">
|
||||
<BuildableProductRunnable
|
||||
runnableDebuggingMode = "0">
|
||||
<BuildableReference
|
||||
BuildableIdentifier = "primary"
|
||||
BlueprintIdentifier = "BB0000000000000000000009"
|
||||
BuildableName = "Punktfunk-iOS.app"
|
||||
BlueprintName = "Punktfunk-iOS"
|
||||
ReferencedContainer = "container:Punktfunk.xcodeproj">
|
||||
</BuildableReference>
|
||||
</BuildableProductRunnable>
|
||||
</LaunchAction>
|
||||
<ProfileAction
|
||||
buildConfiguration = "Release"
|
||||
shouldUseLaunchSchemeArgsEnv = "YES"
|
||||
savedToolIdentifier = ""
|
||||
useCustomWorkingDirectory = "NO"
|
||||
debugDocumentVersioning = "YES">
|
||||
<BuildableProductRunnable
|
||||
runnableDebuggingMode = "0">
|
||||
<BuildableReference
|
||||
BuildableIdentifier = "primary"
|
||||
BlueprintIdentifier = "BB0000000000000000000009"
|
||||
BuildableName = "Punktfunk-iOS.app"
|
||||
BlueprintName = "Punktfunk-iOS"
|
||||
ReferencedContainer = "container:Punktfunk.xcodeproj">
|
||||
</BuildableReference>
|
||||
</BuildableProductRunnable>
|
||||
</ProfileAction>
|
||||
<AnalyzeAction
|
||||
buildConfiguration = "Debug">
|
||||
</AnalyzeAction>
|
||||
<ArchiveAction
|
||||
buildConfiguration = "Release"
|
||||
revealArchiveInOrganizer = "YES">
|
||||
</ArchiveAction>
|
||||
</Scheme>
|
||||
+26
-3
@@ -97,7 +97,10 @@ signing, bundle id `io.unom.punktfunk`. Notes:
|
||||
- **Tests from Xcode**: the package tests run with `swift test`; to get them on ⌘U, add
|
||||
`PunktfunkKitTests` once via Edit Scheme → Test → + (Xcode persists it into the shared
|
||||
scheme — a hand-written package-test reference doesn't resolve headlessly).
|
||||
- `xcodebuild -project Punktfunk.xcodeproj -scheme Punktfunk build` works headlessly.
|
||||
- `xcodebuild -project Punktfunk.xcodeproj -scheme Punktfunk build` works headlessly;
|
||||
same for `-scheme Punktfunk-iOS -destination 'generic/platform=iOS Simulator'` (run it
|
||||
in a simulator via `xcrun simctl install/launch` — `SIMCTL_CHILD_PUNKTFUNK_AUTOCONNECT=…`
|
||||
passes the dev autoconnect env through).
|
||||
|
||||
## Notes for whoever picks this up next
|
||||
|
||||
@@ -170,8 +173,28 @@ signing, bundle id `io.unom.punktfunk`. Notes:
|
||||
while the app has focus, and focus loss also auto-releases everything held. One live capture per process (the GC
|
||||
mouse/keyboard singletons have a single handler slot — ownership is tracked so a stale
|
||||
capture's stop() can't clobber a newer one).
|
||||
9. **iOS**: same package (`BUILD_IOS=1` for the xcframework slice); `StreamView` needs the
|
||||
`UIViewRepresentable` twin and touch→input mapping.
|
||||
9. **iOS/iPadOS — ported and first-lit** (iPad simulator ↔ the real host, 60 fps).
|
||||
`BUILD_IOS=1 bash scripts/build-xcframework.sh` builds device + universal-simulator
|
||||
slices; the Xcode project has a second target, **Punktfunk-iOS**, sharing the same
|
||||
synchronized sources. The iOS `StreamView` (StreamViewIOS.swift — same name/signature
|
||||
as the macOS one, so the SwiftUI shell is identical) hosts the shared `StreamPump` in
|
||||
a view controller for `prefersPointerLocked`: with a hardware mouse/trackpad that is
|
||||
the iPadOS cursor capture (system honors it fullscreen-and-frontmost; in Stage
|
||||
Manager it degrades to both-cursors forwarding). Touch is always forwarded — every
|
||||
finger gets a wire touch id and coordinates map through the aspect-fit letterbox
|
||||
into host-mode pixels (surface == host mode, so the host rescale is the identity).
|
||||
`InputCapture` is cross-platform (GC works the same on iPadOS; ⌘⎋ is detected from
|
||||
the HID stream there); audio routes via `AVAudioSession` (the Settings device
|
||||
pickers are macOS-only). For the iPad-with-external-display setup: the target
|
||||
enables multiple scenes + indirect input events — on Stage Manager iPads, drag the
|
||||
punktfunk window onto the external screen and the stream runs there with full
|
||||
keyboard/mouse/touch. Known gaps: `prefersPointerLocked` is declared on the stream
|
||||
view controller but UIHostingController doesn't forward the preference from
|
||||
representable children, so the system cursor stays visible (relative-mouse
|
||||
forwarding works regardless — fixing it means putting the controller into the UIKit
|
||||
presentation chain, e.g. a full-screen UIKit presentation on session start); and
|
||||
AVAudioSession interruptions (calls, Siri) don't auto-restart the audio engines yet
|
||||
(reconnect recovers).
|
||||
|
||||
## Known limitations of the current host (relevant to client UX)
|
||||
|
||||
|
||||
@@ -36,7 +36,9 @@ struct AddHostSheet: View {
|
||||
}
|
||||
.padding(16)
|
||||
}
|
||||
#if os(macOS)
|
||||
.frame(width: 380)
|
||||
.fixedSize(horizontal: false, vertical: true)
|
||||
#endif
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,7 +8,9 @@
|
||||
// the only way into hosts running --require-pairing. Once pinned, reconnects are silent
|
||||
// and a changed host identity refuses to connect.
|
||||
|
||||
#if os(macOS)
|
||||
import AppKit
|
||||
#endif
|
||||
import PunktfunkKit
|
||||
import SwiftUI
|
||||
|
||||
@@ -21,6 +23,9 @@ struct ContentView: View {
|
||||
@AppStorage("punktfunk.compositor") private var compositor = 0
|
||||
@State private var showAddHost = false
|
||||
@State private var pairingTarget: StoredHost?
|
||||
#if os(iOS)
|
||||
@State private var showSettings = false
|
||||
#endif
|
||||
|
||||
var body: some View {
|
||||
Group {
|
||||
@@ -70,7 +75,9 @@ struct ContentView: View {
|
||||
trustCard(fp)
|
||||
}
|
||||
}
|
||||
#if os(macOS)
|
||||
.frame(minWidth: 640, minHeight: 360)
|
||||
#endif
|
||||
.background(Color.black)
|
||||
}
|
||||
|
||||
@@ -106,17 +113,38 @@ struct ContentView: View {
|
||||
.help("Add a host")
|
||||
}
|
||||
ToolbarItem {
|
||||
#if os(macOS)
|
||||
SettingsLink {
|
||||
Label("Settings", systemImage: "gearshape")
|
||||
}
|
||||
.help("Stream mode and settings")
|
||||
#else
|
||||
Button {
|
||||
showSettings = true
|
||||
} label: {
|
||||
Label("Settings", systemImage: "gearshape")
|
||||
}
|
||||
#endif
|
||||
}
|
||||
}
|
||||
}
|
||||
#if os(macOS)
|
||||
.frame(minWidth: 480, minHeight: 360)
|
||||
#endif
|
||||
.sheet(isPresented: $showAddHost) {
|
||||
AddHostSheet { store.add($0) }
|
||||
}
|
||||
#if os(iOS)
|
||||
.sheet(isPresented: $showSettings) {
|
||||
NavigationStack {
|
||||
SettingsView()
|
||||
.navigationTitle("Settings")
|
||||
.toolbar {
|
||||
Button("Done") { showSettings = false }
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif
|
||||
.alert(
|
||||
"Connection failed",
|
||||
isPresented: Binding(
|
||||
@@ -239,7 +267,11 @@ struct ContentView: View {
|
||||
model.rejectTrust()
|
||||
pairingTarget = host
|
||||
}
|
||||
#if os(macOS)
|
||||
.buttonStyle(.link)
|
||||
#else
|
||||
.buttonStyle(.borderless)
|
||||
#endif
|
||||
.font(.callout)
|
||||
}
|
||||
.padding(28)
|
||||
@@ -287,11 +319,20 @@ struct ContentView: View {
|
||||
.font(.system(.caption, design: .monospaced))
|
||||
// While captured the cursor is hidden+frozen, so the button is keyboard-only
|
||||
// (⌘⎋ or Cmd+Tab release the cursor; released, it's clickable again).
|
||||
#if os(macOS)
|
||||
Text(model.mouseCaptured
|
||||
? "⌘⎋ releases the mouse"
|
||||
: "Click the stream to capture input")
|
||||
.font(.caption2)
|
||||
.opacity(0.8)
|
||||
#else
|
||||
// Touch always plays directly; ⌘⎋ (hardware keyboard) toggles kb/mouse.
|
||||
Text(model.mouseCaptured
|
||||
? "⌘⎋ releases keyboard & mouse"
|
||||
: "⌘⎋ captures keyboard & mouse")
|
||||
.font(.caption2)
|
||||
.opacity(0.8)
|
||||
#endif
|
||||
Button("Disconnect (⌘D)") { model.disconnect() }
|
||||
.font(.caption)
|
||||
.keyboardShortcut("d", modifiers: .command)
|
||||
|
||||
@@ -25,7 +25,11 @@ struct PairSheet: View {
|
||||
let onPaired: (Data) -> Void
|
||||
|
||||
@State private var pin = ""
|
||||
#if os(macOS)
|
||||
@State private var clientName = Host.current().localizedName ?? "Mac"
|
||||
#else
|
||||
@State private var clientName = UIDevice.current.name
|
||||
#endif
|
||||
@State private var busy = false
|
||||
@State private var errorText: String?
|
||||
@State private var token = CeremonyToken()
|
||||
@@ -77,8 +81,10 @@ struct PairSheet: View {
|
||||
}
|
||||
.padding(16)
|
||||
}
|
||||
#if os(macOS)
|
||||
.frame(width: 400)
|
||||
.fixedSize(horizontal: false, vertical: true)
|
||||
#endif
|
||||
.interactiveDismissDisabled(busy)
|
||||
.onDisappear { token.cancelled = true } // any other dismissal path
|
||||
}
|
||||
|
||||
@@ -1,23 +1,30 @@
|
||||
// PunktfunkClient — the macOS client app (also runs unbundled via swift run).
|
||||
// Hosts grid → trust-on-first-use → StreamView (AVSampleBufferDisplayLayer HEVC) + input.
|
||||
|
||||
#if os(macOS)
|
||||
import AppKit
|
||||
#endif
|
||||
import SwiftUI
|
||||
|
||||
@main
|
||||
struct PunktfunkClientApp: App {
|
||||
#if os(macOS)
|
||||
@NSApplicationDelegateAdaptor(AppDelegate.self) private var appDelegate
|
||||
#endif
|
||||
|
||||
var body: some Scene {
|
||||
WindowGroup("punktfunk") {
|
||||
ContentView()
|
||||
}
|
||||
#if os(macOS)
|
||||
Settings {
|
||||
SettingsView()
|
||||
}
|
||||
#endif
|
||||
}
|
||||
}
|
||||
|
||||
#if os(macOS)
|
||||
final class AppDelegate: NSObject, NSApplicationDelegate {
|
||||
func applicationDidFinishLaunching(_ notification: Notification) {
|
||||
// `swift run` launches an unbundled binary; promote it to a regular app so the
|
||||
@@ -30,3 +37,4 @@ final class AppDelegate: NSObject, NSApplicationDelegate {
|
||||
true
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
@@ -2,7 +2,9 @@
|
||||
// virtual output at exactly this size/refresh — there is no scaling anywhere in the
|
||||
// pipeline.
|
||||
|
||||
#if os(macOS)
|
||||
import AppKit
|
||||
#endif
|
||||
import PunktfunkKit
|
||||
import SwiftUI
|
||||
|
||||
@@ -11,11 +13,13 @@ struct SettingsView: View {
|
||||
@AppStorage("punktfunk.height") private var height = 1080
|
||||
@AppStorage("punktfunk.hz") private var hz = 60
|
||||
@AppStorage("punktfunk.compositor") private var compositor = 0
|
||||
@AppStorage("punktfunk.micEnabled") private var micEnabled = true
|
||||
#if os(macOS)
|
||||
@AppStorage("punktfunk.speakerUID") private var speakerUID = ""
|
||||
@AppStorage("punktfunk.micUID") private var micUID = ""
|
||||
@AppStorage("punktfunk.micEnabled") private var micEnabled = true
|
||||
@State private var outputDevices: [AudioDevice] = []
|
||||
@State private var inputDevices: [AudioDevice] = []
|
||||
#endif
|
||||
|
||||
var body: some View {
|
||||
Form {
|
||||
@@ -39,6 +43,7 @@ struct SettingsView: View {
|
||||
.foregroundStyle(.secondary)
|
||||
}
|
||||
Section {
|
||||
#if os(macOS)
|
||||
Picker("Speaker", selection: $speakerUID) {
|
||||
Text("System default").tag("")
|
||||
ForEach(outputDevices) { device in
|
||||
@@ -49,7 +54,9 @@ struct SettingsView: View {
|
||||
Text("Unavailable device").tag(speakerUID)
|
||||
}
|
||||
}
|
||||
#endif
|
||||
Toggle("Send microphone to the host", isOn: $micEnabled)
|
||||
#if os(macOS)
|
||||
Picker("Microphone", selection: $micUID) {
|
||||
Text("System default").tag("")
|
||||
ForEach(inputDevices) { device in
|
||||
@@ -61,6 +68,7 @@ struct SettingsView: View {
|
||||
}
|
||||
}
|
||||
.disabled(!micEnabled)
|
||||
#endif
|
||||
} header: {
|
||||
Text("Audio")
|
||||
} footer: {
|
||||
@@ -89,19 +97,29 @@ struct SettingsView: View {
|
||||
}
|
||||
}
|
||||
.formStyle(.grouped)
|
||||
#if os(macOS)
|
||||
.frame(width: 380)
|
||||
.fixedSize()
|
||||
.onAppear {
|
||||
outputDevices = AudioDevices.outputs()
|
||||
inputDevices = AudioDevices.inputs()
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
private func fillFromMainScreen() {
|
||||
#if os(macOS)
|
||||
guard let screen = NSScreen.main else { return }
|
||||
let scale = screen.backingScaleFactor
|
||||
width = Int(screen.frame.width * scale)
|
||||
height = Int(screen.frame.height * scale)
|
||||
hz = screen.maximumFramesPerSecond
|
||||
#else
|
||||
// nativeBounds is portrait-oriented pixels — streams are landscape.
|
||||
let bounds = UIScreen.main.nativeBounds
|
||||
width = Int(max(bounds.width, bounds.height))
|
||||
height = Int(min(bounds.width, bounds.height))
|
||||
hz = UIScreen.main.maximumFramesPerSecond
|
||||
#endif
|
||||
}
|
||||
}
|
||||
|
||||
@@ -25,6 +25,10 @@
|
||||
|
||||
#if os(macOS)
|
||||
import AppKit
|
||||
#endif
|
||||
#if os(iOS)
|
||||
import UIKit
|
||||
#endif
|
||||
import Foundation
|
||||
import GameController
|
||||
import PunktfunkCore
|
||||
@@ -36,7 +40,9 @@ public final class InputCapture {
|
||||
private var observers: [NSObjectProtocol] = []
|
||||
private var mice: [GCMouse] = []
|
||||
private var keyboards: [GCKeyboard] = []
|
||||
#if os(macOS)
|
||||
private var keyEventMonitor: Any?
|
||||
#endif
|
||||
|
||||
// Main-queue-only state (see header comment).
|
||||
private var residualX: Float = 0
|
||||
@@ -53,6 +59,9 @@ public final class InputCapture {
|
||||
/// reaches GCKeyboard, racing the NSEvent monitor — latched here so it can't type
|
||||
/// an Escape into the host in either toggle direction.
|
||||
private var suppressedVK: UInt32?
|
||||
/// Physical ⌘ keys currently held (tracked even while released — the ⌘⎋ toggle and
|
||||
/// its Esc suppression need it in both states).
|
||||
private var cmdKeysDown: Set<UInt32> = []
|
||||
|
||||
/// While true, mouse/keyboard flow to the host and key NSEvents are swallowed
|
||||
/// locally; while false the user is interacting with the local UI (dragging the
|
||||
@@ -119,8 +128,13 @@ public final class InputCapture {
|
||||
if let k = n.object as? GCKeyboard { self?.attach(keyboard: k) }
|
||||
})
|
||||
// Focus loss: GC stops delivering, so release everything still held host-side.
|
||||
#if os(macOS)
|
||||
let resignActive = NSApplication.didResignActiveNotification
|
||||
#else
|
||||
let resignActive = UIApplication.willResignActiveNotification
|
||||
#endif
|
||||
observers.append(NotificationCenter.default.addObserver(
|
||||
forName: NSApplication.didResignActiveNotification, object: nil, queue: .main
|
||||
forName: resignActive, object: nil, queue: .main
|
||||
) { [weak self] _ in
|
||||
self?.releaseAll()
|
||||
})
|
||||
@@ -128,6 +142,8 @@ public final class InputCapture {
|
||||
// that one combo is intercepted: swallowing keys wholesale at the monitor level
|
||||
// risks starving GC's own delivery, so the no-beep behavior lives in
|
||||
// StreamLayerView (first responder consumes keyDown/keyUp while captured).
|
||||
// (On iOS there is no NSEvent monitor — the GC key handler detects the combo.)
|
||||
#if os(macOS)
|
||||
keyEventMonitor = NSEvent.addLocalMonitorForEvents(
|
||||
matching: [.keyDown]
|
||||
) { [weak self] event in
|
||||
@@ -140,16 +156,19 @@ public final class InputCapture {
|
||||
}
|
||||
return event
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
public func stop() {
|
||||
releaseAll()
|
||||
observers.forEach(NotificationCenter.default.removeObserver(_:))
|
||||
observers.removeAll()
|
||||
#if os(macOS)
|
||||
if let monitor = keyEventMonitor {
|
||||
NSEvent.removeMonitor(monitor)
|
||||
keyEventMonitor = nil
|
||||
}
|
||||
#endif
|
||||
// Don't clobber the handlers if a newer capture has taken the global devices.
|
||||
if Self.activeCapture === self || Self.activeCapture == nil {
|
||||
for mouse in mice {
|
||||
@@ -172,8 +191,12 @@ public final class InputCapture {
|
||||
|
||||
deinit { stop() }
|
||||
|
||||
/// Send release events for everything currently held, and drop the motion residuals.
|
||||
/// Send release events for everything currently held, and drop the motion residuals
|
||||
/// and modifier/latch tracking (GC delivers nothing while inactive, so a ⌘ released
|
||||
/// in another app would otherwise stay "held" here forever — hijacking Esc).
|
||||
private func releaseAll() {
|
||||
cmdKeysDown.removeAll()
|
||||
suppressedVK = nil
|
||||
for vk in pressedVKs {
|
||||
connection.send(.key(vk, down: false))
|
||||
}
|
||||
@@ -264,16 +287,32 @@ public final class InputCapture {
|
||||
keyboards.append(keyboard)
|
||||
keyboard.keyboardInput?.keyChangedHandler = { [weak self] _, _, keyCode, pressed in
|
||||
guard let self, let vk = Self.hidToVK[keyCode.rawValue] else { return }
|
||||
if vk == 0x5B || vk == 0x5C { // physical ⌘ state, tracked in both states
|
||||
if pressed {
|
||||
self.cmdKeysDown.insert(vk)
|
||||
} else {
|
||||
self.cmdKeysDown.remove(vk)
|
||||
}
|
||||
}
|
||||
// The ⌘⎋ toggle's Esc — checked before the forwarding gate, because in the
|
||||
// engage direction forwarding is already true when this fires.
|
||||
if vk == self.suppressedVK {
|
||||
if !pressed { self.suppressedVK = nil }
|
||||
return
|
||||
}
|
||||
#if os(iOS)
|
||||
// No NSEvent monitor here — the toggle combo is detected from the HID
|
||||
// stream itself.
|
||||
if pressed, vk == 0x1B, !self.cmdKeysDown.isEmpty {
|
||||
self.suppressedVK = 0x1B
|
||||
self.onToggleCapture?()
|
||||
return
|
||||
}
|
||||
#endif
|
||||
guard self.forwarding else { return }
|
||||
// Release direction of the toggle: GC's Esc-down can beat the NSEvent
|
||||
// monitor — never type Esc into the host while ⌘ is held (⌘⎋ is reserved).
|
||||
if vk == 0x1B, self.pressedVKs.contains(0x5B) || self.pressedVKs.contains(0x5C) {
|
||||
if vk == 0x1B, !self.cmdKeysDown.isEmpty {
|
||||
return
|
||||
}
|
||||
if pressed {
|
||||
@@ -325,4 +364,3 @@ public final class InputCapture {
|
||||
return m
|
||||
}()
|
||||
}
|
||||
#endif
|
||||
|
||||
@@ -14,7 +14,6 @@
|
||||
// AVAudioEngine ties input+output to one aggregate clock, separate engines keep
|
||||
// arbitrary mic/speaker combinations trivial.
|
||||
|
||||
#if os(macOS)
|
||||
import AVFoundation
|
||||
import os
|
||||
|
||||
@@ -140,9 +139,29 @@ public final class SessionAudio {
|
||||
}
|
||||
|
||||
/// Start playback (and, if enabled+authorized, the mic uplink). Empty UIDs = system
|
||||
/// default device. Main thread (engine setup); returns after the engines start —
|
||||
/// the mic may start slightly later if the permission prompt is pending.
|
||||
/// default device; on iOS the UIDs are ignored entirely (routes are
|
||||
/// AVAudioSession-managed). Main thread (engine setup); returns after the engines
|
||||
/// start — the mic may start slightly later if the permission prompt is pending.
|
||||
public func start(speakerUID: String, micUID: String, micEnabled: Bool) {
|
||||
#if os(iOS)
|
||||
// Route + policy live in the session, not per-engine: stereo playback, mic
|
||||
// capture when enabled, Bluetooth allowed. Failure is non-fatal (defaults).
|
||||
let session = AVAudioSession.sharedInstance()
|
||||
do {
|
||||
if micEnabled {
|
||||
// .defaultToSpeaker: .playAndRecord otherwise routes to the iPhone
|
||||
// EARPIECE; only affects the built-in route (headphones/BT still win).
|
||||
try session.setCategory(
|
||||
.playAndRecord, mode: .default,
|
||||
options: [.allowBluetoothA2DP, .defaultToSpeaker])
|
||||
} else {
|
||||
try session.setCategory(.playback, mode: .default)
|
||||
}
|
||||
try session.setActive(true)
|
||||
} catch {
|
||||
log.warning("AVAudioSession setup failed: \(error.localizedDescription)")
|
||||
}
|
||||
#endif
|
||||
startPlayback(speakerUID: speakerUID)
|
||||
guard micEnabled else { return }
|
||||
switch AVCaptureDevice.authorizationStatus(for: .audio) {
|
||||
@@ -180,6 +199,16 @@ public final class SessionAudio {
|
||||
if wasDraining {
|
||||
_ = drainDone.wait(timeout: .now() + .milliseconds(400))
|
||||
}
|
||||
#if os(iOS)
|
||||
// Release the session so audio we interrupted (Music, podcasts) gets its
|
||||
// resume cue.
|
||||
do {
|
||||
try AVAudioSession.sharedInstance().setActive(
|
||||
false, options: .notifyOthersOnDeactivation)
|
||||
} catch {
|
||||
log.warning("AVAudioSession deactivation failed: \(error.localizedDescription)")
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
// MARK: - Playback (host → speaker)
|
||||
@@ -190,6 +219,7 @@ public final class SessionAudio {
|
||||
let ring = AudioRing(capacity: 96_000, prefill: 1920)
|
||||
|
||||
let engine = AVAudioEngine()
|
||||
#if os(macOS)
|
||||
if !speakerUID.isEmpty {
|
||||
if let dev = AudioDevices.deviceID(forUID: speakerUID),
|
||||
let unit = engine.outputNode.audioUnit {
|
||||
@@ -200,6 +230,7 @@ public final class SessionAudio {
|
||||
log.warning("speaker \(speakerUID) not present — using default")
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
// Engine-native deinterleaved float; the render block deinterleaves from the ring.
|
||||
guard let format = AVAudioFormat(standardFormatWithSampleRate: 48_000, channels: 2)
|
||||
@@ -282,6 +313,7 @@ public final class SessionAudio {
|
||||
private func startCapture(micUID: String) {
|
||||
let engine = AVAudioEngine()
|
||||
let input = engine.inputNode
|
||||
#if os(macOS)
|
||||
if !micUID.isEmpty {
|
||||
if let dev = AudioDevices.deviceID(forUID: micUID), let unit = input.audioUnit {
|
||||
if !Self.setDevice(dev, on: unit) {
|
||||
@@ -291,6 +323,7 @@ public final class SessionAudio {
|
||||
log.warning("microphone \(micUID) not present — using default")
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
let inFormat = input.outputFormat(forBus: 0)
|
||||
guard inFormat.sampleRate > 0, inFormat.channelCount > 0 else {
|
||||
@@ -376,11 +409,12 @@ public final class SessionAudio {
|
||||
log.info("mic uplink started (\(micUID.isEmpty ? "default input" : micUID))")
|
||||
}
|
||||
|
||||
#if os(macOS)
|
||||
private static func setDevice(_ id: AudioDeviceID, on unit: AudioUnit) -> Bool {
|
||||
var dev = id
|
||||
return AudioUnitSetProperty(
|
||||
unit, kAudioOutputUnitProperty_CurrentDevice, kAudioUnitScope_Global, 0,
|
||||
&dev, UInt32(MemoryLayout<AudioDeviceID>.size)) == noErr
|
||||
}
|
||||
#endif
|
||||
}
|
||||
#endif
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
// The platform-independent heart of the presenters: one thread pulling AUs from the
|
||||
// connection into an AVSampleBufferDisplayLayer, with the format description refreshed
|
||||
// on every IDR (the host opens with an IDR carrying in-band parameter sets; recovery
|
||||
// keyframes re-send them — there is no out-of-band extradata, ever). Shared by the
|
||||
// macOS StreamLayerView and the iOS/iPadOS stream view.
|
||||
|
||||
import AVFoundation
|
||||
import Foundation
|
||||
|
||||
/// Cancellation handle owned by exactly one pump thread — a restart hands the old pump
|
||||
/// its own token, so it can never be revived by a newer start().
|
||||
private final class PumpToken: @unchecked Sendable {
|
||||
private let lock = NSLock()
|
||||
private var live = true
|
||||
var isLive: Bool {
|
||||
lock.lock()
|
||||
defer { lock.unlock() }
|
||||
return live
|
||||
}
|
||||
func cancel() {
|
||||
lock.lock()
|
||||
live = false
|
||||
lock.unlock()
|
||||
}
|
||||
}
|
||||
|
||||
/// One pump per instance; create a fresh StreamPump per start (cancel is permanent).
|
||||
final class StreamPump {
|
||||
private let token = PumpToken()
|
||||
|
||||
/// Pump thread: pull AUs, wrap, enqueue. Non-IDR AUs before the first format
|
||||
/// description are dropped. `onFrame`/`onSessionEnd` fire on the pump thread.
|
||||
func start(
|
||||
connection: PunktfunkConnection,
|
||||
layer: AVSampleBufferDisplayLayer,
|
||||
onFrame: (@Sendable (AccessUnit) -> Void)?,
|
||||
onSessionEnd: (@Sendable () -> Void)?
|
||||
) {
|
||||
let token = token
|
||||
layer.flush() // drop any frames a previous connection left queued
|
||||
|
||||
let thread = Thread {
|
||||
var format: CMVideoFormatDescription?
|
||||
while token.isLive {
|
||||
do {
|
||||
guard let au = try connection.nextAU(timeoutMs: 100) else { continue }
|
||||
onFrame?(au)
|
||||
if let f = AnnexB.formatDescription(fromIDR: au.data) {
|
||||
format = f // refreshed on every IDR (mode changes included)
|
||||
}
|
||||
if layer.status == .failed {
|
||||
// Decode wedged: flush and re-gate on the next in-band parameter
|
||||
// sets — resuming with a delta frame can't recover. (A
|
||||
// request-IDR channel on punktfunk/1 is a host-side TODO; with the
|
||||
// host's infinite GOP this may otherwise stay black until the
|
||||
// next recovery keyframe.)
|
||||
layer.flush()
|
||||
format = AnnexB.formatDescription(fromIDR: au.data)
|
||||
}
|
||||
guard let f = format,
|
||||
let sample = AnnexB.sampleBuffer(au: au, format: f),
|
||||
token.isLive // don't enqueue a stale frame after a restart
|
||||
else { continue }
|
||||
layer.enqueue(sample)
|
||||
} catch {
|
||||
if token.isLive {
|
||||
onSessionEnd?()
|
||||
}
|
||||
break // session closed
|
||||
}
|
||||
}
|
||||
}
|
||||
thread.name = "punktfunk-pump"
|
||||
thread.qualityOfService = .userInteractive
|
||||
thread.start()
|
||||
}
|
||||
|
||||
/// Stop pumping (≤ one poll timeout). Does not close the connection.
|
||||
func stop() {
|
||||
token.cancel()
|
||||
}
|
||||
|
||||
deinit { token.cancel() }
|
||||
}
|
||||
@@ -99,25 +99,8 @@ public struct StreamView: NSViewRepresentable {
|
||||
}
|
||||
|
||||
public final class StreamLayerView: NSView {
|
||||
/// Cancellation handle owned by exactly one pump thread — a restart hands the old pump
|
||||
/// its own token, so it can never be revived by a newer start().
|
||||
private final class PumpToken: @unchecked Sendable {
|
||||
private let lock = NSLock()
|
||||
private var live = true
|
||||
var isLive: Bool {
|
||||
lock.lock()
|
||||
defer { lock.unlock() }
|
||||
return live
|
||||
}
|
||||
func cancel() {
|
||||
lock.lock()
|
||||
live = false
|
||||
lock.unlock()
|
||||
}
|
||||
}
|
||||
|
||||
private let displayLayer = AVSampleBufferDisplayLayer()
|
||||
private var token: PumpToken?
|
||||
private var pump: StreamPump?
|
||||
public private(set) var connection: PunktfunkConnection?
|
||||
private let cursorCapture = CursorCapture()
|
||||
private var inputCapture: InputCapture?
|
||||
@@ -261,7 +244,7 @@ public final class StreamLayerView: NSView {
|
||||
// A click is explicit intent AND may arrive mid-activation (acceptsFirstMouse:
|
||||
// NSApp.isActive / isKeyWindow are still false for the click coming in from
|
||||
// another app) — only the auto-engage paths require already-held key status.
|
||||
guard captureEnabled, !captured, token != nil, window != nil,
|
||||
guard captureEnabled, !captured, pump != nil, window != nil,
|
||||
fromClick || (NSApp.isActive && window?.isKeyWindow == true)
|
||||
else { return }
|
||||
cursorCapture.capture(in: self)
|
||||
@@ -297,11 +280,7 @@ public final class StreamLayerView: NSView {
|
||||
onSessionEnd: (@Sendable () -> Void)? = nil
|
||||
) {
|
||||
stop()
|
||||
let token = PumpToken()
|
||||
self.token = token
|
||||
self.connection = connection
|
||||
let layer = displayLayer
|
||||
layer.flush() // drop any frames a previous connection left queued
|
||||
|
||||
// The view owns the session's input capture: handlers attach now, but nothing is
|
||||
// forwarded until capture engages (captureEnabled + auto-engage or a click).
|
||||
@@ -324,40 +303,11 @@ public final class StreamLayerView: NSView {
|
||||
capture.start()
|
||||
inputCapture = capture
|
||||
|
||||
let thread = Thread {
|
||||
var format: CMVideoFormatDescription?
|
||||
while token.isLive {
|
||||
do {
|
||||
guard let au = try connection.nextAU(timeoutMs: 100) else { continue }
|
||||
onFrame?(au)
|
||||
if let f = AnnexB.formatDescription(fromIDR: au.data) {
|
||||
format = f // refreshed on every IDR (mode changes included)
|
||||
}
|
||||
if layer.status == .failed {
|
||||
// Decode wedged: flush and re-gate on the next in-band parameter
|
||||
// sets — resuming with a delta frame can't recover. (A
|
||||
// request-IDR channel on punktfunk/1 is a host-side TODO; with the
|
||||
// host's infinite GOP this may otherwise stay black until the
|
||||
// next recovery keyframe.)
|
||||
layer.flush()
|
||||
format = AnnexB.formatDescription(fromIDR: au.data)
|
||||
}
|
||||
guard let f = format,
|
||||
let sample = AnnexB.sampleBuffer(au: au, format: f),
|
||||
token.isLive // don't enqueue a stale frame after a restart
|
||||
else { continue }
|
||||
layer.enqueue(sample)
|
||||
} catch {
|
||||
if token.isLive {
|
||||
onSessionEnd?()
|
||||
}
|
||||
break // session closed
|
||||
}
|
||||
}
|
||||
}
|
||||
thread.name = "punktfunk-pump"
|
||||
thread.qualityOfService = .userInteractive
|
||||
thread.start()
|
||||
let pump = StreamPump()
|
||||
pump.start(
|
||||
connection: connection, layer: displayLayer,
|
||||
onFrame: onFrame, onSessionEnd: onSessionEnd)
|
||||
self.pump = pump
|
||||
requestAutoCapture() // entering a session is the deliberate "capture me" moment
|
||||
}
|
||||
|
||||
@@ -367,15 +317,15 @@ public final class StreamLayerView: NSView {
|
||||
releaseCapture()
|
||||
inputCapture?.stop()
|
||||
inputCapture = nil
|
||||
token?.cancel()
|
||||
token = nil
|
||||
pump?.stop()
|
||||
pump = nil
|
||||
connection = nil
|
||||
}
|
||||
|
||||
deinit {
|
||||
appObservers.forEach(NotificationCenter.default.removeObserver(_:))
|
||||
windowObservers.forEach(NotificationCenter.default.removeObserver(_:))
|
||||
token?.cancel()
|
||||
pump?.stop()
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
@@ -0,0 +1,272 @@
|
||||
// iOS/iPadOS presenter: the same AVSampleBufferDisplayLayer + StreamPump as macOS,
|
||||
// hosted in a UIViewController so the scene can pointer-lock (the iPadOS equivalent of
|
||||
// the Mac's cursor capture — with a hardware mouse/trackpad the system cursor is hidden
|
||||
// and GCMouse's raw deltas drive the host cursor alone; the system only honors the lock
|
||||
// fullscreen-and-frontmost, so in Stage Manager it degrades to Mac-style "both cursors
|
||||
// visible" forwarding).
|
||||
//
|
||||
// Touch is the primary input and is always forwarded (touching the video IS explicit
|
||||
// intent): every finger maps to a wire touch id, coordinates are mapped through the
|
||||
// aspect-fit letterbox into host-mode pixels, so surface == host mode and the host's
|
||||
// rescale is the identity. Hardware keyboard/mouse forwarding shares InputCapture with
|
||||
// macOS — auto-engaged when streaming starts, ⌘⎋ toggles (detected from the HID stream;
|
||||
// there is no NSEvent monitor here).
|
||||
//
|
||||
// The public type is named StreamView like its macOS twin (each is platform-gated), so
|
||||
// the SwiftUI app layer is identical on both platforms.
|
||||
|
||||
#if os(iOS)
|
||||
import AVFoundation
|
||||
import GameController
|
||||
import PunktfunkCore
|
||||
import SwiftUI
|
||||
import UIKit
|
||||
|
||||
public struct StreamView: UIViewControllerRepresentable {
|
||||
private let connection: PunktfunkConnection
|
||||
private let captureEnabled: Bool
|
||||
private let onCaptureChange: ((Bool) -> Void)?
|
||||
private let onFrame: (@Sendable (AccessUnit) -> Void)?
|
||||
private let onSessionEnd: (@Sendable () -> Void)?
|
||||
|
||||
public init(
|
||||
connection: PunktfunkConnection,
|
||||
captureEnabled: Bool = true,
|
||||
onCaptureChange: ((Bool) -> Void)? = nil,
|
||||
onFrame: (@Sendable (AccessUnit) -> Void)? = nil,
|
||||
onSessionEnd: (@Sendable () -> Void)? = nil
|
||||
) {
|
||||
self.connection = connection
|
||||
self.captureEnabled = captureEnabled
|
||||
self.onCaptureChange = onCaptureChange
|
||||
self.onFrame = onFrame
|
||||
self.onSessionEnd = onSessionEnd
|
||||
}
|
||||
|
||||
public func makeUIViewController(context: Context) -> StreamViewController {
|
||||
let controller = StreamViewController()
|
||||
controller.onCaptureChange = onCaptureChange
|
||||
controller.captureEnabled = captureEnabled
|
||||
controller.start(connection: connection, onFrame: onFrame, onSessionEnd: onSessionEnd)
|
||||
return controller
|
||||
}
|
||||
|
||||
public func updateUIViewController(_ controller: StreamViewController, context: Context) {
|
||||
controller.onCaptureChange = onCaptureChange
|
||||
controller.captureEnabled = captureEnabled
|
||||
if controller.connection !== connection {
|
||||
controller.start(connection: connection, onFrame: onFrame, onSessionEnd: onSessionEnd)
|
||||
}
|
||||
}
|
||||
|
||||
public static func dismantleUIViewController(
|
||||
_ controller: StreamViewController, coordinator: ()
|
||||
) {
|
||||
controller.stop()
|
||||
}
|
||||
}
|
||||
|
||||
public final class StreamViewController: UIViewController {
|
||||
public private(set) var connection: PunktfunkConnection?
|
||||
private var pump: StreamPump?
|
||||
private var inputCapture: InputCapture?
|
||||
private var captured = false
|
||||
private var observers: [NSObjectProtocol] = []
|
||||
|
||||
var onCaptureChange: ((Bool) -> Void)?
|
||||
|
||||
var captureEnabled = true {
|
||||
didSet {
|
||||
guard captureEnabled != oldValue else { return }
|
||||
setCaptured(captureEnabled)
|
||||
}
|
||||
}
|
||||
|
||||
private var streamView: StreamLayerUIView {
|
||||
// swiftlint:disable:next force_cast
|
||||
view as! StreamLayerUIView
|
||||
}
|
||||
|
||||
public override func loadView() {
|
||||
view = StreamLayerUIView()
|
||||
}
|
||||
|
||||
public override var prefersPointerLocked: Bool { captured }
|
||||
public override var prefersHomeIndicatorAutoHidden: Bool { true }
|
||||
|
||||
func start(
|
||||
connection: PunktfunkConnection,
|
||||
onFrame: (@Sendable (AccessUnit) -> Void)?,
|
||||
onSessionEnd: (@Sendable () -> Void)?
|
||||
) {
|
||||
stop()
|
||||
self.connection = connection
|
||||
loadViewIfNeeded()
|
||||
// Read the LIVE mode per touch batch — an accepted requestMode() mid-stream
|
||||
// changes the letterbox, and touches must follow it.
|
||||
streamView.currentHostMode = { [weak connection] in
|
||||
guard let connection else { return .zero }
|
||||
let mode = connection.currentMode()
|
||||
return CGSize(width: Double(mode.width), height: Double(mode.height))
|
||||
}
|
||||
streamView.onTouchEvent = { [weak connection] event in
|
||||
connection?.send(event)
|
||||
}
|
||||
|
||||
let capture = InputCapture(connection: connection)
|
||||
capture.onToggleCapture = { [weak self] in
|
||||
guard let self else { return }
|
||||
self.setCaptured(!self.captured)
|
||||
}
|
||||
capture.onPreempted = { [weak self] in
|
||||
self?.setCaptured(false)
|
||||
}
|
||||
capture.start()
|
||||
inputCapture = capture
|
||||
|
||||
let pump = StreamPump()
|
||||
pump.start(
|
||||
connection: connection, layer: streamView.displayLayer,
|
||||
onFrame: onFrame, onSessionEnd: onSessionEnd)
|
||||
self.pump = pump
|
||||
|
||||
// GC only delivers while active; everything held is flushed by InputCapture's
|
||||
// own resign observer — here we just mirror the capture state for the HUD and
|
||||
// the pointer lock.
|
||||
observers.append(NotificationCenter.default.addObserver(
|
||||
forName: UIApplication.willResignActiveNotification, object: nil, queue: .main
|
||||
) { [weak self] _ in
|
||||
self?.setCaptured(false)
|
||||
})
|
||||
|
||||
if captureEnabled {
|
||||
setCaptured(true) // entering a session is the deliberate "capture me" moment
|
||||
}
|
||||
}
|
||||
|
||||
func stop() {
|
||||
setCaptured(false)
|
||||
observers.forEach(NotificationCenter.default.removeObserver(_:))
|
||||
observers.removeAll()
|
||||
inputCapture?.stop()
|
||||
inputCapture = nil
|
||||
pump?.stop()
|
||||
pump = nil
|
||||
connection = nil
|
||||
streamView.onTouchEvent = nil
|
||||
streamView.currentHostMode = nil
|
||||
}
|
||||
|
||||
private func setCaptured(_ on: Bool) {
|
||||
if on {
|
||||
guard captureEnabled, !captured, pump != nil else { return }
|
||||
inputCapture?.setForwarding(true)
|
||||
captured = true
|
||||
} else {
|
||||
guard captured else { return }
|
||||
inputCapture?.setForwarding(false)
|
||||
captured = false
|
||||
}
|
||||
setNeedsUpdateOfPrefersPointerLocked()
|
||||
let onCaptureChange = onCaptureChange
|
||||
let captured = captured
|
||||
DispatchQueue.main.async { onCaptureChange?(captured) }
|
||||
}
|
||||
|
||||
deinit {
|
||||
observers.forEach(NotificationCenter.default.removeObserver(_:))
|
||||
pump?.stop()
|
||||
}
|
||||
}
|
||||
|
||||
/// The layer-backed video surface + touch source. Touches are mapped through the
|
||||
/// aspect-fit letterbox into host-mode pixels (surface == host mode, so the host-side
|
||||
/// rescale is the identity); touches outside the video area are clamped onto its edge.
|
||||
final class StreamLayerUIView: UIView {
|
||||
override class var layerClass: AnyClass { AVSampleBufferDisplayLayer.self }
|
||||
var displayLayer: AVSampleBufferDisplayLayer {
|
||||
// swiftlint:disable:next force_cast
|
||||
layer as! AVSampleBufferDisplayLayer
|
||||
}
|
||||
|
||||
/// Reads the LIVE negotiated mode in pixels (the touch coordinate space).
|
||||
var currentHostMode: (() -> CGSize)?
|
||||
var onTouchEvent: ((PunktfunkInputEvent) -> Void)?
|
||||
|
||||
/// Wire touch ids per active UITouch; ids are reused after the touch ends.
|
||||
private var touchIDs: [ObjectIdentifier: UInt32] = [:]
|
||||
|
||||
override init(frame: CGRect) {
|
||||
super.init(frame: frame)
|
||||
displayLayer.videoGravity = .resizeAspect
|
||||
isMultipleTouchEnabled = true
|
||||
backgroundColor = .black
|
||||
}
|
||||
|
||||
@available(*, unavailable)
|
||||
required init?(coder: NSCoder) { fatalError("not used") }
|
||||
|
||||
override func touchesBegan(_ touches: Set<UITouch>, with event: UIEvent?) {
|
||||
forward(touches, kind: .down)
|
||||
}
|
||||
override func touchesMoved(_ touches: Set<UITouch>, with event: UIEvent?) {
|
||||
forward(touches, kind: .move)
|
||||
}
|
||||
override func touchesEnded(_ touches: Set<UITouch>, with event: UIEvent?) {
|
||||
forward(touches, kind: .up)
|
||||
}
|
||||
override func touchesCancelled(_ touches: Set<UITouch>, with event: UIEvent?) {
|
||||
forward(touches, kind: .up)
|
||||
}
|
||||
|
||||
private enum TouchKind { case down, move, up }
|
||||
|
||||
private func forward(_ touches: Set<UITouch>, kind: TouchKind) {
|
||||
guard let hostMode = currentHostMode?(),
|
||||
hostMode.width > 0, hostMode.height > 0, onTouchEvent != nil
|
||||
else { return }
|
||||
let video = AVMakeRect(aspectRatio: hostMode, insideRect: bounds)
|
||||
guard video.width > 0, video.height > 0 else { return }
|
||||
for touch in touches {
|
||||
let key = ObjectIdentifier(touch)
|
||||
let id: UInt32
|
||||
switch kind {
|
||||
case .down:
|
||||
id = nextFreeID()
|
||||
touchIDs[key] = id
|
||||
case .move, .up:
|
||||
guard let known = touchIDs[key] else { continue }
|
||||
id = known
|
||||
}
|
||||
if kind == .up {
|
||||
touchIDs.removeValue(forKey: key)
|
||||
onTouchEvent?(.touchUp(id: id))
|
||||
continue
|
||||
}
|
||||
let p = touch.location(in: self)
|
||||
let x = Int32(((p.x - video.minX) / video.width * hostMode.width)
|
||||
.rounded().clamped(to: 0...(hostMode.width - 1)))
|
||||
let y = Int32(((p.y - video.minY) / video.height * hostMode.height)
|
||||
.rounded().clamped(to: 0...(hostMode.height - 1)))
|
||||
let w = UInt32(hostMode.width)
|
||||
let h = UInt32(hostMode.height)
|
||||
onTouchEvent?(
|
||||
kind == .down
|
||||
? .touchDown(id: id, x: x, y: y, surfaceWidth: w, surfaceHeight: h)
|
||||
: .touchMove(id: id, x: x, y: y, surfaceWidth: w, surfaceHeight: h))
|
||||
}
|
||||
}
|
||||
|
||||
private func nextFreeID() -> UInt32 {
|
||||
var id: UInt32 = 0
|
||||
while touchIDs.values.contains(id) { id += 1 }
|
||||
return id
|
||||
}
|
||||
}
|
||||
|
||||
extension CGFloat {
|
||||
fileprivate func clamped(to range: ClosedRange<CGFloat>) -> CGFloat {
|
||||
Swift.min(Swift.max(self, range.lowerBound), range.upperBound)
|
||||
}
|
||||
}
|
||||
#endif
|
||||
@@ -11,7 +11,7 @@ set -euo pipefail
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
TARGETS_MAC=(aarch64-apple-darwin x86_64-apple-darwin)
|
||||
BUILD_IOS="${BUILD_IOS:-0}" # BUILD_IOS=1 adds an iOS slice (requires the ios target installed)
|
||||
BUILD_IOS="${BUILD_IOS:-0}" # BUILD_IOS=1 adds iOS device + simulator slices (rustup targets aarch64-apple-ios{,-sim})
|
||||
|
||||
# Deployment targets must match Package.swift's platforms, or every consumer link emits
|
||||
# "object file was built for newer macOS version" warnings.
|
||||
@@ -20,6 +20,8 @@ for t in "${TARGETS_MAC[@]}"; do
|
||||
done
|
||||
if [[ "$BUILD_IOS" == "1" ]]; then
|
||||
IPHONEOS_DEPLOYMENT_TARGET=17.0 cargo build --release -p punktfunk-core --features quic --target aarch64-apple-ios
|
||||
IPHONEOS_DEPLOYMENT_TARGET=17.0 cargo build --release -p punktfunk-core --features quic --target aarch64-apple-ios-sim
|
||||
IPHONEOS_DEPLOYMENT_TARGET=17.0 cargo build --release -p punktfunk-core --features quic --target x86_64-apple-ios
|
||||
fi
|
||||
|
||||
STAGE="$(mktemp -d)"
|
||||
@@ -48,7 +50,14 @@ EOF
|
||||
|
||||
ARGS=(-library "$STAGE/macos/libpunktfunk_core.a" -headers "$STAGE/include")
|
||||
if [[ "$BUILD_IOS" == "1" ]]; then
|
||||
# Universal simulator lib (arm64 Macs run arm64 sims, but generic builds link x86_64 too).
|
||||
mkdir -p "$STAGE/iossim"
|
||||
lipo -create \
|
||||
target/aarch64-apple-ios-sim/release/libpunktfunk_core.a \
|
||||
target/x86_64-apple-ios/release/libpunktfunk_core.a \
|
||||
-output "$STAGE/iossim/libpunktfunk_core.a"
|
||||
ARGS+=(-library target/aarch64-apple-ios/release/libpunktfunk_core.a -headers "$STAGE/include")
|
||||
ARGS+=(-library "$STAGE/iossim/libpunktfunk_core.a" -headers "$STAGE/include")
|
||||
fi
|
||||
|
||||
# Cargo does NOT fingerprint MACOSX_DEPLOYMENT_TARGET — units cached from a build without
|
||||
|
||||
Reference in New Issue
Block a user