feat(client): freeze-until-reanchor loss recovery on Android + Apple via shared core gate
After unrecoverable loss the host keeps sending delta frames that reference a picture the client never received; hardware decoders conceal these as gray/ garbage with a success status. Linux already withheld them and held the last good frame until a proven clean re-anchor — this brings that behavior to the Android and Apple clients. Extract the Linux pump's freeze state machine into a shared `ReanchorGate` in punktfunk-core (reanchor.rs, 18 tests) exposed over the C ABI (ABI v6, additive — no wire change) for the Swift clients. Migrate the Linux/Deck pump (pf-client-core) onto it as the parity proof (no-op refactor). Then wire: - Android (decode.rs, both sync + async loops): arm on the frame-index gap, a pts-keyed flag map carries the wire flags to the output-buffer release, fold the gate per drained output, gate.poll replaces the dropped-climb block. - Apple Stage2Pipeline (default): arm on a gap (new noteFrameIndexGap), withhold at the ring-submit seam (CAMetalLayer holds its last drawable), poll framesDropped, fold VT decode errors through the no-output streak. - Apple StreamPump (stage-1): fold at enqueue, withhold via kCMSampleAttachmentKey_DoNotDisplay so the layer keeps decoding (reference chain intact) but holds the last displayed frame. - Apple VideoDecoder: thread the AU's wire flags to the async decode callback via a retained FrameContext refcon (replaces the receivedNs bit-pattern scalar). Lifts only on a proven re-anchor (IDR / RFI anchor / 2nd recovery mark) with a 500 ms backstop so a lost re-anchor can never freeze forever. Apple: swift build clean, 123/123 tests pass (incl. VideoToolboxRoundTripTests). On-glass loss-injection validation still owed. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -28,6 +28,11 @@ final class StreamPump {
|
||||
// Coalesced host keyframe requests (100 ms throttle — see KeyframeRecovery).
|
||||
let recovery = KeyframeRecovery()
|
||||
recovery.bind(connection)
|
||||
// Post-loss freeze-until-reanchor (shared core policy via the C ABI). Stage-1 has no per-frame
|
||||
// decode callback, so the gate is folded at ENQUEUE (from the AU's wire flags): a withheld
|
||||
// frame is still enqueued but flagged DoNotDisplay so the layer's decoder keeps the reference
|
||||
// chain fed while the last GOOD picture stays on glass — until a clean re-anchor lifts it.
|
||||
let gate = ReanchorGate(framesDropped: connection.framesDropped())
|
||||
// The layer is non-Sendable but its enqueue/flush are documented thread-safe, and after
|
||||
// this point only the pump thread drives it — assert that so the @Sendable Thread closure
|
||||
// may capture it.
|
||||
@@ -77,13 +82,17 @@ final class StreamPump {
|
||||
awaitingIDR = true
|
||||
}
|
||||
if awaitingIDR { recovery.request() }
|
||||
// Freeze backstop: a drop-count climb arms the gate (should the frame-index gap
|
||||
// below be lost too), and an overdue freeze re-asks for the re-anchor.
|
||||
if gate.poll(framesDropped: dropped) { recovery.request() }
|
||||
|
||||
guard let au = try connection.nextAU(timeoutMs: 100) else { return true }
|
||||
// Loss recovery (RFI): a forward frame-index gap fires a throttled reference-
|
||||
// frame-invalidation request so an RFI-capable host (AMD LTR / NVENC) recovers
|
||||
// with a cheap clean P-frame instead of a full IDR. The framesDropped-driven
|
||||
// recovery above stays the backstop for when the recovery frame itself is lost.
|
||||
connection.noteFrameIndex(au.frameIndex)
|
||||
// The same gap is the earliest, most precise signal to ARM the display freeze.
|
||||
if connection.noteFrameIndexGap(au.frameIndex) { gate.arm() }
|
||||
onFrame?(au)
|
||||
let idrFormat = connection.videoCodec.formatDescription(fromKeyframe: au.data)
|
||||
if let f = idrFormat {
|
||||
@@ -107,6 +116,7 @@ final class StreamPump {
|
||||
// delta into a failed layer can't recover it.
|
||||
if !wasFailed { pumpLog.warning("video: display layer .failed — flushing + re-anchoring") }
|
||||
layer.flush()
|
||||
gate.arm() // a wedged decoder is a loss — freeze until the re-anchor
|
||||
if idrFormat == nil {
|
||||
format = nil
|
||||
awaitingIDR = true
|
||||
@@ -117,6 +127,13 @@ final class StreamPump {
|
||||
let sample = connection.videoCodec.sampleBuffer(au: au, format: f),
|
||||
!token.isStopped // don't enqueue a stale frame after a restart
|
||||
else { return true }
|
||||
// Freeze-until-reanchor: while holding, WITHHOLD this concealed post-loss frame by
|
||||
// flagging it DoNotDisplay — the layer still decodes it (keeping the reference
|
||||
// chain fed) but shows the last GOOD picture until a clean re-anchor lifts the
|
||||
// gate. Folded from the AU's wire flags (stage-1 has no decode callback).
|
||||
if !gate.onDecoded(flags: au.flags) {
|
||||
StreamPump.setDoNotDisplay(sample)
|
||||
}
|
||||
layer.enqueue(sample)
|
||||
return true
|
||||
} catch {
|
||||
@@ -133,6 +150,21 @@ final class StreamPump {
|
||||
thread.start()
|
||||
}
|
||||
|
||||
/// Flag a sample decode-but-don't-display (`kCMSampleAttachmentKey_DoNotDisplay`). Used to
|
||||
/// withhold decoder-concealed post-loss frames while the re-anchor gate holds: the layer keeps
|
||||
/// its reference chain fed without flipping the frozen picture. No-op if the attachments array
|
||||
/// can't be materialized (then the frame just displays — the freeze degrades to the old behavior).
|
||||
private static func setDoNotDisplay(_ sample: CMSampleBuffer) {
|
||||
guard let attachments = CMSampleBufferGetSampleAttachmentsArray(
|
||||
sample, createIfNecessary: true), CFArrayGetCount(attachments) > 0
|
||||
else { return }
|
||||
let dict = unsafeBitCast(CFArrayGetValueAtIndex(attachments, 0), to: CFMutableDictionary.self)
|
||||
CFDictionarySetValue(
|
||||
dict,
|
||||
Unmanaged.passUnretained(kCMSampleAttachmentKey_DoNotDisplay).toOpaque(),
|
||||
Unmanaged.passUnretained(kCFBooleanTrue).toOpaque())
|
||||
}
|
||||
|
||||
/// Stop pumping (≤ one poll timeout). Does not close the connection.
|
||||
func stop() {
|
||||
token.stop()
|
||||
|
||||
Reference in New Issue
Block a user