fix(clients): pen batches split at the wire cap and the heartbeat survives tracking mode

Three drawing-path defects on the pen clients:

- The Apple in-range heartbeat was re-armed on every emit (~120 run-loop
  mutations/s during a stroke) and lived in .default run-loop mode only — a
  tracking-mode excursion >200 ms with a stationary pen crossed the host's
  force-release failsafe and dropped the held stroke. Now one long-lived
  50 ms timer in .common mode, resending only after ≥50 ms of send silence.
- Both clients TRUNCATED an over-8-sample coalesced run instead of splitting
  it into consecutive batches (the send_pen contract): Apple suffix(8)
  dropped the oldest samples, the Android JNI clamped count. An over-cap run
  means the UI thread hitched — exactly when dropping stroke geometry hurts
  most. Apple splits in emit(); the Android JNI loops send_pen over ≤8-sample
  chunks (Kotlin's per-emit ceiling is now 64 = a >250 ms stall).
- The iOS letterbox mapper did a lock+FFI currentMode() read per coalesced
  sample on the main thread, contending the ABI lock the batch send takes
  next — now a 250 ms TTL cache (mid-stream requestMode still tracked).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
2026-07-31 17:41:15 +02:00
co-authored by Claude Fable 5
parent d383161723
commit c002ca8746
4 changed files with 106 additions and 61 deletions
@@ -14,7 +14,11 @@ private const val PEN_TOUCHING = 2f
private const val PEN_BARREL1 = 4f
private const val PEN_BARREL2 = 8f
private const val STRIDE = 10
private const val MAX_SAMPLES = 8
// Ceiling on samples per emit, NOT the wire batch size: the JNI layer splits an over-8 run into
// consecutive wire batches (never truncates — a long historical run means the UI thread hitched,
// which is exactly when dropping its head would notch the stroke). 64 samples ≈ >250 ms of
// 240 Hz history; anything past that clamp is a pathological stall, not stroke geometry.
private const val MAX_SAMPLES = 64
/**
* Android stylus → the state-full pen plane (design/pen-tablet-input.md §7): pressure, tilt
@@ -117,7 +121,8 @@ internal class StylusStream(private val handle: Long) {
NativeBridge.nativeSendPen(handle, last, 1)
}
/** Historical (coalesced) samples oldest-first, then the current one — a single batch. */
/** Historical (coalesced) samples oldest-first, then the current one — one emit; the JNI
* layer splits runs longer than the wire's 8-sample batch cap into consecutive sends. */
private fun emitSamples(me: MotionEvent, idx: Int, size: IntSize) {
val history = minOf(me.historySize, MAX_SAMPLES - 1)
var count = 0
+50 -42
View File
@@ -185,7 +185,12 @@ pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeHostSupport
/// Floats per sample in the `nativeSendPen` flat array.
const PEN_JNI_STRIDE: usize = 10;
/// `NativeBridge.nativeSendPen(handle, samples, count)` — one stylus batch of STATE-FULL
/// Sample ceiling per `nativeSendPen` call: over-cap runs are SPLIT into consecutive ≤8-sample
/// `send_pen` batches (the send_pen contract — never truncated), so this only bounds the stack
/// buffer. 64 samples ≈ >250 ms of 240 Hz history = a pathological UI-thread stall.
const PEN_JNI_MAX_SAMPLES: usize = PEN_BATCH_MAX * 8;
/// `NativeBridge.nativeSendPen(handle, samples, count)` — one stylus emit of STATE-FULL
/// samples, `count` × [`PEN_JNI_STRIDE`] floats, oldest first:
/// `[state, tool, x, y, pressure, distance, tilt_deg, azimuth_deg, roll_deg, dt_us]`.
/// `state` = the wire `PEN_*` bits; `tool` 0=pen 1=eraser; `x`/`y`/`pressure`/`distance`
@@ -203,53 +208,56 @@ pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeSendPen(
if handle == 0 || count <= 0 {
return;
}
let count = (count as usize).min(PEN_BATCH_MAX);
let mut buf = [0f32; PEN_BATCH_MAX * PEN_JNI_STRIDE];
let count = (count as usize).min(PEN_JNI_MAX_SAMPLES);
let mut buf = [0f32; PEN_JNI_MAX_SAMPLES * PEN_JNI_STRIDE];
let flat = &mut buf[..count * PEN_JNI_STRIDE];
if env.get_float_array_region(&samples, 0, flat).is_err() {
return; // short array — a bridge bug, never worth a crash on the input path
}
let mut batch = [PenSample::default(); PEN_BATCH_MAX];
for (slot, s) in batch.iter_mut().zip(flat.chunks_exact(PEN_JNI_STRIDE)) {
if !s[2].is_finite() || !s[3].is_finite() {
return; // never forward a NaN coordinate
}
*slot = PenSample {
state: s[0] as u8,
tool: if s[1] as u8 == 1 {
PenTool::Eraser
} else {
PenTool::Pen
},
x: s[2].clamp(0.0, 1.0),
y: s[3].clamp(0.0, 1.0),
pressure: (s[4].clamp(0.0, 1.0) * 65535.0) as u16,
distance: if s[5] < 0.0 {
PEN_DISTANCE_UNKNOWN
} else {
(s[5].clamp(0.0, 1.0) * 65534.0) as u16
},
tilt_deg: if s[6] < 0.0 {
PEN_TILT_UNKNOWN
} else {
(s[6].clamp(0.0, 90.0)) as u8
},
azimuth_deg: if s[7] < 0.0 {
PEN_ANGLE_UNKNOWN
} else {
(s[7] as u16) % 360
},
roll_deg: if s[8] < 0.0 {
PEN_ANGLE_UNKNOWN
} else {
(s[8] as u16) % 360
},
dt_us: s[9].clamp(0.0, 65535.0) as u16,
};
}
// SAFETY: live handle per the nativeConnect/nativeClose contract; send_pen is &self.
let h = unsafe { &*(handle as *const SessionHandle) };
let _ = h.client.send_pen(&batch[..count]);
let mut batch = [PenSample::default(); PEN_BATCH_MAX];
for run in flat.chunks(PEN_BATCH_MAX * PEN_JNI_STRIDE) {
let n = run.len() / PEN_JNI_STRIDE;
for (slot, s) in batch.iter_mut().zip(run.chunks_exact(PEN_JNI_STRIDE)) {
if !s[2].is_finite() || !s[3].is_finite() {
return; // never forward a NaN coordinate
}
*slot = PenSample {
state: s[0] as u8,
tool: if s[1] as u8 == 1 {
PenTool::Eraser
} else {
PenTool::Pen
},
x: s[2].clamp(0.0, 1.0),
y: s[3].clamp(0.0, 1.0),
pressure: (s[4].clamp(0.0, 1.0) * 65535.0) as u16,
distance: if s[5] < 0.0 {
PEN_DISTANCE_UNKNOWN
} else {
(s[5].clamp(0.0, 1.0) * 65534.0) as u16
},
tilt_deg: if s[6] < 0.0 {
PEN_TILT_UNKNOWN
} else {
(s[6].clamp(0.0, 90.0)) as u8
},
azimuth_deg: if s[7] < 0.0 {
PEN_ANGLE_UNKNOWN
} else {
(s[7] as u16) % 360
},
roll_deg: if s[8] < 0.0 {
PEN_ANGLE_UNKNOWN
} else {
(s[8] as u16) % 360
},
dt_us: s[9].clamp(0.0, 65535.0) as u16,
};
}
let _ = h.client.send_pen(&batch[..n]);
}
}
/// `NativeBridge.nativeSendText(handle, text)` — committed IME text, one `TextInput` event per
@@ -40,6 +40,8 @@ final class PencilStream: NSObject, UIPencilInteractionDelegate {
private(set) var hoverActive = false
private var last = PencilStream.idleSample()
private var heartbeat: Timer?
/// When the last batch (event or keepalive) went out the heartbeat tick's idle test.
private var lastSendTs: TimeInterval = 0
// MARK: - Contact path (UITouch, `.pencil` only)
@@ -52,11 +54,12 @@ final class PencilStream: NSObject, UIPencilInteractionDelegate {
touching = true
inRange = true
// Coalesced samples restore the Pencil's full capture rate (UIKit delivers at
// display cadence); oldest first, `dt_us` preserving their spacing.
// display cadence); oldest first, `dt_us` preserving their spacing. The whole run
// is kept emit() splits anything over the wire's batch cap.
let raw = event?.coalescedTouches(for: touch) ?? [touch]
var batch: [PunktfunkPenSample] = []
var prevTs: TimeInterval?
for t in raw.suffix(Int(PUNKTFUNK_PEN_BATCH_MAX)) {
for t in raw {
guard let s = contactSample(t, in: view, prevTs: prevTs) else { continue }
prevTs = t.timestamp
batch.append(s)
@@ -202,27 +205,46 @@ final class PencilStream: NSObject, UIPencilInteractionDelegate {
guard !batch.isEmpty else { return }
last = batch[batch.count - 1]
last.dt_us = 0
send?(batch)
armHeartbeat()
// A run longer than the wire's batch cap is SPLIT into consecutive sends, never
// truncated (the send_pen contract): an over-cap run means the main thread hitched
// past a frame of 240 Hz samples, and dropping its head would notch exactly the
// stroke geometry a drawing app can least afford to lose.
var start = 0
while start < batch.count {
let end = min(start + Int(PUNKTFUNK_PEN_BATCH_MAX), batch.count)
send?(Array(batch[start..<end]))
start = end
}
lastSendTs = CACurrentMediaTime()
syncHeartbeat()
}
/// The 100 ms keepalive while in range (see the file header). 80 ms leaves headroom
/// under the host's 200 ms failsafe even with one lost datagram.
private func armHeartbeat() {
heartbeat?.invalidate()
/// The 100 ms keepalive while in range (see the file header): ONE long-lived 50 ms timer
/// in `.common` run-loop mode, armed on range entry and torn down on exit. `.default`-mode
/// scheduling pauses during run-loop tracking, where a stationary pen would silently cross
/// the host's 200 ms force-release mid-stroke; the old per-emit re-arm also churned the run
/// loop once per event during a stroke. The tick resends only after 50 ms of send silence,
/// so an active stroke costs nothing and the worst-case gap stays 100 ms the wire bound.
private func syncHeartbeat() {
guard inRange || touching else {
heartbeat?.invalidate()
heartbeat = nil
return
}
heartbeat = Timer.scheduledTimer(withTimeInterval: 0.08, repeats: true) {
[weak self] _ in
guard heartbeat == nil else { return }
let timer = Timer(timeInterval: 0.05, repeats: true) { [weak self] t in
guard let self, self.inRange || self.touching else {
self?.heartbeat?.invalidate()
t.invalidate()
self?.heartbeat = nil
return
}
self.send?([self.last])
if CACurrentMediaTime() - self.lastSendTs >= 0.05 {
self.send?([self.last])
self.lastSendTs = CACurrentMediaTime()
}
}
heartbeat = timer
RunLoop.main.add(timer, forMode: .common)
}
// MARK: - Angle conversions
@@ -320,12 +320,22 @@ public final class StreamViewController: StreamViewControllerBase {
// prior session (stop() doesn't clear it). Otherwise a stale `true` could later
// re-engage capture on a foreground that the new session never asked for.
wasCapturedOnResign = false
// Read the LIVE mode per touch batch an accepted requestMode() mid-stream
// changes the letterbox, and touches must follow it.
// The letterbox must follow an accepted requestMode() mid-stream, so this stays a live
// read but behind a short TTL: the pencil path maps every coalesced sample through
// here (8 per event at panel rate, main thread), and a per-sample FFI read re-takes
// the ABI lock the batch send itself needs next. 250 ms staleness is invisible next to
// the video reconfigure a mode switch performs anyway. Main-thread-only closure.
var cachedMode = CGSize.zero
var cachedAt = CACurrentMediaTime() - 1
streamView.currentHostMode = { [weak connection] in
guard let connection else { return .zero }
let mode = connection.currentMode()
return CGSize(width: Double(mode.width), height: Double(mode.height))
let now = CACurrentMediaTime()
if now - cachedAt > 0.25 {
guard let connection else { return .zero }
let mode = connection.currentMode()
cachedMode = CGSize(width: Double(mode.width), height: Double(mode.height))
cachedAt = now
}
return cachedMode
}
streamView.onTouchEvent = { [weak self, weak connection] event in
// Touch IS the intent during a trusted session, but must not leak to the host