diff --git a/clients/android/app/src/main/kotlin/io/unom/punktfunk/StreamScreen.kt b/clients/android/app/src/main/kotlin/io/unom/punktfunk/StreamScreen.kt index 2cec8b19..4e21f75f 100644 --- a/clients/android/app/src/main/kotlin/io/unom/punktfunk/StreamScreen.kt +++ b/clients/android/app/src/main/kotlin/io/unom/punktfunk/StreamScreen.kt @@ -471,12 +471,22 @@ fun StreamScreen(handle: Long, micEnabled: Boolean, onDisconnect: () -> Unit) { // vocabulary) or real multi-touch passthrough — see TouchInput.kt. Passthrough gets no // keyboard gesture: its fingers belong to the host verbatim (a swipe there may BE a // host-OS gesture), so intercepting three fingers would corrupt real multi-touch. + // Stylus lane (design/pen-tablet-input.md §7): against a HOST_CAP_PEN host a stylus + // splits out of BOTH touch models onto the pen plane; its heartbeat coroutine keeps a + // stationary held stroke alive (and its cancellation lifts everything on teardown). + val stylus = remember(handle) { + if (NativeBridge.nativeHostSupportsPen(handle)) StylusStream(handle) else null + } + if (stylus != null) { + LaunchedEffect(stylus) { stylus.heartbeatLoop() } + } Box( Modifier.fillMaxSize().pointerInput(handle, touchMode) { when (touchMode) { - TouchMode.TOUCH -> streamTouchPassthrough(handle) + TouchMode.TOUCH -> streamTouchPassthrough(handle, stylus) else -> streamTouchInput( handle, + stylus, trackpad = touchMode == TouchMode.TRACKPAD, invertScroll = initialSettings.invertScroll, onCycleStats = { statsVerbosity = statsVerbosity.next() }, diff --git a/clients/android/app/src/main/kotlin/io/unom/punktfunk/StylusInput.kt b/clients/android/app/src/main/kotlin/io/unom/punktfunk/StylusInput.kt new file mode 100644 index 00000000..f6220c20 --- /dev/null +++ b/clients/android/app/src/main/kotlin/io/unom/punktfunk/StylusInput.kt @@ -0,0 +1,195 @@ +package io.unom.punktfunk + +import android.view.MotionEvent +import androidx.compose.ui.ExperimentalComposeUiApi +import androidx.compose.ui.input.pointer.PointerEvent +import androidx.compose.ui.input.pointer.PointerType +import androidx.compose.ui.unit.IntSize +import io.unom.punktfunk.kit.NativeBridge +import kotlinx.coroutines.delay + +// Wire PEN_* state bits (punktfunk_core::quic::pen; mirrored, asserted by the Rust shim's docs). +private const val PEN_IN_RANGE = 1f +private const val PEN_TOUCHING = 2f +private const val PEN_BARREL1 = 4f +private const val PEN_BARREL2 = 8f +private const val STRIDE = 10 +private const val MAX_SAMPLES = 8 + +/** + * Android stylus → the state-full pen plane (design/pen-tablet-input.md §7): pressure, tilt + * (`AXIS_TILT`, radians from the surface normal), azimuth (`AXIS_ORIENTATION` — Android's 0 = + * "pointed away from the user" IS the wire's north, no offset needed), hover with + * `AXIS_DISTANCE`, the eraser tool, both stylus barrel buttons, and historical (coalesced) + * samples batched oldest-first for full capture-rate fidelity. Android has no barrel-roll + * axis — roll stays unknown on this client. + * + * Both touch loops call [intercept] first; stylus/eraser pointers are consumed here (against a + * pen-capable host) and never reach the finger paths, independent of the touch-input mode. + * [heartbeatLoop] implements the ≤100 ms keepalive wire contract: a stationary held stylus is + * silent in Android's input pipeline, and the host force-releases a stroke after 200 ms + * without samples. + */ +internal class StylusStream(private val handle: Long) { + private var inRange = false + private var touching = false + private var sawHover = false + private val last = FloatArray(STRIDE) + private val batch = FloatArray(MAX_SAMPLES * STRIDE) + + init { + idle(last) + } + + /** + * Consume the event's stylus pointers into pen samples. Returns true when this event + * carried any (the caller's finger/gesture handling must then skip those changes). + */ + @OptIn(ExperimentalComposeUiApi::class) + fun intercept(ev: PointerEvent, size: IntSize): Boolean { + val stylusChanges = ev.changes.filter { + it.type == PointerType.Stylus || it.type == PointerType.Eraser + } + if (stylusChanges.isEmpty()) return false + stylusChanges.forEach { it.consume() } + val me = ev.motionEvent ?: return true + if (size.width <= 0 || size.height <= 0) return true + // At most one stylus exists — find its pointer index by tool type. + val idx = (0 until me.pointerCount).firstOrNull { + me.getToolType(it) == MotionEvent.TOOL_TYPE_STYLUS || + me.getToolType(it) == MotionEvent.TOOL_TYPE_ERASER + } ?: return true + + when (me.actionMasked) { + MotionEvent.ACTION_DOWN, MotionEvent.ACTION_POINTER_DOWN, + MotionEvent.ACTION_MOVE, + -> { + touching = true + inRange = true + emitSamples(me, idx, size) + } + MotionEvent.ACTION_HOVER_ENTER, MotionEvent.ACTION_HOVER_MOVE -> { + sawHover = true + inRange = true + touching = false + emitSamples(me, idx, size) + } + MotionEvent.ACTION_UP, MotionEvent.ACTION_POINTER_UP -> { + touching = false + // Hover-capable hardware keeps proximity (HOVER_EXIT owns the leave); + // anything else leaves range on lift — the host never parks a phantom pen. + inRange = sawHover + emitSamples(me, idx, size) + } + MotionEvent.ACTION_HOVER_EXIT, MotionEvent.ACTION_CANCEL -> release() + else -> {} + } + return true + } + + /** Session/composition teardown: leave range so the host lifts anything still inked. */ + fun reset() { + if (inRange || touching) release() + sawHover = false + } + + /** The ≤100 ms keepalive (80 ms leaves headroom for one lost datagram). Runs until + * cancelled; resends the last state-full sample while the pen is in range. */ + suspend fun heartbeatLoop() { + try { + while (true) { + delay(80) + if (inRange || touching) { + last[9] = 0f // dt + NativeBridge.nativeSendPen(handle, last, 1) + } + } + } finally { + reset() + } + } + + private fun release() { + touching = false + inRange = false + last[0] = 0f // state: out of range + last[4] = 0f // pressure + NativeBridge.nativeSendPen(handle, last, 1) + } + + /** Historical (coalesced) samples oldest-first, then the current one — a single batch. */ + private fun emitSamples(me: MotionEvent, idx: Int, size: IntSize) { + val history = minOf(me.historySize, MAX_SAMPLES - 1) + var count = 0 + var prevT = if (history > 0) me.getHistoricalEventTime(0) else me.eventTime + for (h in (me.historySize - history) until me.historySize) { + val t = me.getHistoricalEventTime(h) + fill( + batch, count * STRIDE, size, + x = me.getHistoricalX(idx, h), y = me.getHistoricalY(idx, h), + pressure = me.getHistoricalPressure(idx, h), + tiltRad = me.getHistoricalAxisValue(MotionEvent.AXIS_TILT, idx, h), + orientRad = me.getHistoricalAxisValue(MotionEvent.AXIS_ORIENTATION, idx, h), + distance = me.getHistoricalAxisValue(MotionEvent.AXIS_DISTANCE, idx, h), + buttons = me.buttonState, tool = me.getToolType(idx), + dtUs = ((t - prevT) * 1000).coerceIn(0, 65535).toFloat(), + ) + prevT = t + count++ + } + fill( + batch, count * STRIDE, size, + x = me.getX(idx), y = me.getY(idx), pressure = me.getPressure(idx), + tiltRad = me.getAxisValue(MotionEvent.AXIS_TILT, idx), + orientRad = me.getAxisValue(MotionEvent.AXIS_ORIENTATION, idx), + distance = me.getAxisValue(MotionEvent.AXIS_DISTANCE, idx), + buttons = me.buttonState, tool = me.getToolType(idx), + dtUs = ((me.eventTime - prevT) * 1000).coerceIn(0, 65535).toFloat(), + ) + count++ + batch.copyInto(last, 0, (count - 1) * STRIDE, count * STRIDE) + NativeBridge.nativeSendPen(handle, batch, count) + } + + private fun fill( + out: FloatArray, + off: Int, + size: IntSize, + x: Float, + y: Float, + pressure: Float, + tiltRad: Float, + orientRad: Float, + distance: Float, + buttons: Int, + tool: Int, + dtUs: Float, + ) { + var state = 0f + if (inRange || touching) state += PEN_IN_RANGE + if (touching) state += PEN_TOUCHING + if (buttons and MotionEvent.BUTTON_STYLUS_PRIMARY != 0) state += PEN_BARREL1 + if (buttons and MotionEvent.BUTTON_STYLUS_SECONDARY != 0) state += PEN_BARREL2 + out[off + 0] = state + out[off + 1] = if (tool == MotionEvent.TOOL_TYPE_ERASER) 1f else 0f + out[off + 2] = (x / (size.width - 1).coerceAtLeast(1)).coerceIn(0f, 1f) + out[off + 3] = (y / (size.height - 1).coerceAtLeast(1)).coerceIn(0f, 1f) + out[off + 4] = if (touching) pressure.coerceIn(0f, 1f) else 0f + // AXIS_DISTANCE units are device-arbitrary; 0..1 covers real hardware, and 0 while + // hovering legitimately means "at the hover floor". + out[off + 5] = if (touching) 0f else distance.coerceIn(0f, 1f) + out[off + 6] = Math.toDegrees(tiltRad.toDouble()).toFloat().coerceIn(0f, 90f) + // AXIS_ORIENTATION: 0 = pointed away from the user (= wire north), clockwise, −π..π. + out[off + 7] = ((Math.toDegrees(orientRad.toDouble()) + 360.0) % 360.0).toFloat() + out[off + 8] = -1f // no barrel-roll axis on Android + out[off + 9] = dtUs + } + + private fun idle(out: FloatArray) { + out.fill(0f) + out[5] = -1f // distance unknown + out[6] = -1f // tilt unknown + out[7] = -1f // azimuth unknown + out[8] = -1f // roll unknown + } +} diff --git a/clients/android/app/src/main/kotlin/io/unom/punktfunk/TouchInput.kt b/clients/android/app/src/main/kotlin/io/unom/punktfunk/TouchInput.kt index af4d298e..5adfe0ff 100644 --- a/clients/android/app/src/main/kotlin/io/unom/punktfunk/TouchInput.kt +++ b/clients/android/app/src/main/kotlin/io/unom/punktfunk/TouchInput.kt @@ -1,9 +1,11 @@ package io.unom.punktfunk import androidx.compose.foundation.gestures.awaitEachGesture -import androidx.compose.foundation.gestures.awaitFirstDown +import androidx.compose.ui.input.pointer.AwaitPointerEventScope import androidx.compose.ui.input.pointer.PointerId +import androidx.compose.ui.input.pointer.PointerInputChange import androidx.compose.ui.input.pointer.PointerInputScope +import androidx.compose.ui.input.pointer.PointerType import androidx.compose.ui.input.pointer.changedToDownIgnoreConsumed import androidx.compose.ui.input.pointer.changedToUpIgnoreConsumed import androidx.compose.ui.input.pointer.positionChanged @@ -56,7 +58,26 @@ private const val ACCEL_MAX = 3.0f * normalizes and maps into the output). On teardown (stream leaves composition) every still-held * contact is lifted so nothing stays stuck on the host. */ -internal suspend fun PointerInputScope.streamTouchPassthrough(handle: Long) { +/** Whether this change belongs to the stylus lane (only when a pen-capable host is live). */ +private fun isStylus(c: PointerInputChange, stylus: StylusStream?): Boolean = + stylus != null && (c.type == PointerType.Stylus || c.type == PointerType.Eraser) + +/** [awaitFirstDown] with the stylus lane split out: pen events feed [stylus] and never start a + * mouse/touch gesture. Toward a pen-less host ([stylus] == null) a stylus stays a finger. */ +private suspend fun AwaitPointerEventScope.awaitFirstFingerDown( + stylus: StylusStream?, +): PointerInputChange { + while (true) { + val ev = awaitPointerEvent() + stylus?.intercept(ev, size) + val down = ev.changes.firstOrNull { + it.changedToDownIgnoreConsumed() && !isStylus(it, stylus) + } + if (down != null) return down + } +} + +internal suspend fun PointerInputScope.streamTouchPassthrough(handle: Long, stylus: StylusStream?) { val ids = mutableMapOf() fun alloc(p: PointerId): Int { var id = 0 @@ -68,10 +89,12 @@ internal suspend fun PointerInputScope.streamTouchPassthrough(handle: Long) { awaitPointerEventScope { while (true) { val ev = awaitPointerEvent() + stylus?.intercept(ev, size) val sw = size.width val sh = size.height if (sw <= 0 || sh <= 0) continue for (c in ev.changes) { + if (isStylus(c, stylus)) continue // the pen plane owns it val x = c.position.x.roundToInt().coerceIn(0, sw - 1) val y = c.position.y.roundToInt().coerceIn(0, sh - 1) when { @@ -98,6 +121,7 @@ internal suspend fun PointerInputScope.streamTouchPassthrough(handle: Long) { internal suspend fun PointerInputScope.streamTouchInput( handle: Long, + stylus: StylusStream?, trackpad: Boolean, invertScroll: Boolean, onCycleStats: () -> Unit, @@ -120,7 +144,7 @@ internal suspend fun PointerInputScope.streamTouchInput( ) } awaitEachGesture { - val down = awaitFirstDown(requireUnconsumed = false) + val down = awaitFirstFingerDown(stylus) val startX = down.position.x val startY = down.position.y // A touch landing just after a quick tap nearby = tap-and-drag: hold the left @@ -157,7 +181,8 @@ internal suspend fun PointerInputScope.streamTouchInput( while (true) { val ev = awaitPointerEvent() - val pressed = ev.changes.filter { it.pressed } + stylus?.intercept(ev, size) + val pressed = ev.changes.filter { it.pressed && !isStylus(it, stylus) } if (pressed.isEmpty()) { upTime = ev.changes.firstOrNull()?.uptimeMillis ?: upTime break diff --git a/clients/android/kit/src/main/kotlin/io/unom/punktfunk/kit/NativeBridge.kt b/clients/android/kit/src/main/kotlin/io/unom/punktfunk/kit/NativeBridge.kt index f62c5d6d..7596e2f2 100644 --- a/clients/android/kit/src/main/kotlin/io/unom/punktfunk/kit/NativeBridge.kt +++ b/clients/android/kit/src/main/kotlin/io/unom/punktfunk/kit/NativeBridge.kt @@ -287,6 +287,22 @@ object NativeBridge { /** One key transition. vk: Windows VK (0 = dropped by Rust). mods: VK modifier mask (0 for now). */ external fun nativeSendKey(handle: Long, vk: Int, down: Boolean, mods: Int) + /** + * Whether the host advertised full-fidelity stylus injection (`HOST_CAP_PEN`) — the gate + * for splitting stylus pointers out of the touch path onto the pen plane. False on `0`. + */ + external fun nativeHostSupportsPen(handle: Long): Boolean + + /** + * One stylus batch of STATE-FULL samples (the pen plane; design/pen-tablet-input.md §7): + * [count] × 10 floats, oldest first — `[state, tool, x, y, pressure, distance, tilt_deg, + * azimuth_deg, roll_deg, dt_us]`. `state` = the wire in-range/touching/barrel bits; `tool` + * 0=pen 1=eraser; x/y/pressure/distance normalized 0..1; distance/tilt/azimuth/roll < 0 = + * unknown. Send only when [nativeHostSupportsPen]; repeat the last sample ≤100 ms while the + * pen is in range (the host force-releases a silent stroke after 200 ms). + */ + external fun nativeSendPen(handle: Long, samples: FloatArray, count: Int) + /** * Whether the host advertised committed-text injection (`HOST_CAP_TEXT_INPUT`) — its inject * backend can type Unicode text directly. Picks the real IME `InputConnection` (autocorrect, diff --git a/clients/android/native/src/session/input.rs b/clients/android/native/src/session/input.rs index ca1f2267..97dc750c 100644 --- a/clients/android/native/src/session/input.rs +++ b/clients/android/native/src/session/input.rs @@ -6,11 +6,14 @@ //! conventions: buttons 1=left/2=middle/3=right/4=X1/5=X2; scroll axis 0=vertical/1=horizontal, //! signed 120-unit delta, +=up/right; keys are Windows VK (mapped from KEYCODE_* on the Kotlin side). -use jni::objects::{JByteBuffer, JObject, JString}; +use jni::objects::{JByteBuffer, JFloatArray, JObject, JString}; use jni::sys::{jboolean, jint, jlong}; use jni::JNIEnv; use punktfunk_core::input::{InputEvent, InputKind}; -use punktfunk_core::quic::{RichInput, HID_REPORT_MAX, HOST_CAP_TEXT_INPUT}; +use punktfunk_core::quic::{ + PenSample, PenTool, RichInput, HID_REPORT_MAX, HOST_CAP_PEN, HOST_CAP_TEXT_INPUT, + PEN_ANGLE_UNKNOWN, PEN_BATCH_MAX, PEN_DISTANCE_UNKNOWN, PEN_TILT_UNKNOWN, +}; use super::SessionHandle; @@ -162,6 +165,93 @@ pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeTextInputSu u8::from(h.client.host_caps() & HOST_CAP_TEXT_INPUT != 0) } +/// `NativeBridge.nativeHostSupportsPen(handle)` — the host advertised `HOST_CAP_PEN`, so the +/// Kotlin side splits stylus pointers out of the touch path onto the pen plane +/// (design/pen-tablet-input.md §7). `0` handle → false. +#[no_mangle] +pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeHostSupportsPen( + _env: JNIEnv, + _this: JObject, + handle: jlong, +) -> jboolean { + if handle == 0 { + return 0; + } + // SAFETY: live handle per the nativeConnect/nativeClose contract; host_caps is &self. + let h = unsafe { &*(handle as *const SessionHandle) }; + u8::from(h.client.host_caps() & HOST_CAP_PEN != 0) +} + +/// Floats per sample in the `nativeSendPen` flat array. +const PEN_JNI_STRIDE: usize = 10; + +/// `NativeBridge.nativeSendPen(handle, samples, count)` — one stylus batch of STATE-FULL +/// samples, `count` × [`PEN_JNI_STRIDE`] floats, oldest first: +/// `[state, tool, x, y, pressure, distance, tilt_deg, azimuth_deg, roll_deg, dt_us]`. +/// `state` = the wire `PEN_*` bits; `tool` 0=pen 1=eraser; `x`/`y`/`pressure`/`distance` +/// normalized 0..1; `distance`/`tilt_deg`/`azimuth_deg`/`roll_deg` < 0 = unknown. Call only +/// against a [`nativeHostSupportsPen`] host; the client heartbeats the last sample ≤100 ms +/// while in range (Kotlin side — see `StylusStream`). +#[no_mangle] +pub extern "system" fn Java_io_unom_punktfunk_kit_NativeBridge_nativeSendPen( + env: JNIEnv, + _this: JObject, + handle: jlong, + samples: JFloatArray, + count: jint, +) { + if handle == 0 || count <= 0 { + return; + } + let count = (count as usize).min(PEN_BATCH_MAX); + let mut buf = [0f32; PEN_BATCH_MAX * PEN_JNI_STRIDE]; + let flat = &mut buf[..count * PEN_JNI_STRIDE]; + if env.get_float_array_region(&samples, 0, flat).is_err() { + return; // short array — a bridge bug, never worth a crash on the input path + } + let mut batch = [PenSample::default(); PEN_BATCH_MAX]; + for (slot, s) in batch.iter_mut().zip(flat.chunks_exact(PEN_JNI_STRIDE)) { + if !s[2].is_finite() || !s[3].is_finite() { + return; // never forward a NaN coordinate + } + *slot = PenSample { + state: s[0] as u8, + tool: if s[1] as u8 == 1 { + PenTool::Eraser + } else { + PenTool::Pen + }, + x: s[2].clamp(0.0, 1.0), + y: s[3].clamp(0.0, 1.0), + pressure: (s[4].clamp(0.0, 1.0) * 65535.0) as u16, + distance: if s[5] < 0.0 { + PEN_DISTANCE_UNKNOWN + } else { + (s[5].clamp(0.0, 1.0) * 65534.0) as u16 + }, + tilt_deg: if s[6] < 0.0 { + PEN_TILT_UNKNOWN + } else { + (s[6].clamp(0.0, 90.0)) as u8 + }, + azimuth_deg: if s[7] < 0.0 { + PEN_ANGLE_UNKNOWN + } else { + (s[7] as u16) % 360 + }, + roll_deg: if s[8] < 0.0 { + PEN_ANGLE_UNKNOWN + } else { + (s[8] as u16) % 360 + }, + dt_us: s[9].clamp(0.0, 65535.0) as u16, + }; + } + // SAFETY: live handle per the nativeConnect/nativeClose contract; send_pen is &self. + let h = unsafe { &*(handle as *const SessionHandle) }; + let _ = h.client.send_pen(&batch[..count]); +} + /// `NativeBridge.nativeSendText(handle, text)` — committed IME text, one `TextInput` event per /// Unicode scalar (`code` = the scalar; multi-char commits are consecutive events in order). /// Control characters are skipped — Enter/Backspace/Tab ride the VK key path. Call only when