add cursor flinging

This commit is contained in:
Mona Mayrhofer 2026-03-03 17:46:51 +01:00
parent 5f89601ef2
commit d0a458b450
WARNING! Although there is a key with this ID in the database it does not verify this commit! This commit is SUSPICIOUS.
GPG key ID: 374AB152BDEBA1AE
16 changed files with 511 additions and 1692 deletions

View file

@ -8,6 +8,7 @@ edition = "2024"
[dependencies]
dioxus = { version = "0.7.3", features = ["router", "fullstack", "logger"] }
dioxus-html = { version = "0.7.3", features = ["serialize"] }
serde = { version = "1.0.228", features = ["derive"] }
wayland-client = { version = "0.31.12", optional = true }
@ -18,6 +19,7 @@ wayland-protocols-misc = { version = "0.3.10", features = ["client"], optional =
wayland-protocols = { version = "0.32.10", features = ["client", "staging"], optional = true }
xkb = {version = "0.3.0", optional = true}
memfile = {version = "0.3.2", optional = true}
wasmtimer = "0.4.3"
[features]
default = ["web"]

View file

@ -0,0 +1,3 @@
.controls {
flex-grow: 1;
}

View file

@ -0,0 +1,8 @@
.keyboard-area {
height: 48px;
input {
width: 100%;
height: 100%;
}
}

View file

@ -3,8 +3,9 @@ body,
#main {
margin: 0;
padding: 0;
min-height: 100vh;
width: 100vw;
min-height: 100dvh;
width: 100dvw;
overflow: hidden;
}
#main {

View file

@ -1,5 +1,6 @@
.mouse-area {
background-color: white;
background: radial-gradient(#0004 15%, transparent 20%), white;
background-size: 15px 15px;
touch-action: none;
flex-grow: 1;

View file

@ -0,0 +1,99 @@
use dioxus::{
fullstack::{CborEncoding, WebSocketOptions, Websocket, extract::State, use_websocket},
html::{
geometry::{ElementSpace, euclid::Point2D},
input_data::MouseButton,
},
logger::tracing,
prelude::*,
};
use serde::{Deserialize, Serialize};
use crate::components::{KeyboardArea, MouseArea};
#[component]
pub fn Controls() -> Element {
#[css_module("/assets/styling/controls.module.css")]
struct Styles;
let mut socket = use_websocket(move || mouse_move(WebSocketOptions::new()));
use_future(move || async move {
loop {
// Wait for the socket to connect
_ = socket.connect().await;
// Loop poll with recv. Throws an error when the connection closes, making it possible
// to run code before the socket re-connects when the name input changes
while let Ok(message) = socket.recv().await {
tracing::info!("Received message: {:?}", message);
}
}
});
let event_handler = use_callback(move |evt| {
spawn(async move {
_ = socket.send(evt).await;
});
});
rsx! {
div {
class: Styles::controls,
MouseArea { onevent: event_handler }
KeyboardArea { onevent: event_handler }
}
}
}
#[derive(Serialize, Deserialize, Debug)]
pub enum ClientEvent {
MouseMove { dx: f64, dy: f64 },
MouseScroll { dx: f64, dy: f64 },
Click { button: MouseButton },
KeyPressEvent { key: String },
TextInputStartEvent,
TextInputEvent { text: String },
TextInputDoneEvent { text: String },
}
#[derive(Serialize, Deserialize, Debug)]
enum ServerEvent {
Ping,
}
#[expect(clippy::unused_async)]
#[get("/api/mouse_move_ws", mouse_service: State<crate::server::input_proxy_service::InputProxyService>)]
async fn mouse_move(
options: WebSocketOptions
) -> Result<Websocket<ClientEvent, ServerEvent, CborEncoding>> {
Ok(options.on_upgrade(move |mut socket| async move {
_ = socket.send(ServerEvent::Ping).await;
while let Ok(event) = socket.recv().await {
match event {
ClientEvent::MouseMove { dx, dy } => {
mouse_service.move_mouse(dx, dy).await;
},
ClientEvent::MouseScroll { dx, dy } => {
mouse_service.mouse_scroll(dx, dy).await;
},
ClientEvent::Click { button } => {
mouse_service.click(button).await;
},
ClientEvent::KeyPressEvent { key } => {
mouse_service.key_press_event(key).await;
},
ClientEvent::TextInputEvent { text } => {
mouse_service.text_input(text).await;
},
ClientEvent::TextInputStartEvent => {
mouse_service.text_input_start().await;
},
ClientEvent::TextInputDoneEvent { text } => {
mouse_service.text_input_end(text).await;
},
}
}
}))
}

View file

@ -0,0 +1,57 @@
use dioxus::prelude::*;
use crate::components::controls::ClientEvent;
#[component]
pub fn KeyboardArea(onevent: EventHandler<ClientEvent>) -> Element {
#[css_module("/assets/styling/keyboard_area.module.css")]
struct Styles;
let mut input_state = use_signal(String::new);
let input_handler = use_callback(move |evt: Event<FormData>| {
let v = evt.value();
input_state.set(v.clone());
onevent.call(ClientEvent::TextInputEvent { text: v })
});
let key_press_handler = use_callback(move |evt: Event<KeyboardData>| {
if input_state.read().is_empty() {
match evt.key() {
Key::Character(_) => {},
_ => {
onevent.call(ClientEvent::KeyPressEvent {
key: evt.key().to_string(),
});
},
}
} else if evt.key() == Key::Enter {
onevent.call(ClientEvent::TextInputDoneEvent {
text: input_state.replace(String::new()),
});
}
});
let input_focus_handler = use_callback(move |_: Event<FocusData>| {
input_state.set(String::new());
onevent.call(ClientEvent::TextInputStartEvent);
});
let input_blur_handler = use_callback(move |_: Event<FocusData>| {
onevent.call(ClientEvent::TextInputDoneEvent {
text: input_state.replace(String::new()),
});
});
rsx! {
div {
class: Styles::keyboard_area,
input {
oninput: input_handler,
value: input_state,
onkeydown: key_press_handler,
onfocus: input_focus_handler,
onblur: input_blur_handler
}
}
}
}

View file

@ -1,2 +1,6 @@
mod controls;
mod keyboard_area;
mod mouse_area;
pub use controls::Controls;
pub use keyboard_area::KeyboardArea;
pub use mouse_area::MouseArea;

View file

@ -1,5 +1,10 @@
use std::{
collections::{HashMap, VecDeque},
ops::Sub,
time::Duration,
};
use dioxus::{
fullstack::{CborEncoding, WebSocketOptions, Websocket, extract::State, use_websocket},
html::{
geometry::{ElementSpace, euclid::Point2D},
input_data::MouseButton,
@ -7,184 +12,160 @@ use dioxus::{
logger::tracing,
prelude::*,
};
use serde::{Deserialize, Serialize};
use dioxus_html::geometry::euclid::Vector2D;
use crate::{components::controls::ClientEvent, utils::mouse_filter_buffer::MouseFilterBuffer};
const FLING_START_THRESHOLD_VELOCITY: f64 = 500.0;
const FLING_STOP_THRESHOLD_VELOCITY: f64 = 100.0;
const FLING_DAMPENING: f64 = 0.98;
pub struct PointerRegistryData {
initial_position: Point2D<f64, ElementSpace>,
last_positions: MouseFilterBuffer,
}
pub struct FlingerData {
velocity: Vector2D<f64, ElementSpace>,
}
#[derive(Default)]
pub struct PointerRegistry {
pointers: HashMap<i32, PointerRegistryData>,
}
#[derive(Default)]
pub struct FlingerRegistry {
flinger: Option<FlingerData>,
}
#[component]
pub fn MouseArea() -> Element {
pub fn MouseArea(onevent: EventHandler<ClientEvent>) -> Element {
#[css_module("/assets/styling/mouse_area.module.css")]
struct Styles;
let mut last_cursor_position = use_signal::<Option<Point2D<f64, ElementSpace>>>(|| None);
let mut socket = use_websocket(move || mouse_move(WebSocketOptions::new()));
use_future(move || async move {
loop {
// Wait for the socket to connect
_ = socket.connect().await;
// Loop poll with recv. Throws an error when the connection closes, making it possible
// to run code before the socket re-connects when the name input changes
while let Ok(message) = socket.recv().await {
tracing::info!("Received message: {:?}", message);
}
}
});
let mut registry = use_signal::<PointerRegistry>(PointerRegistry::default);
let mut flingers = use_signal::<FlingerRegistry>(FlingerRegistry::default);
let pointer_move_handler = use_callback(move |evt: Event<PointerData>| {
if evt.held_buttons().contains(MouseButton::Primary) {
let mut registry = registry.write();
if let Some(data) = registry.pointers.get_mut(&evt.pointer_id()) {
evt.prevent_default();
let point = evt.element_coordinates();
let last_position = last_cursor_position.write().replace(point);
let last_position = data.last_positions.back();
let delta = point - last_position.position;
if let Some(last_position) = last_position {
let delta = point - last_position;
data.last_positions
.push(point, wasmtimer::std::SystemTime::now());
spawn(async move {
_ = socket
.send(ClientEvent::MouseMove {
dx: delta.x,
dy: delta.y,
})
.await;
if registry.pointers.len() == 1 {
onevent.call(ClientEvent::MouseMove {
dx: delta.x,
dy: delta.y,
});
} else if registry.pointers.len() == 2 {
onevent.call(ClientEvent::MouseScroll {
dx: -delta.x,
dy: -delta.y,
});
}
}
});
let pointer_down_handler = use_callback(move |evt: Event<PointerData>| {
//If any pointer is down, we cancel the flingers
flingers.write().flinger.take();
let point = evt.element_coordinates();
*last_cursor_position.write() = Some(point);
registry.write().pointers.insert(
evt.pointer_id(),
PointerRegistryData {
last_positions: MouseFilterBuffer::new(
10,
point,
wasmtimer::std::SystemTime::now(),
),
initial_position: point,
},
);
});
let pointer_up_handler = use_callback(move |evt: Event<PointerData>| {
let point = evt.element_coordinates();
let mut registry = registry.write();
let data = registry.pointers.remove(&evt.pointer_id());
if let Some(data) = data {
let distance_moved = data.initial_position - point;
let release_velocity = data.last_positions.average_velocity_since(
wasmtimer::std::SystemTime::now().sub(Duration::from_millis(100)),
);
tracing::info!("Release Velocity: {:?}", release_velocity.length());
let pointer_click_handler = use_callback(move |evt: Event<MouseData>| {
spawn(async move {
_ = socket.send(ClientEvent::Click).await;
});
});
let key_down_handler = use_callback(move |evt: Event<KeyboardData>| {
spawn(async move {
_ = socket
.send(ClientEvent::KeyEvent {
key: evt.key().to_string(),
is_pressed: true,
})
.await;
});
});
// let key_up_handler = use_callback(move |evt: Event<KeyboardData>| {
// spawn(async move {
// _ = socket
// .send(ClientEvent::KeyEvent {
// key: evt.key().to_string(),
// is_pressed: false,
// })
// .await;
// });
// });
let mut input_state = use_signal(String::new);
let input_handler = use_callback(move |evt: Event<FormData>| {
let v = evt.value();
input_state.set(v.clone());
spawn(async move {
_ = socket.send(ClientEvent::TextInputEvent { text: v }).await;
});
});
let key_press_handler = use_callback(move |evt: Event<KeyboardData>| {
if evt.key() == Key::Enter {
spawn(async move {
_ = socket
.send(ClientEvent::TextInputDoneEvent {
text: input_state.replace(String::new()),
})
.await;
});
if distance_moved.length() <= 1.0 {
match registry.pointers.len() {
0 => {
onevent.call(ClientEvent::Click {
button: MouseButton::Primary,
});
},
1 => {
onevent.call(ClientEvent::Click {
button: MouseButton::Secondary,
});
},
_ => {},
}
} else if release_velocity.length() > FLING_START_THRESHOLD_VELOCITY {
//We only fling if there are no other pointers
if registry.pointers.is_empty() {
flingers.write().flinger = Some(FlingerData {
velocity: release_velocity,
});
}
}
}
});
let input_focus_handler = use_callback(move |evt: Event<FocusData>| {
input_state.set(String::new());
spawn(async move {
_ = socket.send(ClientEvent::TextInputStartEvent).await;
});
});
let input_blur_handler = use_callback(move |evt: Event<FocusData>| {
spawn(async move {
_ = socket
.send(ClientEvent::TextInputDoneEvent {
text: input_state.replace(String::new()),
})
.await;
});
use_future(move || async move {
let mut last_frame_time = wasmtimer::std::SystemTime::now();
loop {
wasmtimer::tokio::sleep(Duration::from_millis(16)).await;
let new_frame_time = wasmtimer::std::SystemTime::now();
let delta_seconds = new_frame_time
.duration_since(last_frame_time)
.unwrap()
.as_secs_f64();
last_frame_time = new_frame_time;
let mut flinger = flingers.write();
let new_flinger = flinger.flinger.as_ref().and_then(|flinger| {
if flinger.velocity.length() < FLING_STOP_THRESHOLD_VELOCITY {
None
} else {
onevent.call(ClientEvent::MouseMove {
dx: flinger.velocity.x * delta_seconds,
dy: flinger.velocity.y * delta_seconds,
});
//tracing::info!("Fling: {:?}", flinger.velocity);
Some(FlingerData {
velocity: flinger.velocity * FLING_DAMPENING,
})
}
});
flinger.flinger = new_flinger;
}
});
rsx! {
div {
input {
oninput: input_handler,
value: input_state,
onkeypress: key_press_handler,
onfocus: input_focus_handler,
onblur: input_blur_handler }
div {
class: Styles::mouse_area,
class: Styles::mouse_area,
onpointermove: pointer_move_handler,
onpointerdown: pointer_down_handler,
onclick: pointer_click_handler
}
onpointermove: pointer_move_handler,
onpointerdown: pointer_down_handler,
onpointerup: pointer_up_handler,
}
}
}
#[derive(Serialize, Deserialize, Debug)]
enum ClientEvent {
MouseMove { dx: f64, dy: f64 },
Click,
KeyEvent { key: String, is_pressed: bool },
TextInputStartEvent,
TextInputEvent { text: String },
TextInputDoneEvent { text: String },
}
#[derive(Serialize, Deserialize, Debug)]
enum ServerEvent {
Ping,
}
#[expect(clippy::unused_async)]
#[get("/api/mouse_move_ws", mouse_service: State<crate::server::input_proxy_service::InputProxyService>)]
async fn mouse_move(
options: WebSocketOptions
) -> Result<Websocket<ClientEvent, ServerEvent, CborEncoding>> {
Ok(options.on_upgrade(move |mut socket| async move {
_ = socket.send(ServerEvent::Ping).await;
while let Ok(event) = socket.recv().await {
match event {
ClientEvent::MouseMove { dx, dy } => {
mouse_service.move_mouse(dx, dy).await;
},
ClientEvent::Click => {
mouse_service.click().await;
},
ClientEvent::KeyEvent { key, is_pressed } => {
mouse_service.key_event(key, is_pressed).await;
},
ClientEvent::TextInputEvent { text } => {
mouse_service.text_input(text).await;
},
ClientEvent::TextInputStartEvent => {
mouse_service.text_input_start().await;
},
ClientEvent::TextInputDoneEvent { text } => {
mouse_service.text_input_end(text).await;
},
}
}
}))
}

View file

@ -4,6 +4,7 @@ use dioxus::prelude::*;
use views::{Home, Navbar};
mod components;
mod utils;
mod views;
#[cfg(feature = "server")]
@ -45,6 +46,7 @@ fn App() -> Element {
rsx! {
document::Link { rel: "icon", href: FAVICON }
document::Link { rel: "stylesheet", href: MAIN_CSS }
document::Meta { name: "viewport", content: "width=device-width, initial-scale=1.0" }
Router::<Route> {}
}

View file

@ -4,13 +4,14 @@ use dioxus::{
fullstack::{FullstackContext, extract::FromRef},
logger::tracing,
};
use dioxus_html::input_data::MouseButton;
use memfile::MemFile;
use rustix::time::{ClockId, clock_gettime};
use tokio::sync::Mutex;
use wayland_client::{
Connection, Dispatch, EventQueue, Proxy, QueueHandle,
protocol::{
wl_pointer::ButtonState,
wl_pointer::{Axis, ButtonState},
wl_registry::{Event, WlRegistry},
wl_seat::{self, WlSeat},
},
@ -29,12 +30,12 @@ use wayland_protocols_wlr::virtual_pointer::v1::client::{
zwlr_virtual_pointer_v1::ZwlrVirtualPointerV1,
};
use crate::server::keymap;
use crate::server::keymap::{self, web_key_to_linux_keycode};
// https://github.com/torvalds/linux/blob/master/include/uapi/linux/input-event-codes.h
const BUTTON_LEFT: u32 = 0x110;
const BTN_RIGHT: u32 = 0x111;
const BTN_MIDDLE: u32 = 0x112;
const BUTTON_RIGHT: u32 = 0x111;
const BUTTON_MIDDLE: u32 = 0x112;
// https://wayland.app/protocols/wayland#wl_keyboard:enum:keymap_format
const NO_KEYMAP: u32 = 0;
@ -78,37 +79,72 @@ impl InputProxyService {
}
}
pub async fn click(&self) {
pub async fn mouse_scroll(
&self,
dx: f64,
dy: f64,
) {
let guard = self.input_proxy_service_state.lock().await;
if let Some(pointer) = &guard.state.virtual_pointer() {
tracing::info!("Do click");
let time = get_wayland_timestamp();
pointer.button(time, BUTTON_LEFT, ButtonState::Pressed);
pointer.frame();
pointer.button(time, BUTTON_LEFT, ButtonState::Released);
pointer.axis(time, Axis::HorizontalScroll, dx);
pointer.axis(time, Axis::VerticalScroll, dy);
pointer.frame();
guard.event_queue.flush().unwrap();
}
}
pub async fn key_event(
pub async fn click(
&self,
button: MouseButton,
) {
let guard = self.input_proxy_service_state.lock().await;
if let Some(pointer) = &guard.state.virtual_pointer() {
tracing::info!("Do click");
let time = get_wayland_timestamp();
let button = match button {
MouseButton::Primary => BUTTON_LEFT,
MouseButton::Secondary => BUTTON_RIGHT,
MouseButton::Auxiliary => BUTTON_MIDDLE,
MouseButton::Fourth | MouseButton::Fifth | MouseButton::Unknown => {
return;
},
};
pointer.button(time, button, ButtonState::Pressed);
pointer.frame();
pointer.button(time, button, ButtonState::Released);
pointer.frame();
guard.event_queue.flush().unwrap();
}
}
pub async fn key_press_event(
&self,
key: String,
is_pressed: bool,
) {
todo!();
let guard = self.input_proxy_service_state.lock().await;
if let Some(keyboard) = guard.state.virtual_keyboard() {
let time = get_wayland_timestamp();
let key = web_key_to_linux_keycode(key.as_str()).unwrap();
keyboard.key(time, key, 1);
keyboard.key(time, key, 0);
guard.event_queue.flush().unwrap();
}
}
pub async fn text_input_start(&self) {
let mut guard = self.input_proxy_service_state.lock().await;
if let Some(keyboard) = guard.state.input_method_mut() {
keyboard.input_method_state += 1;
if let Some(ime) = guard.state.input_method_mut() {
ime.input_method_state += 1;
tracing::info!("Text Input Start");
keyboard.input_method.delete_surrounding_text(4, 4);
keyboard.input_method.commit(keyboard.input_method_state);
ime.input_method.delete_surrounding_text(4, 4);
ime.input_method.commit(ime.input_method_state);
guard.event_queue.flush().unwrap();
}
@ -119,12 +155,12 @@ impl InputProxyService {
) {
let mut guard = self.input_proxy_service_state.lock().await;
if let Some(keyboard) = guard.state.input_method_mut() {
keyboard.input_method_state += 1;
if let Some(ime) = guard.state.input_method_mut() {
ime.input_method_state += 1;
tracing::info!("Text Input {key}");
keyboard.input_method.set_preedit_string(key, 0, 0);
keyboard.input_method.commit(keyboard.input_method_state);
ime.input_method.set_preedit_string(key, 0, 0);
ime.input_method.commit(ime.input_method_state);
guard.event_queue.flush().unwrap();
}
}
@ -134,12 +170,12 @@ impl InputProxyService {
) {
let mut guard = self.input_proxy_service_state.lock().await;
if let Some(keyboard) = guard.state.input_method_mut() {
keyboard.input_method_state += 1;
if let Some(ime) = guard.state.input_method_mut() {
ime.input_method_state += 1;
tracing::info!("Text Input End");
keyboard.input_method.commit_string(text);
keyboard.input_method.commit(keyboard.input_method_state);
ime.input_method.commit_string(text);
ime.input_method.commit(ime.input_method_state);
guard.event_queue.flush().unwrap();
}
}

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1 @@
pub mod mouse_filter_buffer;

View file

@ -0,0 +1,80 @@
use std::collections::VecDeque;
use dioxus_html::geometry::{
ElementSpace,
euclid::{Point2D, Vector2D},
};
use wasmtimer::std::SystemTime;
pub struct MouseFilterBufferEntry {
pub position: Point2D<f64, ElementSpace>,
pub time: SystemTime,
}
pub struct MouseFilterBuffer {
buffer: VecDeque<MouseFilterBufferEntry>,
}
impl MouseFilterBuffer {
pub fn new(
size: usize,
initial: Point2D<f64, ElementSpace>,
initial_timestamp: SystemTime,
) -> Self {
let mut buffer = VecDeque::with_capacity(size);
buffer.push_back(MouseFilterBufferEntry {
position: initial,
time: initial_timestamp,
});
Self { buffer }
}
pub fn push(
&mut self,
point: Point2D<f64, ElementSpace>,
time: SystemTime,
) {
if self.buffer.len() == self.buffer.capacity() {
self.buffer.pop_front();
}
self.buffer.push_back(MouseFilterBufferEntry {
position: point,
time,
});
}
pub fn back(&self) -> &MouseFilterBufferEntry {
self.buffer
.back()
.expect("Mouse filter buffer may never have less than one element")
}
pub fn average_velocity_since(
&self,
start_time: SystemTime,
) -> Vector2D<f64, ElementSpace> {
let mut total_distance = Vector2D::zero();
let mut i = self.buffer.iter();
let mut last = if let Some(it) = i.find(|it| it.time >= start_time) {
it
} else {
return Vector2D::zero();
};
let start_time = last.time;
let mut last_time = last.time;
for point in i {
total_distance += (point.position - last.position);
last = point;
last_time = point.time;
}
if total_distance == Vector2D::zero() {
return Vector2D::zero();
}
total_distance / (last_time.duration_since(start_time).unwrap().as_secs_f64())
}
}

View file

@ -1,4 +1,4 @@
use crate::components::MouseArea;
use crate::components::{Controls, MouseArea};
use dioxus::prelude::*;
/// The Home page component that will be rendered when the current route is `[Route::Home]`
@ -10,7 +10,7 @@ pub fn Home() -> Element {
rsx! {
div {
class: Styles::container,
MouseArea { }
Controls { }
}
}
}