sketch for key code transfer

crates/cursor-move-webapp/src/components/mouse_area.rs

@@ -1,6 +1,9 @@
 use dioxus::{
     fullstack::{CborEncoding, WebSocketOptions, Websocket, extract::State, use_websocket},
-    html::geometry::{ElementSpace, euclid::Point2D},
+    html::{
+        geometry::{ElementSpace, euclid::Point2D},
+        input_data::MouseButton,
+    },
     logger::tracing,
     prelude::*,
 };
@@ -28,27 +31,70 @@ pub fn MouseArea() -> Element {
         }
     });
 
+    let pointer_move_handler = use_callback(move |evt: Event<PointerData>| {
+        if evt.held_buttons().contains(MouseButton::Primary) {
+            evt.prevent_default();
+            let point = evt.element_coordinates();
+            let last_position = last_cursor_position.write().replace(point);
+
+            if let Some(last_position) = last_position {
+                let delta = point - last_position;
+
+                spawn(async move {
+                    _ = socket
+                        .send(ClientEvent::MouseMove {
+                            dx: delta.x,
+                            dy: delta.y,
+                        })
+                        .await;
+                });
+            }
+        }
+    });
+
+    let pointer_down_handler = use_callback(move |evt: Event<PointerData>| {
+        let point = evt.element_coordinates();
+        *last_cursor_position.write() = Some(point);
+    });
+
+    let pointer_click_handler = use_callback(move |evt: Event<MouseData>| {
+        spawn(async move {
+            _ = socket.send(ClientEvent::Click).await;
+        });
+    });
+
+    let key_down_handler = use_callback(move |evt: Event<KeyboardData>| {
+        tracing::info!("Keydown");
+        spawn(async move {
+            _ = socket
+                .send(ClientEvent::KeyEvent {
+                    key: evt.key().to_string(),
+                    is_pressed: true,
+                })
+                .await;
+        });
+    });
+
+    let key_up_handler = use_callback(move |evt: Event<KeyboardData>| {
+        spawn(async move {
+            _ = socket
+                .send(ClientEvent::KeyEvent {
+                    key: evt.key().to_string(),
+                    is_pressed: false,
+                })
+                .await;
+        });
+    });
+
     rsx! {
         div {
-            class: Styles::mouse_area,
+            input { onkeydown: key_down_handler, onkeyup: key_up_handler}
+            div {
+                class: Styles::mouse_area,
 
-            onpointermove: move |evt| {
-                evt.prevent_default();
-                let point = evt.element_coordinates();
-                let last_position = last_cursor_position.write().replace(point);
-
-                if let Some(last_position) = last_position {
-                    let delta = point - last_position;
-
-                    spawn(async move {
-                        _ = socket.send(ClientEvent::MouseMove { dx: delta.x, dy: delta.y }).await;
-                    });
-                }
-            },
-
-            onpointerdown: move |evt| {
-                let point = evt.element_coordinates();
-                *last_cursor_position.write() = Some(point);
+                onpointermove: pointer_move_handler,
+                onpointerdown: pointer_down_handler,
+                onclick: pointer_click_handler
             }
         }
     }
@@ -57,6 +103,8 @@ pub fn MouseArea() -> Element {
 #[derive(Serialize, Deserialize, Debug)]
 enum ClientEvent {
     MouseMove { dx: f64, dy: f64 },
+    Click,
+    KeyEvent { key: String, is_pressed: bool },
 }
 
 #[derive(Serialize, Deserialize, Debug)]
@@ -65,15 +113,25 @@ enum ServerEvent {
 }
 
 #[expect(clippy::unused_async)]
-#[get("/api/mouse_move_ws", mouse_service: State<crate::server::mouse_service::MouseService>)]
+#[get("/api/mouse_move_ws", mouse_service: State<crate::server::input_proxy_service::InputProxyService>)]
 async fn mouse_move(
     options: WebSocketOptions
 ) -> Result<Websocket<ClientEvent, ServerEvent, CborEncoding>> {
     Ok(options.on_upgrade(move |mut socket| async move {
         _ = socket.send(ServerEvent::Ping).await;
 
-        while let Ok(ClientEvent::MouseMove { dx, dy }) = socket.recv().await {
-            mouse_service.move_mouse(dx, dy).await;
+        while let Ok(event) = socket.recv().await {
+            match event {
+                ClientEvent::MouseMove { dx, dy } => {
+                    mouse_service.move_mouse(dx, dy).await;
+                },
+                ClientEvent::Click => {
+                    mouse_service.click().await;
+                },
+                ClientEvent::KeyEvent { key, is_pressed } => {
+                    mouse_service.key_event(key, is_pressed).await;
+                },
+            }
         }
     }))
 }

crates/cursor-move-webapp/src/main.rs

@@ -30,11 +30,11 @@ fn main() {
     // before launching our app.
     #[cfg(feature = "server")]
     dioxus::serve(|| async move {
-        use crate::server::mouse_service::MouseService;
+        use crate::server::input_proxy_service::InputProxyService;
         use dioxus::server::axum::Extension;
 
         let router = dioxus::server::router(App);
-        let router = router.layer(Extension(MouseService::start()));
+        let router = router.layer(Extension(InputProxyService::start()));
 
         Ok(router)
     });

crates/cursor-move-webapp/src/server/input_proxy_service.rs

@@ -0,0 +1,408 @@
+use std::{io::Write, str::FromStr, sync::Arc, time::Duration};
+
+use dioxus::{
+    fullstack::{FullstackContext, extract::FromRef},
+    html::Key,
+    logger::tracing,
+};
+use memfile::MemFile;
+use rustix::time::{ClockId, clock_gettime};
+use tokio::sync::Mutex;
+use wayland_client::{
+    Connection, Dispatch, EventQueue, Proxy, QueueHandle,
+    protocol::{
+        wl_pointer::ButtonState,
+        wl_registry::{Event, WlRegistry},
+        wl_seat::{self, WlSeat},
+    },
+};
+use wayland_protocols_misc::zwp_virtual_keyboard_v1::client::{
+    zwp_virtual_keyboard_manager_v1::ZwpVirtualKeyboardManagerV1,
+    zwp_virtual_keyboard_v1::ZwpVirtualKeyboardV1,
+};
+use wayland_protocols_wlr::virtual_pointer::v1::client::{
+    zwlr_virtual_pointer_manager_v1::ZwlrVirtualPointerManagerV1,
+    zwlr_virtual_pointer_v1::ZwlrVirtualPointerV1,
+};
+
+use crate::server::keymap;
+
+// https://github.com/torvalds/linux/blob/master/include/uapi/linux/input-event-codes.h
+const BUTTON_LEFT: u32 = 0x110;
+const BTN_RIGHT: u32 = 0x111;
+const BTN_MIDDLE: u32 = 0x112;
+
+// https://wayland.app/protocols/wayland#wl_keyboard:enum:keymap_format
+const NO_KEYMAP: u32 = 0;
+const XKB_V1: u32 = 1;
+
+pub fn get_wayland_timestamp() -> u32 {
+    let ts = clock_gettime(ClockId::Monotonic);
+    u32::try_from(
+        Duration::new(
+            u64::try_from(ts.tv_sec).unwrap(),
+            u32::try_from(ts.tv_nsec).unwrap(),
+        )
+        .as_millis(),
+    )
+    .unwrap()
+}
+
+#[derive(Clone)]
+pub struct InputProxyService {
+    input_proxy_service_state: Arc<Mutex<InputProxy>>,
+}
+
+impl InputProxyService {
+    pub fn start() -> Self {
+        Self {
+            input_proxy_service_state: Arc::new(Mutex::new(InputProxy::new())),
+        }
+    }
+
+    pub async fn move_mouse(
+        &self,
+        dx: f64,
+        dy: f64,
+    ) {
+        let guard = self.input_proxy_service_state.lock().await;
+        if let Some(pointer) = &guard.state.virtual_pointer() {
+            let time = get_wayland_timestamp();
+            pointer.motion(time, dx, dy);
+            pointer.frame();
+            guard.event_queue.flush().unwrap();
+        }
+    }
+
+    pub async fn click(&self) {
+        let guard = self.input_proxy_service_state.lock().await;
+        if let Some(pointer) = &guard.state.virtual_pointer() {
+            tracing::info!("Do click");
+            let time = get_wayland_timestamp();
+
+            pointer.button(time, BUTTON_LEFT, ButtonState::Pressed);
+            pointer.frame();
+            pointer.button(time, BUTTON_LEFT, ButtonState::Released);
+            pointer.frame();
+            guard.event_queue.flush().unwrap();
+        }
+    }
+
+    pub async fn key_event(
+        &self,
+        key: String,
+        is_pressed: bool,
+    ) {
+        let guard = self.input_proxy_service_state.lock().await;
+        if let Some(keyboard) = &guard.state.virtual_keyboard() {
+            let time = get_wayland_timestamp();
+
+            let key = Key::from_str(key.as_str()).unwrap().legacy_charcode();
+
+            keyboard.key(time, key, if is_pressed { 1 } else { 0 });
+
+            guard.event_queue.flush().unwrap();
+        }
+    }
+}
+
+impl FromRef<FullstackContext> for InputProxyService {
+    fn from_ref(state: &FullstackContext) -> Self {
+        state.extension::<Self>().unwrap()
+    }
+}
+
+pub struct InputProxy {
+    state: InputProxyServiceState,
+    event_queue: EventQueue<InputProxyServiceState>,
+}
+
+impl InputProxy {
+    pub fn new() -> Self {
+        let connection = Connection::connect_to_env().unwrap();
+        let display = connection.display();
+
+        let mut event_queue = connection.new_event_queue();
+        let queue_handle = event_queue.handle();
+        let _ = display.get_registry(&queue_handle, ());
+
+        let mut input_proxy_service_state = InputProxyServiceState::new(queue_handle);
+        event_queue
+            .roundtrip(&mut input_proxy_service_state)
+            .unwrap();
+
+        Self {
+            state: input_proxy_service_state,
+            event_queue,
+        }
+    }
+}
+
+struct Keymap {
+    file: MemFile,
+    size: u32,
+}
+
+enum InputProxyServiceState {
+    Incomplete {
+        queue_handle: QueueHandle<Self>,
+        seat: Option<WlSeat>,
+        virtual_pointer_manager: Option<ZwlrVirtualPointerManagerV1>,
+        virtual_keyboard_manager: Option<ZwpVirtualKeyboardManagerV1>,
+        keymap: Keymap,
+    },
+    Running {
+        seat: WlSeat,
+        virtual_pointer_manager: ZwlrVirtualPointerManagerV1,
+        virtual_keyboard_manager: ZwpVirtualKeyboardManagerV1,
+
+        virtual_pointer: ZwlrVirtualPointerV1,
+        virtual_keyboard: ZwpVirtualKeyboardV1,
+    },
+}
+impl InputProxyServiceState {
+    fn new(queue_handle: QueueHandle<Self>) -> Self {
+        let filestr = keymap::KEYMAP;
+        let mut file = memfile::CreateOptions::new().create("keymap").unwrap();
+        file.write_all(filestr.as_bytes()).unwrap();
+
+        let keymap = Keymap {
+            file,
+            size: u32::try_from(filestr.len()).unwrap(),
+        };
+
+        Self::Incomplete {
+            queue_handle,
+            keymap,
+            seat: None,
+            virtual_pointer_manager: None,
+            virtual_keyboard_manager: None,
+        }
+    }
+
+    const fn virtual_keyboard(&self) -> Option<&ZwpVirtualKeyboardV1> {
+        match self {
+            Self::Running {
+                virtual_keyboard, ..
+            } => Some(virtual_keyboard),
+            Self::Incomplete { .. } => None,
+        }
+    }
+
+    const fn virtual_pointer(&self) -> Option<&ZwlrVirtualPointerV1> {
+        match self {
+            Self::Running {
+                virtual_pointer, ..
+            } => Some(virtual_pointer),
+            Self::Incomplete { .. } => None,
+        }
+    }
+
+    pub fn set_seat(
+        &mut self,
+        seat: WlSeat,
+    ) {
+        if let Self::Incomplete {
+            seat: existing @ None,
+            ..
+        } = self
+        {
+            *existing = Some(seat);
+            tracing::info!("Obtained Seat!");
+            self.try_upgrade();
+        } else {
+            tracing::info!("Received duplicate wl_seat");
+        }
+    }
+
+    pub fn set_virtual_pointer_manager(
+        &mut self,
+        virtual_pointer_manager: ZwlrVirtualPointerManagerV1,
+    ) {
+        if let Self::Incomplete {
+            virtual_pointer_manager: existing @ None,
+            ..
+        } = self
+        {
+            *existing = Some(virtual_pointer_manager);
+            tracing::info!("Obtained Virtual Pointer Manager!");
+            self.try_upgrade();
+        } else {
+            tracing::info!("Received duplicate ZwlrVirtualPointerManagerV1");
+        }
+    }
+
+    pub fn set_virtual_keyboard_manager(
+        &mut self,
+        virtual_keyboard_manager: ZwpVirtualKeyboardManagerV1,
+    ) {
+        if let Self::Incomplete {
+            virtual_keyboard_manager: existing @ None,
+            ..
+        } = self
+        {
+            *existing = Some(virtual_keyboard_manager);
+            tracing::info!("Obtained Virtual Keyboard Manager!");
+            self.try_upgrade();
+        } else {
+            tracing::info!("Received duplicate ZwpVirtualKeyboardManagerV1");
+        }
+    }
+
+    pub fn try_upgrade(&mut self) {
+        if let Self::Incomplete {
+            queue_handle,
+            keymap,
+            seat: oseat @ Some(..),
+            virtual_pointer_manager: ovpm @ Some(..),
+            virtual_keyboard_manager: ovkm @ Some(..),
+        } = self
+        {
+            let virtual_keyboard = ovkm.as_ref().unwrap().create_virtual_keyboard(
+                oseat.as_ref().unwrap(),
+                queue_handle,
+                (),
+            );
+
+            virtual_keyboard.keymap(XKB_V1, keymap.file.as_fd(), keymap.size);
+
+            let virtual_pointer = ovpm.as_ref().unwrap().create_virtual_pointer(
+                Some(oseat.as_ref().unwrap()),
+                queue_handle,
+                (),
+            );
+            tracing::info!("InputProxyServiceState upgraded to running");
+            *self = Self::Running {
+                seat: oseat.take().unwrap(),
+                virtual_pointer_manager: ovpm.take().unwrap(),
+                virtual_keyboard_manager: ovkm.take().unwrap(),
+                virtual_pointer,
+                virtual_keyboard,
+            };
+        }
+    }
+}
+
+impl Dispatch<ZwlrVirtualPointerV1, ()> for InputProxyServiceState {
+    fn event(
+        _state: &mut Self,
+        _proxy: &ZwlrVirtualPointerV1,
+        _event: <ZwlrVirtualPointerV1 as Proxy>::Event,
+        _data: &(),
+        _conn: &Connection,
+        _qhandle: &QueueHandle<Self>,
+    ) {
+        // No events for ZwlrVirtualPointerV1: https://wayland.app/protocols/wlr-virtual-pointer-unstable-v1
+        tracing::warn!("Unknown event received from ZwlrVirtualPointerV1");
+    }
+}
+
+impl Dispatch<ZwpVirtualKeyboardV1, ()> for InputProxyServiceState {
+    fn event(
+        _state: &mut Self,
+        _proxy: &ZwpVirtualKeyboardV1,
+        _event: <ZwpVirtualKeyboardV1 as Proxy>::Event,
+        _data: &(),
+        _conn: &Connection,
+        _qhandle: &QueueHandle<Self>,
+    ) {
+        // No events for ZwpVirtualKeyboardV1: https://wayland.app/protocols/virtual-keyboard-unstable-v1
+        tracing::warn!("Unknown event received from ZwpVirtualKeyboardV1");
+    }
+}
+
+impl Dispatch<ZwlrVirtualPointerManagerV1, ()> for InputProxyServiceState {
+    fn event(
+        _state: &mut Self,
+        _proxy: &ZwlrVirtualPointerManagerV1,
+        _event: <ZwlrVirtualPointerManagerV1 as Proxy>::Event,
+        _data: &(),
+        _conn: &Connection,
+        _qhandle: &QueueHandle<Self>,
+    ) {
+        // No events for ZwlrVirtualPointerManagerV1: https://wayland.app/protocols/wlr-virtual-pointer-unstable-v1
+        tracing::warn!("Unknown event received from ZwlrVirtualPointerManagerV1");
+    }
+}
+
+impl Dispatch<ZwpVirtualKeyboardManagerV1, ()> for InputProxyServiceState {
+    fn event(
+        _state: &mut Self,
+        _proxy: &ZwpVirtualKeyboardManagerV1,
+        _event: <ZwpVirtualKeyboardManagerV1 as Proxy>::Event,
+        _data: &(),
+        _conn: &Connection,
+        _qhandle: &QueueHandle<Self>,
+    ) {
+        // No events for ZwpVirtualKeyboardManagerV1: https://wayland.app/protocols/virtual-keyboard-unstable-v1
+        tracing::warn!("Unknown event received from ZwpVirtualKeyboardManagerV1");
+    }
+}
+
+impl Dispatch<WlSeat, ()> for InputProxyServiceState {
+    fn event(
+        _state: &mut Self,
+        _proxy: &WlSeat,
+        event: <WlSeat as Proxy>::Event,
+        _data: &(),
+        _conn: &Connection,
+        _qhandle: &QueueHandle<Self>,
+    ) {
+        match event {
+            wl_seat::Event::Capabilities { capabilities } => {
+                tracing::info!("WlSeat capabilities: {:?}", capabilities);
+            },
+            wl_seat::Event::Name { name } => {
+                tracing::info!("WlSeat name: {:?}", name);
+            },
+            _ => {
+                tracing::warn!("Unknown event received from WlSeat");
+            },
+        }
+    }
+}
+
+impl Dispatch<WlRegistry, ()> for InputProxyServiceState {
+    fn event(
+        app_data: &mut Self,
+        registry: &WlRegistry,
+        event: Event,
+        _udata: &(),
+        _conn: &Connection,
+        queue_handle: &QueueHandle<Self>,
+    ) {
+        match event {
+            Event::Global {
+                name,
+                interface,
+                version,
+            } => match interface.as_str() {
+                "wl_seat" => {
+                    let seat = registry.bind::<WlSeat, _, _>(name, version, queue_handle, ());
+                    app_data.set_seat(seat);
+                },
+                "zwlr_virtual_pointer_manager_v1" => {
+                    let manager = registry.bind::<ZwlrVirtualPointerManagerV1, _, _>(
+                        name,
+                        version,
+                        queue_handle,
+                        (),
+                    );
+                    app_data.set_virtual_pointer_manager(manager);
+                },
+                "zwp_virtual_keyboard_manager_v1" => {
+                    let manager = registry.bind::<ZwpVirtualKeyboardManagerV1, _, _>(
+                        name,
+                        version,
+                        queue_handle,
+                        (),
+                    );
+                    app_data.set_virtual_keyboard_manager(manager);
+                },
+                _ => {},
+            },
+            Event::GlobalRemove { .. } => todo!(),
+            _ => todo!(),
+        }
+    }
+}

crates/cursor-move-webapp/src/server/mod.rs

@@ -1 +1,2 @@
-pub mod mouse_service;
+pub mod input_proxy_service;
+pub mod keymap;

crates/cursor-move-webapp/src/server/mouse_service.rs

@@ -1,143 +0,0 @@
-use std::{
-    sync::Arc,
-    time::{SystemTime, UNIX_EPOCH},
-};
-
-use dioxus::{
-    fullstack::{FullstackContext, extract::FromRef},
-    logger::tracing,
-};
-use tokio::sync::Mutex;
-use wayland_client::{Connection, Dispatch, EventQueue, Proxy, QueueHandle, protocol::wl_registry};
-use wayland_protocols_wlr::virtual_pointer::v1::client::{
-    zwlr_virtual_pointer_manager_v1::ZwlrVirtualPointerManagerV1,
-    zwlr_virtual_pointer_v1::ZwlrVirtualPointerV1,
-};
-
-#[derive(Clone)]
-pub struct MouseService {
-    input_proxy_service_state: Arc<Mutex<InputProxy>>,
-}
-
-impl MouseService {
-    pub fn start() -> Self {
-        Self {
-            input_proxy_service_state: Arc::new(Mutex::new(InputProxy::new())),
-        }
-    }
-
-    pub async fn move_mouse(
-        &self,
-        dx: f64,
-        dy: f64,
-    ) {
-        let guard = self.input_proxy_service_state.lock().await;
-        if let Some(pointer) = &guard.state.virtual_pointer {
-            let time = SystemTime::now()
-                .duration_since(UNIX_EPOCH)
-                .unwrap()
-                .as_millis();
-
-            pointer.motion(time as u32, dx, dy);
-            pointer.frame();
-            guard.event_queue.flush().unwrap();
-        }
-    }
-}
-
-impl FromRef<FullstackContext> for MouseService {
-    fn from_ref(state: &FullstackContext) -> Self {
-        state.extension::<Self>().unwrap()
-    }
-}
-
-pub struct InputProxy {
-    state: InputProxyServiceState,
-    event_queue: EventQueue<InputProxyServiceState>,
-}
-
-impl InputProxy {
-    pub fn new() -> Self {
-        let connection = Connection::connect_to_env().unwrap();
-        let display = connection.display();
-
-        let mut event_queue = connection.new_event_queue();
-        let queue_handle = event_queue.handle();
-        let _ = display.get_registry(&queue_handle, ());
-
-        let mut input_proxy_service_state = InputProxyServiceState::default();
-        event_queue
-            .roundtrip(&mut input_proxy_service_state)
-            .unwrap();
-
-        Self {
-            state: input_proxy_service_state,
-            event_queue,
-        }
-    }
-}
-
-#[derive(Default)]
-struct InputProxyServiceState {
-    virtual_pointer: Option<ZwlrVirtualPointerV1>,
-}
-
-impl Dispatch<ZwlrVirtualPointerV1, ()> for InputProxyServiceState {
-    fn event(
-        _state: &mut Self,
-        _proxy: &ZwlrVirtualPointerV1,
-        _event: <ZwlrVirtualPointerV1 as Proxy>::Event,
-        _data: &(),
-        _conn: &Connection,
-        _qhandle: &QueueHandle<Self>,
-    ) {
-        tracing::info!("VPointerData");
-    }
-}
-
-impl Dispatch<ZwlrVirtualPointerManagerV1, ()> for InputProxyServiceState {
-    fn event(
-        _state: &mut Self,
-        _proxy: &ZwlrVirtualPointerManagerV1,
-        _event: <ZwlrVirtualPointerManagerV1 as Proxy>::Event,
-        _data: &(),
-        _conn: &Connection,
-        _qhandle: &QueueHandle<Self>,
-    ) {
-        tracing::info!("ZwlrEvent");
-    }
-}
-
-impl Dispatch<wl_registry::WlRegistry, ()> for InputProxyServiceState {
-    fn event(
-        app_data: &mut Self,
-        registry: &wl_registry::WlRegistry,
-        event: wl_registry::Event,
-        _udata: &(),
-        _conn: &Connection,
-        queue_handle: &QueueHandle<Self>,
-    ) {
-        println!("WlRegistry Event");
-        if let wl_registry::Event::Global {
-            name,
-            interface,
-            version,
-        } = event
-            && interface == "zwlr_virtual_pointer_manager_v1"
-        {
-            app_data.virtual_pointer.get_or_insert_with(|| {
-                let manager = registry.bind::<ZwlrVirtualPointerManagerV1, _, _>(
-                    name,
-                    version,
-                    queue_handle,
-                    (),
-                );
-
-                let pointer = manager.create_virtual_pointer(None, queue_handle, ());
-
-                println!("Virtual pointer manager created");
-                pointer
-            });
-        }
-    }
-}