sketch for key code transfer

This commit is contained in:
Mona Mayrhofer 2026-03-02 22:06:15 +01:00
parent 966fdbbd50
commit a2b086a779
WARNING! Although there is a key with this ID in the database it does not verify this commit! This commit is SUSPICIOUS.
GPG key ID: 374AB152BDEBA1AE
23 changed files with 2231 additions and 229 deletions

View file

@ -13,6 +13,11 @@ serde = { version = "1.0.228", features = ["derive"] }
wayland-client = { version = "0.31.12", optional = true }
wayland-protocols-wlr = { version = "0.3.10", features = ["client"], optional = true }
tokio = {version = "1.49.0", optional = true}
rustix = { version = "1.1.4", optional = true, features = ["time"] }
wayland-protocols-misc = { version = "0.3.10", features = ["client"], optional = true }
wayland-protocols = { version = "0.32.10", features = ["client", "staging"], optional = true }
xkb = {version = "0.3.0", optional = true}
memfile = {version = "0.3.2", optional = true}
[features]
default = ["web"]
@ -23,7 +28,16 @@ desktop = ["dioxus/desktop"]
# The feature that are only required for the mobile = ["dioxus/mobile"] build target should be optional and only enabled in the mobile = ["dioxus/mobile"] feature
mobile = ["dioxus/mobile"]
# The feature that are only required for the server = ["dioxus/server"] build target should be optional and only enabled in the server = ["dioxus/server"] feature
server = ["dioxus/server", "dep:wayland-client", "dep:wayland-protocols-wlr", "dep:tokio"]
server = ["dioxus/server",
"dep:wayland-client",
"dep:wayland-protocols-wlr",
"dep:wayland-protocols-misc",
"dep:wayland-protocols",
"dep:tokio",
"dep:rustix",
"dep:xkb",
"dep:memfile"
]
[lints]
workspace = true

View file

@ -7,5 +7,7 @@
& > * {
flex-grow: 1;
display: flex;
flex-direction: column;
}
}

View file

@ -2,4 +2,5 @@
background-color: white;
touch-action: none;
flex-grow: 1;
}

View file

@ -1,6 +1,9 @@
use dioxus::{
fullstack::{CborEncoding, WebSocketOptions, Websocket, extract::State, use_websocket},
html::geometry::{ElementSpace, euclid::Point2D},
html::{
geometry::{ElementSpace, euclid::Point2D},
input_data::MouseButton,
},
logger::tracing,
prelude::*,
};
@ -28,27 +31,70 @@ pub fn MouseArea() -> Element {
}
});
let pointer_move_handler = use_callback(move |evt: Event<PointerData>| {
if evt.held_buttons().contains(MouseButton::Primary) {
evt.prevent_default();
let point = evt.element_coordinates();
let last_position = last_cursor_position.write().replace(point);
if let Some(last_position) = last_position {
let delta = point - last_position;
spawn(async move {
_ = socket
.send(ClientEvent::MouseMove {
dx: delta.x,
dy: delta.y,
})
.await;
});
}
}
});
let pointer_down_handler = use_callback(move |evt: Event<PointerData>| {
let point = evt.element_coordinates();
*last_cursor_position.write() = Some(point);
});
let pointer_click_handler = use_callback(move |evt: Event<MouseData>| {
spawn(async move {
_ = socket.send(ClientEvent::Click).await;
});
});
let key_down_handler = use_callback(move |evt: Event<KeyboardData>| {
tracing::info!("Keydown");
spawn(async move {
_ = socket
.send(ClientEvent::KeyEvent {
key: evt.key().to_string(),
is_pressed: true,
})
.await;
});
});
let key_up_handler = use_callback(move |evt: Event<KeyboardData>| {
spawn(async move {
_ = socket
.send(ClientEvent::KeyEvent {
key: evt.key().to_string(),
is_pressed: false,
})
.await;
});
});
rsx! {
div {
class: Styles::mouse_area,
input { onkeydown: key_down_handler, onkeyup: key_up_handler}
div {
class: Styles::mouse_area,
onpointermove: move |evt| {
evt.prevent_default();
let point = evt.element_coordinates();
let last_position = last_cursor_position.write().replace(point);
if let Some(last_position) = last_position {
let delta = point - last_position;
spawn(async move {
_ = socket.send(ClientEvent::MouseMove { dx: delta.x, dy: delta.y }).await;
});
}
},
onpointerdown: move |evt| {
let point = evt.element_coordinates();
*last_cursor_position.write() = Some(point);
onpointermove: pointer_move_handler,
onpointerdown: pointer_down_handler,
onclick: pointer_click_handler
}
}
}
@ -57,6 +103,8 @@ pub fn MouseArea() -> Element {
#[derive(Serialize, Deserialize, Debug)]
enum ClientEvent {
MouseMove { dx: f64, dy: f64 },
Click,
KeyEvent { key: String, is_pressed: bool },
}
#[derive(Serialize, Deserialize, Debug)]
@ -65,15 +113,25 @@ enum ServerEvent {
}
#[expect(clippy::unused_async)]
#[get("/api/mouse_move_ws", mouse_service: State<crate::server::mouse_service::MouseService>)]
#[get("/api/mouse_move_ws", mouse_service: State<crate::server::input_proxy_service::InputProxyService>)]
async fn mouse_move(
options: WebSocketOptions
) -> Result<Websocket<ClientEvent, ServerEvent, CborEncoding>> {
Ok(options.on_upgrade(move |mut socket| async move {
_ = socket.send(ServerEvent::Ping).await;
while let Ok(ClientEvent::MouseMove { dx, dy }) = socket.recv().await {
mouse_service.move_mouse(dx, dy).await;
while let Ok(event) = socket.recv().await {
match event {
ClientEvent::MouseMove { dx, dy } => {
mouse_service.move_mouse(dx, dy).await;
},
ClientEvent::Click => {
mouse_service.click().await;
},
ClientEvent::KeyEvent { key, is_pressed } => {
mouse_service.key_event(key, is_pressed).await;
},
}
}
}))
}

View file

@ -30,11 +30,11 @@ fn main() {
// before launching our app.
#[cfg(feature = "server")]
dioxus::serve(|| async move {
use crate::server::mouse_service::MouseService;
use crate::server::input_proxy_service::InputProxyService;
use dioxus::server::axum::Extension;
let router = dioxus::server::router(App);
let router = router.layer(Extension(MouseService::start()));
let router = router.layer(Extension(InputProxyService::start()));
Ok(router)
});

View file

@ -0,0 +1,408 @@
use std::{io::Write, str::FromStr, sync::Arc, time::Duration};
use dioxus::{
fullstack::{FullstackContext, extract::FromRef},
html::Key,
logger::tracing,
};
use memfile::MemFile;
use rustix::time::{ClockId, clock_gettime};
use tokio::sync::Mutex;
use wayland_client::{
Connection, Dispatch, EventQueue, Proxy, QueueHandle,
protocol::{
wl_pointer::ButtonState,
wl_registry::{Event, WlRegistry},
wl_seat::{self, WlSeat},
},
};
use wayland_protocols_misc::zwp_virtual_keyboard_v1::client::{
zwp_virtual_keyboard_manager_v1::ZwpVirtualKeyboardManagerV1,
zwp_virtual_keyboard_v1::ZwpVirtualKeyboardV1,
};
use wayland_protocols_wlr::virtual_pointer::v1::client::{
zwlr_virtual_pointer_manager_v1::ZwlrVirtualPointerManagerV1,
zwlr_virtual_pointer_v1::ZwlrVirtualPointerV1,
};
use crate::server::keymap;
// https://github.com/torvalds/linux/blob/master/include/uapi/linux/input-event-codes.h
const BUTTON_LEFT: u32 = 0x110;
const BTN_RIGHT: u32 = 0x111;
const BTN_MIDDLE: u32 = 0x112;
// https://wayland.app/protocols/wayland#wl_keyboard:enum:keymap_format
const NO_KEYMAP: u32 = 0;
const XKB_V1: u32 = 1;
pub fn get_wayland_timestamp() -> u32 {
let ts = clock_gettime(ClockId::Monotonic);
u32::try_from(
Duration::new(
u64::try_from(ts.tv_sec).unwrap(),
u32::try_from(ts.tv_nsec).unwrap(),
)
.as_millis(),
)
.unwrap()
}
#[derive(Clone)]
pub struct InputProxyService {
input_proxy_service_state: Arc<Mutex<InputProxy>>,
}
impl InputProxyService {
pub fn start() -> Self {
Self {
input_proxy_service_state: Arc::new(Mutex::new(InputProxy::new())),
}
}
pub async fn move_mouse(
&self,
dx: f64,
dy: f64,
) {
let guard = self.input_proxy_service_state.lock().await;
if let Some(pointer) = &guard.state.virtual_pointer() {
let time = get_wayland_timestamp();
pointer.motion(time, dx, dy);
pointer.frame();
guard.event_queue.flush().unwrap();
}
}
pub async fn click(&self) {
let guard = self.input_proxy_service_state.lock().await;
if let Some(pointer) = &guard.state.virtual_pointer() {
tracing::info!("Do click");
let time = get_wayland_timestamp();
pointer.button(time, BUTTON_LEFT, ButtonState::Pressed);
pointer.frame();
pointer.button(time, BUTTON_LEFT, ButtonState::Released);
pointer.frame();
guard.event_queue.flush().unwrap();
}
}
pub async fn key_event(
&self,
key: String,
is_pressed: bool,
) {
let guard = self.input_proxy_service_state.lock().await;
if let Some(keyboard) = &guard.state.virtual_keyboard() {
let time = get_wayland_timestamp();
let key = Key::from_str(key.as_str()).unwrap().legacy_charcode();
keyboard.key(time, key, if is_pressed { 1 } else { 0 });
guard.event_queue.flush().unwrap();
}
}
}
impl FromRef<FullstackContext> for InputProxyService {
fn from_ref(state: &FullstackContext) -> Self {
state.extension::<Self>().unwrap()
}
}
pub struct InputProxy {
state: InputProxyServiceState,
event_queue: EventQueue<InputProxyServiceState>,
}
impl InputProxy {
pub fn new() -> Self {
let connection = Connection::connect_to_env().unwrap();
let display = connection.display();
let mut event_queue = connection.new_event_queue();
let queue_handle = event_queue.handle();
let _ = display.get_registry(&queue_handle, ());
let mut input_proxy_service_state = InputProxyServiceState::new(queue_handle);
event_queue
.roundtrip(&mut input_proxy_service_state)
.unwrap();
Self {
state: input_proxy_service_state,
event_queue,
}
}
}
struct Keymap {
file: MemFile,
size: u32,
}
enum InputProxyServiceState {
Incomplete {
queue_handle: QueueHandle<Self>,
seat: Option<WlSeat>,
virtual_pointer_manager: Option<ZwlrVirtualPointerManagerV1>,
virtual_keyboard_manager: Option<ZwpVirtualKeyboardManagerV1>,
keymap: Keymap,
},
Running {
seat: WlSeat,
virtual_pointer_manager: ZwlrVirtualPointerManagerV1,
virtual_keyboard_manager: ZwpVirtualKeyboardManagerV1,
virtual_pointer: ZwlrVirtualPointerV1,
virtual_keyboard: ZwpVirtualKeyboardV1,
},
}
impl InputProxyServiceState {
fn new(queue_handle: QueueHandle<Self>) -> Self {
let filestr = keymap::KEYMAP;
let mut file = memfile::CreateOptions::new().create("keymap").unwrap();
file.write_all(filestr.as_bytes()).unwrap();
let keymap = Keymap {
file,
size: u32::try_from(filestr.len()).unwrap(),
};
Self::Incomplete {
queue_handle,
keymap,
seat: None,
virtual_pointer_manager: None,
virtual_keyboard_manager: None,
}
}
const fn virtual_keyboard(&self) -> Option<&ZwpVirtualKeyboardV1> {
match self {
Self::Running {
virtual_keyboard, ..
} => Some(virtual_keyboard),
Self::Incomplete { .. } => None,
}
}
const fn virtual_pointer(&self) -> Option<&ZwlrVirtualPointerV1> {
match self {
Self::Running {
virtual_pointer, ..
} => Some(virtual_pointer),
Self::Incomplete { .. } => None,
}
}
pub fn set_seat(
&mut self,
seat: WlSeat,
) {
if let Self::Incomplete {
seat: existing @ None,
..
} = self
{
*existing = Some(seat);
tracing::info!("Obtained Seat!");
self.try_upgrade();
} else {
tracing::info!("Received duplicate wl_seat");
}
}
pub fn set_virtual_pointer_manager(
&mut self,
virtual_pointer_manager: ZwlrVirtualPointerManagerV1,
) {
if let Self::Incomplete {
virtual_pointer_manager: existing @ None,
..
} = self
{
*existing = Some(virtual_pointer_manager);
tracing::info!("Obtained Virtual Pointer Manager!");
self.try_upgrade();
} else {
tracing::info!("Received duplicate ZwlrVirtualPointerManagerV1");
}
}
pub fn set_virtual_keyboard_manager(
&mut self,
virtual_keyboard_manager: ZwpVirtualKeyboardManagerV1,
) {
if let Self::Incomplete {
virtual_keyboard_manager: existing @ None,
..
} = self
{
*existing = Some(virtual_keyboard_manager);
tracing::info!("Obtained Virtual Keyboard Manager!");
self.try_upgrade();
} else {
tracing::info!("Received duplicate ZwpVirtualKeyboardManagerV1");
}
}
pub fn try_upgrade(&mut self) {
if let Self::Incomplete {
queue_handle,
keymap,
seat: oseat @ Some(..),
virtual_pointer_manager: ovpm @ Some(..),
virtual_keyboard_manager: ovkm @ Some(..),
} = self
{
let virtual_keyboard = ovkm.as_ref().unwrap().create_virtual_keyboard(
oseat.as_ref().unwrap(),
queue_handle,
(),
);
virtual_keyboard.keymap(XKB_V1, keymap.file.as_fd(), keymap.size);
let virtual_pointer = ovpm.as_ref().unwrap().create_virtual_pointer(
Some(oseat.as_ref().unwrap()),
queue_handle,
(),
);
tracing::info!("InputProxyServiceState upgraded to running");
*self = Self::Running {
seat: oseat.take().unwrap(),
virtual_pointer_manager: ovpm.take().unwrap(),
virtual_keyboard_manager: ovkm.take().unwrap(),
virtual_pointer,
virtual_keyboard,
};
}
}
}
impl Dispatch<ZwlrVirtualPointerV1, ()> for InputProxyServiceState {
fn event(
_state: &mut Self,
_proxy: &ZwlrVirtualPointerV1,
_event: <ZwlrVirtualPointerV1 as Proxy>::Event,
_data: &(),
_conn: &Connection,
_qhandle: &QueueHandle<Self>,
) {
// No events for ZwlrVirtualPointerV1: https://wayland.app/protocols/wlr-virtual-pointer-unstable-v1
tracing::warn!("Unknown event received from ZwlrVirtualPointerV1");
}
}
impl Dispatch<ZwpVirtualKeyboardV1, ()> for InputProxyServiceState {
fn event(
_state: &mut Self,
_proxy: &ZwpVirtualKeyboardV1,
_event: <ZwpVirtualKeyboardV1 as Proxy>::Event,
_data: &(),
_conn: &Connection,
_qhandle: &QueueHandle<Self>,
) {
// No events for ZwpVirtualKeyboardV1: https://wayland.app/protocols/virtual-keyboard-unstable-v1
tracing::warn!("Unknown event received from ZwpVirtualKeyboardV1");
}
}
impl Dispatch<ZwlrVirtualPointerManagerV1, ()> for InputProxyServiceState {
fn event(
_state: &mut Self,
_proxy: &ZwlrVirtualPointerManagerV1,
_event: <ZwlrVirtualPointerManagerV1 as Proxy>::Event,
_data: &(),
_conn: &Connection,
_qhandle: &QueueHandle<Self>,
) {
// No events for ZwlrVirtualPointerManagerV1: https://wayland.app/protocols/wlr-virtual-pointer-unstable-v1
tracing::warn!("Unknown event received from ZwlrVirtualPointerManagerV1");
}
}
impl Dispatch<ZwpVirtualKeyboardManagerV1, ()> for InputProxyServiceState {
fn event(
_state: &mut Self,
_proxy: &ZwpVirtualKeyboardManagerV1,
_event: <ZwpVirtualKeyboardManagerV1 as Proxy>::Event,
_data: &(),
_conn: &Connection,
_qhandle: &QueueHandle<Self>,
) {
// No events for ZwpVirtualKeyboardManagerV1: https://wayland.app/protocols/virtual-keyboard-unstable-v1
tracing::warn!("Unknown event received from ZwpVirtualKeyboardManagerV1");
}
}
impl Dispatch<WlSeat, ()> for InputProxyServiceState {
fn event(
_state: &mut Self,
_proxy: &WlSeat,
event: <WlSeat as Proxy>::Event,
_data: &(),
_conn: &Connection,
_qhandle: &QueueHandle<Self>,
) {
match event {
wl_seat::Event::Capabilities { capabilities } => {
tracing::info!("WlSeat capabilities: {:?}", capabilities);
},
wl_seat::Event::Name { name } => {
tracing::info!("WlSeat name: {:?}", name);
},
_ => {
tracing::warn!("Unknown event received from WlSeat");
},
}
}
}
impl Dispatch<WlRegistry, ()> for InputProxyServiceState {
fn event(
app_data: &mut Self,
registry: &WlRegistry,
event: Event,
_udata: &(),
_conn: &Connection,
queue_handle: &QueueHandle<Self>,
) {
match event {
Event::Global {
name,
interface,
version,
} => match interface.as_str() {
"wl_seat" => {
let seat = registry.bind::<WlSeat, _, _>(name, version, queue_handle, ());
app_data.set_seat(seat);
},
"zwlr_virtual_pointer_manager_v1" => {
let manager = registry.bind::<ZwlrVirtualPointerManagerV1, _, _>(
name,
version,
queue_handle,
(),
);
app_data.set_virtual_pointer_manager(manager);
},
"zwp_virtual_keyboard_manager_v1" => {
let manager = registry.bind::<ZwpVirtualKeyboardManagerV1, _, _>(
name,
version,
queue_handle,
(),
);
app_data.set_virtual_keyboard_manager(manager);
},
_ => {},
},
Event::GlobalRemove { .. } => todo!(),
_ => todo!(),
}
}
}

File diff suppressed because it is too large Load diff

View file

@ -1 +1,2 @@
pub mod mouse_service;
pub mod input_proxy_service;
pub mod keymap;

View file

@ -1,143 +0,0 @@
use std::{
sync::Arc,
time::{SystemTime, UNIX_EPOCH},
};
use dioxus::{
fullstack::{FullstackContext, extract::FromRef},
logger::tracing,
};
use tokio::sync::Mutex;
use wayland_client::{Connection, Dispatch, EventQueue, Proxy, QueueHandle, protocol::wl_registry};
use wayland_protocols_wlr::virtual_pointer::v1::client::{
zwlr_virtual_pointer_manager_v1::ZwlrVirtualPointerManagerV1,
zwlr_virtual_pointer_v1::ZwlrVirtualPointerV1,
};
#[derive(Clone)]
pub struct MouseService {
input_proxy_service_state: Arc<Mutex<InputProxy>>,
}
impl MouseService {
pub fn start() -> Self {
Self {
input_proxy_service_state: Arc::new(Mutex::new(InputProxy::new())),
}
}
pub async fn move_mouse(
&self,
dx: f64,
dy: f64,
) {
let guard = self.input_proxy_service_state.lock().await;
if let Some(pointer) = &guard.state.virtual_pointer {
let time = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
.as_millis();
pointer.motion(time as u32, dx, dy);
pointer.frame();
guard.event_queue.flush().unwrap();
}
}
}
impl FromRef<FullstackContext> for MouseService {
fn from_ref(state: &FullstackContext) -> Self {
state.extension::<Self>().unwrap()
}
}
pub struct InputProxy {
state: InputProxyServiceState,
event_queue: EventQueue<InputProxyServiceState>,
}
impl InputProxy {
pub fn new() -> Self {
let connection = Connection::connect_to_env().unwrap();
let display = connection.display();
let mut event_queue = connection.new_event_queue();
let queue_handle = event_queue.handle();
let _ = display.get_registry(&queue_handle, ());
let mut input_proxy_service_state = InputProxyServiceState::default();
event_queue
.roundtrip(&mut input_proxy_service_state)
.unwrap();
Self {
state: input_proxy_service_state,
event_queue,
}
}
}
#[derive(Default)]
struct InputProxyServiceState {
virtual_pointer: Option<ZwlrVirtualPointerV1>,
}
impl Dispatch<ZwlrVirtualPointerV1, ()> for InputProxyServiceState {
fn event(
_state: &mut Self,
_proxy: &ZwlrVirtualPointerV1,
_event: <ZwlrVirtualPointerV1 as Proxy>::Event,
_data: &(),
_conn: &Connection,
_qhandle: &QueueHandle<Self>,
) {
tracing::info!("VPointerData");
}
}
impl Dispatch<ZwlrVirtualPointerManagerV1, ()> for InputProxyServiceState {
fn event(
_state: &mut Self,
_proxy: &ZwlrVirtualPointerManagerV1,
_event: <ZwlrVirtualPointerManagerV1 as Proxy>::Event,
_data: &(),
_conn: &Connection,
_qhandle: &QueueHandle<Self>,
) {
tracing::info!("ZwlrEvent");
}
}
impl Dispatch<wl_registry::WlRegistry, ()> for InputProxyServiceState {
fn event(
app_data: &mut Self,
registry: &wl_registry::WlRegistry,
event: wl_registry::Event,
_udata: &(),
_conn: &Connection,
queue_handle: &QueueHandle<Self>,
) {
println!("WlRegistry Event");
if let wl_registry::Event::Global {
name,
interface,
version,
} = event
&& interface == "zwlr_virtual_pointer_manager_v1"
{
app_data.virtual_pointer.get_or_insert_with(|| {
let manager = registry.bind::<ZwlrVirtualPointerManagerV1, _, _>(
name,
version,
queue_handle,
(),
);
let pointer = manager.create_virtual_pointer(None, queue_handle, ());
println!("Virtual pointer manager created");
pointer
});
}
}
}