From 2e1260f0c2d5a45ce7cc78b3875aa31389eab179 Mon Sep 17 00:00:00 2001 From: LemonNeko Date: Fri, 6 Jun 2025 16:32:29 +0800 Subject: [PATCH] feat(stage-tamagotchi): live2d model look at mouse position (#194) * chore: set rust-analyzer path * refactor: let frontend to check if cursor is inside window * feat(stage-tamagotchi): live2d model look at mouse position * chore: improve DX * fix(stage-web): live2d model look at mouse position * fix: typo * fix: keep necessary comments * fix: typecheck --- .vscode/settings.json | 1 + .../src-tauri/src/app_click_through/mod.rs | 1 + .../src/app_click_through/native_macos.rs | 69 +++++++++---------- .../src/app_click_through/native_windows.rs | 56 ++++++++------- .../src-tauri/src/app_click_through/state.rs | 13 ---- .../src-tauri/src/app_click_through/types.rs | 19 +++++ .../src-tauri/src/commands.rs | 6 ++ apps/stage-tamagotchi/src-tauri/src/lib.rs | 35 +++------- apps/stage-tamagotchi/src/pages/index.vue | 64 +++++++++++++++-- .../src/pages/settings/models/index.vue | 11 ++- apps/stage-web/src/pages/index.vue | 11 ++- .../src/pages/settings/models/index.vue | 11 ++- .../stage-ui/src/components/Live2D/Model.vue | 13 +++- .../stage-ui/src/components/Scenes/Live2D.vue | 3 +- .../stage-ui/src/components/Widgets/Stage.vue | 6 +- 15 files changed, 206 insertions(+), 113 deletions(-) create mode 100644 apps/stage-tamagotchi/src-tauri/src/app_click_through/types.rs diff --git a/.vscode/settings.json b/.vscode/settings.json index 1c6ffd46d..fec1f6a39 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -13,6 +13,7 @@ "rust-analyzer.cargo.extraEnv": { "MACOSX_DEPLOYMENT_TARGET": "10.13" }, + "rust-analyzer.cargo.targetDir": "target/rust-analyzer", // Disable the default formatter "prettier.enable": false, diff --git a/apps/stage-tamagotchi/src-tauri/src/app_click_through/mod.rs b/apps/stage-tamagotchi/src-tauri/src/app_click_through/mod.rs index b3362a193..12c860412 100644 --- a/apps/stage-tamagotchi/src-tauri/src/app_click_through/mod.rs +++ b/apps/stage-tamagotchi/src-tauri/src/app_click_through/mod.rs @@ -1,3 +1,4 @@ pub mod native_macos; pub mod native_windows; pub mod state; +pub mod types; diff --git a/apps/stage-tamagotchi/src-tauri/src/app_click_through/native_macos.rs b/apps/stage-tamagotchi/src-tauri/src/app_click_through/native_macos.rs index 4f42ca82b..f9eec9881 100644 --- a/apps/stage-tamagotchi/src-tauri/src/app_click_through/native_macos.rs +++ b/apps/stage-tamagotchi/src-tauri/src/app_click_through/native_macos.rs @@ -1,27 +1,14 @@ #[cfg(target_os = "macos")] use objc2::{class, msg_send}; -/// Get cursor position relative to the window #[cfg(target_os = "macos")] -pub async fn is_cursor_in_window(window: &tauri::Window) -> bool { - use objc2_foundation::{NSPoint, NSRect}; +use super::types::{Point, Size, WindowFrame}; + +#[cfg(target_os = "macos")] +pub fn get_window_frame(window: &tauri::Window) -> WindowFrame { + use objc2_foundation::NSRect; unsafe { - // Get cursor position in screen coordinates (macOS coordinates - origin at bottom left) - let mouse_location: NSPoint = msg_send![class!(NSEvent), mouseLocation]; - - // Get all screens - // - // We need screens count because for multiple-display users, - // in macOS, the Native API returns the mouse coordinates relative to the primary - // display, for example, if we say the primary display is 1920x1080, another two lies - // on both sides of the primary display with the size of 1920x1080 too, mouse coordinates - // will be 0 at the left edge of p-display, and 1920 at the right edge of the p-display, - // -1080 at the left edge of the left-side display, and 2160 at the right edge of the right-side - // display. - let screens: *const objc2::runtime::AnyObject = msg_send![class!(NSScreen), screens]; - let screens_count: usize = msg_send![screens, count]; - // Get window position and size from Tauri // Get the NSWindow from Tauri window to access native properties // @@ -33,26 +20,36 @@ pub async fn is_cursor_in_window(window: &tauri::Window) -> bool { // // We need to get the window's frame in macOS coordinates (bottom-left origin) // and check if the cursor is inside that frame. + // + // For multiple-display users, + // in macOS, the Native API returns the mouse coordinates relative to the primary + // display, for example, if we say the primary display is 1920x1080, another two lies + // on both sides of the primary display with the size of 1920x1080 too, mouse coordinates + // will be 0 at the left edge of p-display, and 1920 at the right edge of the p-display, + // -1080 at the left edge of the left-side display, and 2160 at the right edge of the right-side + // display. let ns_window: *mut objc2::runtime::AnyObject = window.ns_window().unwrap().cast(); let window_frame: NSRect = msg_send![ns_window, frame]; - - // Log all screens information - for _ in 0..screens_count { - // For debugging purpose, screen object, frame size of the screen, visible frame size of the screen, - // and the scale factor of the screen can be obtained as follows: - // - // let screen: *const objc2::runtime::AnyObject = msg_send![screens, objectAtIndex: i]; - // let frame: NSRect = msg_send![screen, frame]; - // let visible_frame: NSRect = msg_send![screen, visibleFrame]; - // let scale_factor: f64 = msg_send![screen, backingScaleFactor]; - - // Check if mouse is inside our window's frame - let is_inside = mouse_location.x >= window_frame.origin.x && mouse_location.x <= (window_frame.origin.x + window_frame.size.width) && mouse_location.y >= window_frame.origin.y && mouse_location.y <= (window_frame.origin.y + window_frame.size.height); - - if is_inside { - return true; - } + WindowFrame { + origin: Point { + x: window_frame.origin.x, + y: window_frame.origin.y, + }, + size: Size { + width: window_frame.size.width, + height: window_frame.size.height, + }, } } - false +} + +#[cfg(target_os = "macos")] +pub fn get_mouse_location() -> Point { + use objc2_foundation::NSPoint; + + unsafe { + // Get cursor position in screen coordinates (macOS coordinates - origin at bottom left) + let mouse_location: NSPoint = msg_send![class!(NSEvent), mouseLocation]; + Point { x: mouse_location.x, y: mouse_location.y } + } } diff --git a/apps/stage-tamagotchi/src-tauri/src/app_click_through/native_windows.rs b/apps/stage-tamagotchi/src-tauri/src/app_click_through/native_windows.rs index 737ada8d0..2cc0f9922 100644 --- a/apps/stage-tamagotchi/src-tauri/src/app_click_through/native_windows.rs +++ b/apps/stage-tamagotchi/src-tauri/src/app_click_through/native_windows.rs @@ -1,35 +1,45 @@ -/// Get cursor position relative to the window #[cfg(target_os = "windows")] -pub async fn is_cursor_in_window(window: &tauri::Window) -> bool { - use windows::Win32::{ - Foundation::{POINT, RECT}, - UI::WindowsAndMessaging::{GetCursorPos, GetWindowRect}, - }; +use super::types::{Point, Size, WindowFrame}; + +#[cfg(target_os = "windows")] +pub fn get_window_frame(window: &tauri::Window) -> WindowFrame { + use windows::Win32::{Foundation::RECT, UI::WindowsAndMessaging::GetWindowRect}; unsafe { let hwnd = window.hwnd().unwrap(); + let mut rect = RECT::default(); + + // Get window rectangle + if GetWindowRect(hwnd, &mut rect).is_ok() { + // Return the coordinates as (left, top, right, bottom) + return WindowFrame { + origin: Point { x: rect.left.into(), y: rect.top.into() }, + size: Size { + width: (rect.right - rect.left).into(), + height: (rect.bottom - rect.top).into(), + }, + }; + } + } + + WindowFrame { + origin: Point { x: 0.0, y: 0.0 }, + size: Size { width: 0.0, height: 0.0 }, + } // Default if unable to get window frame +} + +#[cfg(target_os = "windows")] +pub fn get_mouse_location() -> Point { + use windows::Win32::{Foundation::POINT, UI::WindowsAndMessaging::GetCursorPos}; + + unsafe { let mut cursor_pos = POINT::default(); - let mut window_rect = RECT::default(); // Get cursor position in screen coordinates if GetCursorPos(&mut cursor_pos).is_ok() { - // Get window rectangle - if GetWindowRect(hwnd, &mut window_rect).is_ok() { - // Check if cursor is inside window bounds - return cursor_pos.x >= window_rect.left && cursor_pos.x <= window_rect.right && cursor_pos.y >= window_rect.top && cursor_pos.y <= window_rect.bottom; - } + return Point { x: cursor_pos.x.into(), y: cursor_pos.y.into() }; } - - false } -} -/// Check if modifier key is pressed (Alt key) -#[cfg(target_os = "windows")] -pub fn is_modifier_pressed() -> bool { - use windows::Win32::UI::Input::KeyboardAndMouse::{GetAsyncKeyState, VK_MENU}; - unsafe { - // Check if Alt key is pressed (VK_MENU is the virtual key code for Alt) - GetAsyncKeyState(VK_MENU.0 as i32) < 0 - } + Point { x: 0.0, y: 0.0 } // Default if unable to get cursor position } diff --git a/apps/stage-tamagotchi/src-tauri/src/app_click_through/state.rs b/apps/stage-tamagotchi/src-tauri/src/app_click_through/state.rs index 4683d7a9f..cdd11f776 100644 --- a/apps/stage-tamagotchi/src-tauri/src/app_click_through/state.rs +++ b/apps/stage-tamagotchi/src-tauri/src/app_click_through/state.rs @@ -9,19 +9,6 @@ use tauri::{Emitter, Manager}; pub struct WindowClickThroughState { pub monitoring_enabled: Arc, pub enabled: Arc, - pub cursor_inside: Arc, -} - -pub fn set_cursor_inside(window: &tauri::Window, is_inside: bool) -> Result<(), String> { - let state = window.state::(); - - state.cursor_inside.store(is_inside, Ordering::Relaxed); - - window.set_ignore_cursor_events(is_inside).map_err(|e| format!("Failed to set click-through state: {e}"))?; - - let _ = window.emit("tauri-app:window-click-through:is-inside", is_inside); - - Ok(()) } pub fn set_click_through_enabled(window: &tauri::Window, enabled: bool) -> Result<(), String> { diff --git a/apps/stage-tamagotchi/src-tauri/src/app_click_through/types.rs b/apps/stage-tamagotchi/src-tauri/src/app_click_through/types.rs new file mode 100644 index 000000000..0a1c66bc8 --- /dev/null +++ b/apps/stage-tamagotchi/src-tauri/src/app_click_through/types.rs @@ -0,0 +1,19 @@ +use serde::Serialize; + +#[derive(Debug, Clone, Copy, Serialize)] +pub struct Point { + pub x: f64, + pub y: f64, +} + +#[derive(Debug, Clone, Copy, Serialize)] +pub struct Size { + pub width: f64, + pub height: f64, +} + +#[derive(Debug, Clone, Copy, Serialize)] +pub struct WindowFrame { + pub origin: Point, + pub size: Size, +} diff --git a/apps/stage-tamagotchi/src-tauri/src/commands.rs b/apps/stage-tamagotchi/src-tauri/src/commands.rs index a094b7b28..bc081bad7 100644 --- a/apps/stage-tamagotchi/src-tauri/src/commands.rs +++ b/apps/stage-tamagotchi/src-tauri/src/commands.rs @@ -25,3 +25,9 @@ pub async fn open_chat_window(app: tauri::AppHandle) -> Result<(), tauri::Error> app_windows::chat::new_chat_window(&app)?; Ok(()) } + +#[tauri::command] +pub fn debug_println(msg: serde_json::Value) -> Result<(), tauri::Error> { + println!("{}", msg); + Ok(()) +} diff --git a/apps/stage-tamagotchi/src-tauri/src/lib.rs b/apps/stage-tamagotchi/src-tauri/src/lib.rs index 1ba8d3436..814243e58 100644 --- a/apps/stage-tamagotchi/src-tauri/src/lib.rs +++ b/apps/stage-tamagotchi/src-tauri/src/lib.rs @@ -17,16 +17,15 @@ mod app_windows; mod commands; #[cfg(target_os = "macos")] -use app_click_through::native_macos::is_cursor_in_window; +use app_click_through::native_macos::{get_mouse_location, get_window_frame}; #[cfg(target_os = "windows")] -use app_click_through::native_windows::is_cursor_in_window; -use app_click_through::state::{set_click_through_enabled, set_cursor_inside, WindowClickThroughState}; +use app_click_through::native_windows::{get_mouse_location, get_window_frame}; +use app_click_through::state::{set_click_through_enabled, WindowClickThroughState}; #[tauri::command] -async fn start_monitor_for_clicking_through(window: tauri::Window) -> Result<(), String> { +async fn start_monitor(window: tauri::Window) -> Result<(), String> { let window = window; let state = window.state::(); - let enabled = state.enabled.clone(); let monitoring_enabled = state.monitoring_enabled.clone(); // Already monitoring? @@ -47,29 +46,14 @@ async fn start_monitor_for_clicking_through(window: tauri::Window) -> Result<(), break; } - // If is disabled already, skip until next check - if !enabled.load(Ordering::Relaxed) { - continue; - } - #[cfg(target_os = "macos")] { - let cursor_inside = is_cursor_in_window(&window).await; - - // Only allow disabling click-through when: - // 1. Cursor is OUTSIDE the window AND - // 2. Modifier key is pressed - let _ = set_cursor_inside(&window, cursor_inside); + let _ = window.emit("tauri-app:window-click-through:position-cursor-and-window-frame", (get_mouse_location(), get_window_frame(&window))); } #[cfg(target_os = "windows")] { - let cursor_inside = is_cursor_in_window(&window).await; - - // Only allow disabling click-through when: - // 1. Cursor is OUTSIDE the window AND - // 2. Modifier key is pressed - let _ = set_cursor_inside(&window, cursor_inside); + let _ = window.emit("tauri-app:window-click-through:position-cursor-and-window-frame", (get_mouse_location(), get_window_frame(&window))); } } }); @@ -78,7 +62,7 @@ async fn start_monitor_for_clicking_through(window: tauri::Window) -> Result<(), } #[tauri::command] -async fn stop_monitor_for_clicking_through(window: tauri::Window) -> Result<(), String> { +async fn stop_monitor(window: tauri::Window) -> Result<(), String> { let window = window; let state = window.state::(); @@ -189,8 +173,9 @@ pub fn run() { .invoke_handler(tauri::generate_handler![ commands::open_settings_window, commands::open_chat_window, - start_monitor_for_clicking_through, - stop_monitor_for_clicking_through, + commands::debug_println, + start_monitor, + stop_monitor, start_click_through, stop_click_through, ]) diff --git a/apps/stage-tamagotchi/src/pages/index.vue b/apps/stage-tamagotchi/src/pages/index.vue index 080d6c57d..60b4ff31a 100644 --- a/apps/stage-tamagotchi/src/pages/index.vue +++ b/apps/stage-tamagotchi/src/pages/index.vue @@ -4,6 +4,7 @@ import { useMcpStore } from '@proj-airi/stage-ui/stores' import { connectServer } from '@proj-airi/tauri-plugin-mcp' import { invoke } from '@tauri-apps/api/core' import { listen } from '@tauri-apps/api/event' +import { platform } from '@tauri-apps/plugin-os' import { storeToRefs } from 'pinia' import { computed, onMounted, onUnmounted, ref } from 'vue' @@ -33,13 +34,13 @@ const modeIndicatorClass = computed(() => { }) onMounted(async () => { - await invoke('start_monitor_for_clicking_through') + await invoke('start_monitor') await startClickThrough() }) onUnmounted(async () => { await stopClickThrough() - await invoke('stop_monitor_for_clicking_through') + await invoke('stop_monitor') }) const unlisten: (() => void)[] = [] @@ -56,11 +57,44 @@ const shouldHideView = computed(() => { return isCursorInside.value && !windowStore.isControlActive && windowStore.isIgnoringMouseEvent }) +const live2dFocusAt = ref({ x: window.innerWidth / 2, y: window.innerHeight / 2 }) + +interface Point { + x: number + y: number +} + +interface Size { + width: number + height: number +} + +interface WindowFrame { + origin: Point + size: Size +} + +function onTauriPositionCursorAndWindowFrameEvent(event: { payload: [Point, WindowFrame] }) { + const [mouseLocation, windowFrame] = event.payload + isCursorInside.value = mouseLocation.x >= windowFrame.origin.x && mouseLocation.x <= windowFrame.origin.x + windowFrame.size.width && mouseLocation.y >= windowFrame.origin.y && mouseLocation.y <= windowFrame.origin.y + windowFrame.size.height + + if (platform() === 'macos') { + live2dFocusAt.value = { + x: mouseLocation.x - windowFrame.origin.x, + y: windowFrame.size.height - mouseLocation.y + windowFrame.origin.y, + } + return + } + + live2dFocusAt.value = { + x: mouseLocation.x - windowFrame.origin.x, + y: mouseLocation.y - windowFrame.origin.y, + } +} + onMounted(async () => { // Listen for click-through state changes - unlisten.push(await listen('tauri-app:window-click-through:is-inside', (event: { payload: boolean }) => { - isCursorInside.value = event.payload - })) + unlisten.push(await listen('tauri-app:window-click-through:position-cursor-and-window-frame', onTauriPositionCursorAndWindowFrameEvent)) if (connected.value) return @@ -77,7 +111,22 @@ onMounted(async () => { onUnmounted(() => { unlisten.forEach(fn => fn?.()) + unlisten.length = 0 }) + +if (import.meta.hot) { // For better DX + import.meta.hot.on('vite:beforeUpdate', () => { + unlisten.forEach(fn => fn?.()) + unlisten.length = 0 + invoke('stop_monitor') + }) + import.meta.hot.on('vite:afterUpdate', async () => { + if (unlisten.length === 0) { + unlisten.push(await listen('tauri-app:window-click-through:position-cursor-and-window-frame', onTauriPositionCursorAndWindowFrameEvent)) + } + invoke('start_monitor') + }) +}