feat(stage-tamagotchi): live2d model look at mouse position (#194)

* chore: set rust-analyzer path
* refactor: let frontend to check if cursor is inside window
* feat(stage-tamagotchi): live2d model look at mouse position
* chore: improve DX
* fix(stage-web): live2d model look at mouse position
* fix: typo
* fix: keep necessary comments
* fix: typecheck

.vscode/settings.json

@@ -13,6 +13,7 @@
   "rust-analyzer.cargo.extraEnv": {
     "MACOSX_DEPLOYMENT_TARGET": "10.13"
   },
+  "rust-analyzer.cargo.targetDir": "target/rust-analyzer",
 
   // Disable the default formatter
   "prettier.enable": false,

apps/stage-tamagotchi/src-tauri/src/app_click_through/mod.rs

@@ -1,3 +1,4 @@
 pub mod native_macos;
 pub mod native_windows;
 pub mod state;
+pub mod types;

apps/stage-tamagotchi/src-tauri/src/app_click_through/native_macos.rs

@@ -1,27 +1,14 @@
 #[cfg(target_os = "macos")]
 use objc2::{class, msg_send};
 
-/// Get cursor position relative to the window
 #[cfg(target_os = "macos")]
-pub async fn is_cursor_in_window(window: &tauri::Window) -> bool {
-  use objc2_foundation::{NSPoint, NSRect};
+use super::types::{Point, Size, WindowFrame};
+
+#[cfg(target_os = "macos")]
+pub fn get_window_frame(window: &tauri::Window) -> WindowFrame {
+  use objc2_foundation::NSRect;
 
   unsafe {
-    // Get cursor position in screen coordinates (macOS coordinates - origin at bottom left)
-    let mouse_location: NSPoint = msg_send![class!(NSEvent), mouseLocation];
-
-    // Get all screens
-    //
-    // We need screens count because for multiple-display users,
-    // in macOS, the Native API returns the mouse coordinates relative to the primary
-    // display, for example, if we say the primary display is 1920x1080, another two lies
-    // on both sides of the primary display with the size of 1920x1080 too, mouse coordinates
-    // will be 0 at the left edge of p-display, and 1920 at the right edge of the p-display,
-    // -1080 at the left edge of the left-side display, and 2160 at the right edge of the right-side
-    // display.
-    let screens: *const objc2::runtime::AnyObject = msg_send![class!(NSScreen), screens];
-    let screens_count: usize = msg_send![screens, count];
-
     // Get window position and size from Tauri
     // Get the NSWindow from Tauri window to access native properties
     //
@@ -33,26 +20,36 @@ pub async fn is_cursor_in_window(window: &tauri::Window) -> bool {
     //
     // We need to get the window's frame in macOS coordinates (bottom-left origin)
     // and check if the cursor is inside that frame.
+    //
+    // For multiple-display users,
+    // in macOS, the Native API returns the mouse coordinates relative to the primary
+    // display, for example, if we say the primary display is 1920x1080, another two lies
+    // on both sides of the primary display with the size of 1920x1080 too, mouse coordinates
+    // will be 0 at the left edge of p-display, and 1920 at the right edge of the p-display,
+    // -1080 at the left edge of the left-side display, and 2160 at the right edge of the right-side
+    // display.
     let ns_window: *mut objc2::runtime::AnyObject = window.ns_window().unwrap().cast();
     let window_frame: NSRect = msg_send![ns_window, frame];
-
-    // Log all screens information
-    for _ in 0..screens_count {
-      // For debugging purpose, screen object, frame size of the screen, visible frame size of the screen,
-      // and the scale factor of the screen can be obtained as follows:
-      //
-      // let screen: *const objc2::runtime::AnyObject = msg_send![screens, objectAtIndex: i];
-      // let frame: NSRect = msg_send![screen, frame];
-      // let visible_frame: NSRect = msg_send![screen, visibleFrame];
-      // let scale_factor: f64 = msg_send![screen, backingScaleFactor];
-
-      // Check if mouse is inside our window's frame
-      let is_inside = mouse_location.x >= window_frame.origin.x && mouse_location.x <= (window_frame.origin.x + window_frame.size.width) && mouse_location.y >= window_frame.origin.y && mouse_location.y <= (window_frame.origin.y + window_frame.size.height);
-
-      if is_inside {
-        return true;
-      }
+    WindowFrame {
+      origin: Point {
+        x: window_frame.origin.x,
+        y: window_frame.origin.y,
+      },
+      size:   Size {
+        width:  window_frame.size.width,
+        height: window_frame.size.height,
+      },
     }
   }
-  false
+}
+
+#[cfg(target_os = "macos")]
+pub fn get_mouse_location() -> Point {
+  use objc2_foundation::NSPoint;
+
+  unsafe {
+    // Get cursor position in screen coordinates (macOS coordinates - origin at bottom left)
+    let mouse_location: NSPoint = msg_send![class!(NSEvent), mouseLocation];
+    Point { x: mouse_location.x, y: mouse_location.y }
+  }
 }

apps/stage-tamagotchi/src-tauri/src/app_click_through/native_windows.rs

@@ -1,35 +1,45 @@
-/// Get cursor position relative to the window
 #[cfg(target_os = "windows")]
-pub async fn is_cursor_in_window(window: &tauri::Window) -> bool {
-  use windows::Win32::{
-    Foundation::{POINT, RECT},
-    UI::WindowsAndMessaging::{GetCursorPos, GetWindowRect},
-  };
+use super::types::{Point, Size, WindowFrame};
+
+#[cfg(target_os = "windows")]
+pub fn get_window_frame(window: &tauri::Window) -> WindowFrame {
+  use windows::Win32::{Foundation::RECT, UI::WindowsAndMessaging::GetWindowRect};
 
   unsafe {
     let hwnd = window.hwnd().unwrap();
+    let mut rect = RECT::default();
+
+    // Get window rectangle
+    if GetWindowRect(hwnd, &mut rect).is_ok() {
+      // Return the coordinates as (left, top, right, bottom)
+      return WindowFrame {
+        origin: Point { x: rect.left.into(), y: rect.top.into() },
+        size:   Size {
+          width:  (rect.right - rect.left).into(),
+          height: (rect.bottom - rect.top).into(),
+        },
+      };
+    }
+  }
+
+  WindowFrame {
+    origin: Point { x: 0.0, y: 0.0 },
+    size:   Size { width: 0.0, height: 0.0 },
+  } // Default if unable to get window frame
+}
+
+#[cfg(target_os = "windows")]
+pub fn get_mouse_location() -> Point {
+  use windows::Win32::{Foundation::POINT, UI::WindowsAndMessaging::GetCursorPos};
+
+  unsafe {
     let mut cursor_pos = POINT::default();
-    let mut window_rect = RECT::default();
 
     // Get cursor position in screen coordinates
     if GetCursorPos(&mut cursor_pos).is_ok() {
-      // Get window rectangle
-      if GetWindowRect(hwnd, &mut window_rect).is_ok() {
-        // Check if cursor is inside window bounds
-        return cursor_pos.x >= window_rect.left && cursor_pos.x <= window_rect.right && cursor_pos.y >= window_rect.top && cursor_pos.y <= window_rect.bottom;
-      }
+      return Point { x: cursor_pos.x.into(), y: cursor_pos.y.into() };
     }
-
-    false
   }
-}
 
-/// Check if modifier key is pressed (Alt key)
-#[cfg(target_os = "windows")]
-pub fn is_modifier_pressed() -> bool {
-  use windows::Win32::UI::Input::KeyboardAndMouse::{GetAsyncKeyState, VK_MENU};
-  unsafe {
-    // Check if Alt key is pressed (VK_MENU is the virtual key code for Alt)
-    GetAsyncKeyState(VK_MENU.0 as i32) < 0
-  }
+  Point { x: 0.0, y: 0.0 } // Default if unable to get cursor position
 }

apps/stage-tamagotchi/src-tauri/src/app_click_through/state.rs

@@ -9,19 +9,6 @@ use tauri::{Emitter, Manager};
 pub struct WindowClickThroughState {
   pub monitoring_enabled: Arc<AtomicBool>,
   pub enabled:            Arc<AtomicBool>,
-  pub cursor_inside:      Arc<AtomicBool>,
-}
-
-pub fn set_cursor_inside(window: &tauri::Window, is_inside: bool) -> Result<(), String> {
-  let state = window.state::<WindowClickThroughState>();
-
-  state.cursor_inside.store(is_inside, Ordering::Relaxed);
-
-  window.set_ignore_cursor_events(is_inside).map_err(|e| format!("Failed to set click-through state: {e}"))?;
-
-  let _ = window.emit("tauri-app:window-click-through:is-inside", is_inside);
-
-  Ok(())
 }
 
 pub fn set_click_through_enabled(window: &tauri::Window, enabled: bool) -> Result<(), String> {

apps/stage-tamagotchi/src-tauri/src/app_click_through/types.rs

@@ -0,0 +1,19 @@
+use serde::Serialize;
+
+#[derive(Debug, Clone, Copy, Serialize)]
+pub struct Point {
+  pub x: f64,
+  pub y: f64,
+}
+
+#[derive(Debug, Clone, Copy, Serialize)]
+pub struct Size {
+  pub width:  f64,
+  pub height: f64,
+}
+
+#[derive(Debug, Clone, Copy, Serialize)]
+pub struct WindowFrame {
+  pub origin: Point,
+  pub size:   Size,
+}

apps/stage-tamagotchi/src-tauri/src/commands.rs

@@ -25,3 +25,9 @@ pub async fn open_chat_window(app: tauri::AppHandle) -> Result<(), tauri::Error>
   app_windows::chat::new_chat_window(&app)?;
   Ok(())
 }
+
+#[tauri::command]
+pub fn debug_println(msg: serde_json::Value) -> Result<(), tauri::Error> {
+  println!("{}", msg);
+  Ok(())
+}

apps/stage-tamagotchi/src-tauri/src/lib.rs

@@ -17,16 +17,15 @@ mod app_windows;
 mod commands;
 
 #[cfg(target_os = "macos")]
-use app_click_through::native_macos::is_cursor_in_window;
+use app_click_through::native_macos::{get_mouse_location, get_window_frame};
 #[cfg(target_os = "windows")]
-use app_click_through::native_windows::is_cursor_in_window;
-use app_click_through::state::{set_click_through_enabled, set_cursor_inside, WindowClickThroughState};
+use app_click_through::native_windows::{get_mouse_location, get_window_frame};
+use app_click_through::state::{set_click_through_enabled, WindowClickThroughState};
 
 #[tauri::command]
-async fn start_monitor_for_clicking_through(window: tauri::Window) -> Result<(), String> {
+async fn start_monitor(window: tauri::Window) -> Result<(), String> {
   let window = window;
   let state = window.state::<WindowClickThroughState>();
-  let enabled = state.enabled.clone();
   let monitoring_enabled = state.monitoring_enabled.clone();
 
   // Already monitoring?
@@ -47,29 +46,14 @@ async fn start_monitor_for_clicking_through(window: tauri::Window) -> Result<(),
         break;
       }
 
-      // If is disabled already, skip until next check
-      if !enabled.load(Ordering::Relaxed) {
-        continue;
-      }
-
       #[cfg(target_os = "macos")]
       {
-        let cursor_inside = is_cursor_in_window(&window).await;
-
-        // Only allow disabling click-through when:
-        // 1. Cursor is OUTSIDE the window AND
-        // 2. Modifier key is pressed
-        let _ = set_cursor_inside(&window, cursor_inside);
+        let _ = window.emit("tauri-app:window-click-through:position-cursor-and-window-frame", (get_mouse_location(), get_window_frame(&window)));
       }
 
       #[cfg(target_os = "windows")]
       {
-        let cursor_inside = is_cursor_in_window(&window).await;
-
-        // Only allow disabling click-through when:
-        // 1. Cursor is OUTSIDE the window AND
-        // 2. Modifier key is pressed
-        let _ = set_cursor_inside(&window, cursor_inside);
+        let _ = window.emit("tauri-app:window-click-through:position-cursor-and-window-frame", (get_mouse_location(), get_window_frame(&window)));
       }
     }
   });
@@ -78,7 +62,7 @@ async fn start_monitor_for_clicking_through(window: tauri::Window) -> Result<(),
 }
 
 #[tauri::command]
-async fn stop_monitor_for_clicking_through(window: tauri::Window) -> Result<(), String> {
+async fn stop_monitor(window: tauri::Window) -> Result<(), String> {
   let window = window;
   let state = window.state::<WindowClickThroughState>();
 
@@ -189,8 +173,9 @@ pub fn run() {
     .invoke_handler(tauri::generate_handler![
       commands::open_settings_window,
       commands::open_chat_window,
-      start_monitor_for_clicking_through,
-      stop_monitor_for_clicking_through,
+      commands::debug_println,
+      start_monitor,
+      stop_monitor,
       start_click_through,
       stop_click_through,
     ])

apps/stage-tamagotchi/src/pages/index.vue

@@ -4,6 +4,7 @@ import { useMcpStore } from '@proj-airi/stage-ui/stores'
 import { connectServer } from '@proj-airi/tauri-plugin-mcp'
 import { invoke } from '@tauri-apps/api/core'
 import { listen } from '@tauri-apps/api/event'
+import { platform } from '@tauri-apps/plugin-os'
 import { storeToRefs } from 'pinia'
 import { computed, onMounted, onUnmounted, ref } from 'vue'
 
@@ -33,13 +34,13 @@ const modeIndicatorClass = computed(() => {
 })
 
 onMounted(async () => {
-  await invoke('start_monitor_for_clicking_through')
+  await invoke('start_monitor')
   await startClickThrough()
 })
 
 onUnmounted(async () => {
   await stopClickThrough()
-  await invoke('stop_monitor_for_clicking_through')
+  await invoke('stop_monitor')
 })
 
 const unlisten: (() => void)[] = []
@@ -56,11 +57,44 @@ const shouldHideView = computed(() => {
   return isCursorInside.value && !windowStore.isControlActive && windowStore.isIgnoringMouseEvent
 })
 
+const live2dFocusAt = ref<Point>({ x: window.innerWidth / 2, y: window.innerHeight / 2 })
+
+interface Point {
+  x: number
+  y: number
+}
+
+interface Size {
+  width: number
+  height: number
+}
+
+interface WindowFrame {
+  origin: Point
+  size: Size
+}
+
+function onTauriPositionCursorAndWindowFrameEvent(event: { payload: [Point, WindowFrame] }) {
+  const [mouseLocation, windowFrame] = event.payload
+  isCursorInside.value = mouseLocation.x >= windowFrame.origin.x && mouseLocation.x <= windowFrame.origin.x + windowFrame.size.width && mouseLocation.y >= windowFrame.origin.y && mouseLocation.y <= windowFrame.origin.y + windowFrame.size.height
+
+  if (platform() === 'macos') {
+    live2dFocusAt.value = {
+      x: mouseLocation.x - windowFrame.origin.x,
+      y: windowFrame.size.height - mouseLocation.y + windowFrame.origin.y,
+    }
+    return
+  }
+
+  live2dFocusAt.value = {
+    x: mouseLocation.x - windowFrame.origin.x,
+    y: mouseLocation.y - windowFrame.origin.y,
+  }
+}
+
 onMounted(async () => {
   // Listen for click-through state changes
-  unlisten.push(await listen('tauri-app:window-click-through:is-inside', (event: { payload: boolean }) => {
-    isCursorInside.value = event.payload
-  }))
+  unlisten.push(await listen('tauri-app:window-click-through:position-cursor-and-window-frame', onTauriPositionCursorAndWindowFrameEvent))
 
   if (connected.value)
     return
@@ -77,7 +111,22 @@ onMounted(async () => {
 
 onUnmounted(() => {
   unlisten.forEach(fn => fn?.())
+  unlisten.length = 0
 })
+
+if (import.meta.hot) { // For better DX
+  import.meta.hot.on('vite:beforeUpdate', () => {
+    unlisten.forEach(fn => fn?.())
+    unlisten.length = 0
+    invoke('stop_monitor')
+  })
+  import.meta.hot.on('vite:afterUpdate', async () => {
+    if (unlisten.length === 0) {
+      unlisten.push(await listen('tauri-app:window-click-through:position-cursor-and-window-frame', onTauriPositionCursorAndWindowFrameEvent))
+    }
+    invoke('start_monitor')
+  })
+}
 </script>
 
 <template>
@@ -96,7 +145,10 @@ onUnmounted(() => {
     transition="opacity duration-500 ease-in-out"
   >
     <div relative h-full w-full items-end gap-2 class="view">
-      <WidgetStage h-full w-full flex-1 mb="<md:18" />
+      <WidgetStage h-full w-full flex-1 :focus-at="live2dFocusAt" mb="<md:18" />
+      <!-- <div h-full w-full flex-1 mb="<md:18">
+        HELLO
+      </div> -->
       <div
         absolute bottom-4 left-4 flex gap-1 op-0 transition="opacity duration-500"
         :class="{

apps/stage-tamagotchi/src/pages/settings/models/index.vue

@@ -1,6 +1,6 @@
 <script setup lang="ts">
 import { Live2DCanvas, Live2DModel } from '@proj-airi/stage-ui/components'
-import { useElementBounding } from '@vueuse/core'
+import { useElementBounding, useMouse } from '@vueuse/core'
 import { Vibrant } from 'node-vibrant/browser'
 import { ref } from 'vue'
 
@@ -40,13 +40,20 @@ const {
   showIconAnimation,
   animationIcon,
 } = useIconAnimation('i-solar:people-nearby-bold-duotone')
+
+const positionCursor = useMouse()
 </script>
 
 <template>
   <div flex>
     <div ref="live2dContainerRef" w="50%" h="80vh">
       <Live2DCanvas v-slot="{ app }" ref="live2dCanvasRef" :width="width" :height="height">
-        <Live2DModel :app="app" :mouth-open-size="0" :width="width" :height="height" :paused="false" />
+        <Live2DModel
+          :app="app" :mouth-open-size="0" :width="width" :height="height" :paused="false" :focus-at="{
+            x: positionCursor.x.value,
+            y: positionCursor.y.value,
+          }"
+        />
       </Live2DCanvas>
     </div>
     <Live2DSettings w="50%" h="80vh" :palette="palette" @extract-colors-from-model="extractColorsFromModel" />

apps/stage-web/src/pages/index.vue

@@ -1,6 +1,6 @@
 <script setup lang="ts">
 import { WidgetStage } from '@proj-airi/stage-ui/components'
-import { useDark } from '@vueuse/core'
+import { useDark, useMouse } from '@vueuse/core'
 import { ref } from 'vue'
 
 import Cross from '../components/Backgrounds/Cross.vue'
@@ -15,6 +15,8 @@ const paused = ref(false)
 function handleSettingsOpen(open: boolean) {
   paused.value = open
 }
+
+const positionCursor = useMouse()
 </script>
 
 <template>
@@ -31,7 +33,12 @@ function handleSettingsOpen(open: boolean) {
         </div>
         <!-- page -->
         <div relative flex="~ 1 row gap-y-0 gap-x-2 <md:col">
-          <WidgetStage flex-1 min-w="1/2" :paused="paused" />
+          <WidgetStage
+            flex-1 min-w="1/2" :paused="paused" :focus-at="{
+              x: positionCursor.x.value,
+              y: positionCursor.y.value,
+            }"
+          />
           <InteractiveArea class="flex <md:hidden" flex-1 max-w="500px" min-w="30%" />
           <MobileInteractiveArea class="hidden <md:block" mx2 mb2 @settings-open="handleSettingsOpen" />
         </div>

apps/stage-web/src/pages/settings/models/index.vue

@@ -1,6 +1,6 @@
 <script setup lang="ts">
 import { Live2DCanvas, Live2DModel } from '@proj-airi/stage-ui/components'
-import { useElementBounding } from '@vueuse/core'
+import { useElementBounding, useMouse } from '@vueuse/core'
 import { Vibrant } from 'node-vibrant/browser'
 import { ref } from 'vue'
 
@@ -40,13 +40,20 @@ const {
   showIconAnimation,
   animationIcon,
 } = useIconAnimation('i-solar:people-nearby-bold-duotone')
+
+const positionCursor = useMouse()
 </script>
 
 <template>
   <div flex>
     <div ref="live2dContainerRef" w="50%" h="80vh">
       <Live2DCanvas v-slot="{ app }" ref="live2dCanvasRef" :width="width" :height="height">
-        <Live2DModel :app="app" :mouth-open-size="0" :width="width" :height="height" :paused="false" />
+        <Live2DModel
+          :app="app" :mouth-open-size="0" :width="width" :height="height" :paused="false" :focus-at="{
+            x: positionCursor.x.value,
+            y: positionCursor.y.value,
+          }"
+        />
       </Live2DCanvas>
     </div>
     <Live2DSettings w="50%" h="80vh" :palette="palette" @extract-colors-from-model="extractColorsFromModel" />

packages/stage-ui/src/components/Live2D/Model.vue

@@ -23,12 +23,14 @@ const props = withDefaults(defineProps<{
   width: number
   height: number
   paused: boolean
+  focusAt: { x: number, y: number }
 }>(), {
   mouthOpenSize: 0,
 })
 
 const pixiApp = toRef(() => props.app)
 const paused = toRef(() => props.paused)
+const focusAt = toRef(() => props.focusAt)
 const model = ref<Live2DModel>()
 const initialModelWidth = ref<number>(0)
 const initialModelHeight = ref<number>(0)
@@ -90,10 +92,10 @@ async function loadModel() {
   const modelInstance = new Live2DModel()
 
   if (live2dLoadSource.value === 'file') {
-    await Live2DFactory.setupLive2DModel(modelInstance, [live2dModelFile.value])
+    await Live2DFactory.setupLive2DModel(modelInstance, [live2dModelFile.value], { autoInteract: false })
   }
   else if (live2dLoadSource.value === 'url') {
-    await Live2DFactory.setupLive2DModel(modelInstance, live2dModelUrl.value)
+    await Live2DFactory.setupLive2DModel(modelInstance, live2dModelUrl.value, { autoInteract: false })
   }
 
   model.value = modelInstance
@@ -243,6 +245,13 @@ watch(paused, (value) => {
   value ? pixiApp.value?.stop() : pixiApp.value?.start()
 })
 
+watch(focusAt, (value) => {
+  if (!model.value)
+    return
+
+  model.value.focus(value.x, value.y)
+})
+
 watchDebounced(loadingLive2dModel, (value) => {
   if (!value)
     return

packages/stage-ui/src/components/Scenes/Live2D.vue

@@ -13,6 +13,7 @@ import '../../utils/live2d-zip-loader'
 withDefaults(defineProps<{
   paused: boolean
   mouthOpenSize?: number
+  focusAt: { x: number, y: number }
 }>(), {
   mouthOpenSize: 0,
 })
@@ -25,7 +26,7 @@ const { live2dCurrentMotion } = storeToRefs(useSettings())
 <template>
   <Screen v-slot="{ width, height }" relative>
     <Live2DCanvas v-slot="{ app }" :width="width" :height="height">
-      <Live2DModel :app="app" :mouth-open-size="mouthOpenSize" :width="width" :height="height" :paused="paused" />
+      <Live2DModel :app="app" :mouth-open-size="mouthOpenSize" :width="width" :height="height" :paused="paused" :focus-at="focusAt" />
     </Live2DCanvas>
     <div absolute bottom="3" right="3">
       <div flex="~ row" cursor-pointer>

packages/stage-ui/src/components/Widgets/Stage.vue

@@ -26,7 +26,10 @@ import { useSettings } from '../../stores/settings'
 import Live2DScene from '../Scenes/Live2D.vue'
 import VRMScene from '../Scenes/VRM.vue'
 
-withDefaults(defineProps<{ paused?: boolean }>(), { paused: false })
+withDefaults(defineProps<{
+  paused?: boolean
+  focusAt: { x: number, y: number }
+}>(), { paused: false })
 
 const db = ref<DuckDBWasmDrizzleDatabase>()
 // const transformersProvider = createTransformers({ embedWorkerURL })
@@ -223,6 +226,7 @@ onMounted(async () => {
     <div h-full w-full>
       <Live2DScene
         v-if="stageView === '2d'"
+        :focus-at="focusAt"
         :mouth-open-size="mouthOpenSize"
         min-w="50% <lg:full" min-h="100 sm:100" h-full w-full flex-1
         :paused="paused"