feat(stage-tamagotchi): Make window passthrough outside of Live2D and VRM models (#437)

---------

Co-authored-by: Neko <neko@ayaka.moe>

apps/stage-tamagotchi/src/bindings/tauri-plugins/window-pass-through-on-hover.ts

@@ -4,25 +4,40 @@
 
 /** user-defined commands **/
 
+let passThroughEnabled = false;
 
 export const commands = {
-async startPassThrough() : Promise<Result<null, string>> {
+  async startPassThrough(): Promise<Result<null, string>> {
+    if (passThroughEnabled) {
+      return { status: 'ok', data: null };
+    }
     try {
-    return { status: "ok", data: await TAURI_INVOKE("plugin:window-pass-through-on-hover|start_pass_through") };
-} catch (e) {
-    if(e instanceof Error) throw e;
-    else return { status: "error", error: e  as any };
-}
-},
-async stopPassThrough() : Promise<Result<null, string>> {
+      passThroughEnabled = true;
+      return { status: 'ok', data: await TAURI_INVOKE('plugin:window-pass-through-on-hover|start_pass_through') };
+    }
+    catch (e) {
+      passThroughEnabled = false;
+      if (e instanceof Error)
+        throw e;
+      else return { status: 'error', error: e as any };
+    }
+  },
+  async stopPassThrough(): Promise<Result<null, string>> {
+    if (!passThroughEnabled) {
+      return { status: 'ok', data: null };
+    }
     try {
-    return { status: "ok", data: await TAURI_INVOKE("plugin:window-pass-through-on-hover|stop_pass_through") };
-} catch (e) {
-    if(e instanceof Error) throw e;
-    else return { status: "error", error: e  as any };
-}
-}
-}
+      passThroughEnabled = false;
+      return { status: 'ok', data: await TAURI_INVOKE('plugin:window-pass-through-on-hover|stop_pass_through') };
+    }
+    catch (e) {
+      passThroughEnabled = true;
+      if (e instanceof Error)
+        throw e;
+      else return { status: 'error', error: e as any };
+    }
+  }
+};
 
 /** user-defined events **/
 

apps/stage-tamagotchi/src/composables/tauri-rdev.ts

@@ -42,6 +42,17 @@ export function useTauriRdevEventTarget(): EventTarget {
       eventTarget.dispatchEvent(e)
     })
 
+    unListenFuncs.push(await listen('tauri-plugins:tauri-plugin-rdev:mousemove', (event) => {
+      if (event.payload.event_type.MouseMove) {
+        const { x, y } = event.payload.event_type.MouseMove
+        const e = new MouseEvent('mousemove', {
+          clientX: x,
+          clientY: y,
+        })
+        eventTarget.dispatchEvent(e)
+      }
+    }))
+
     unListenFuncs.push(await listen('tauri-plugins:tauri-plugin-rdev:keyup', (event) => {
       if (typeof event.payload.event_type.KeyRelease === 'object' && 'Unknown' in event.payload.event_type.KeyRelease) {
         if (event.payload.event_type.KeyRelease.Unknown === 62) {

apps/stage-tamagotchi/src/composables/tauri.ts

@@ -36,7 +36,7 @@ export interface WindowFrame {
 
 interface Events {
   'tauri://resize': unknown
-  'tauri://move': unknown
+  'tauri://move': { payload: { x: number, y: number } }
   'tauri://close-requested': unknown
   'tauri://destroyed': unknown
   'tauri://focus': unknown
@@ -72,6 +72,7 @@ export interface AiriTamagotchiEvents extends Events {
   'tauri-plugins:tauri-plugin-rdev:keyup': { time: { secs_since_epoch: number, nanos_since_epoch: number }, name: string, event_type: { KeyRelease: KeyCode | { Unknown: number } } } // similar to 'keyup' events from DOM elements
   'tauri-plugins:tauri-plugin-rdev:mousedown': { time: { secs_since_epoch: number, nanos_since_epoch: number }, name: string, event_type: { ButtonPress: string } } // similar to 'mousedown' events from DOM elements
   'tauri-plugins:tauri-plugin-rdev:mouseup': { time: { secs_since_epoch: number, nanos_since_epoch: number }, name: string, event_type: { ButtonRelease: string } } // similar to 'mouseup' events from DOM elements
+  'tauri-plugins:tauri-plugin-rdev:mousemove': { time: { secs_since_epoch: number, nanos_since_epoch: number }, name: string, event_type: { MouseMove: { x: number, y: number } } } // similar to 'mousemove' events from DOM elements
 
   // MCP
   'mcp_plugin_destroyed': undefined
@@ -288,6 +289,18 @@ export function useTauriWindow() {
     }
   }
 
+  async function getPosition() {
+    try {
+      const imported = await _ensureImported()
+      const window = imported.getCurrentWindow()
+      return await window.innerPosition()
+    }
+    catch (error) {
+      console.error('Failed to get window position:', error)
+      return undefined
+    }
+  }
+
   async function closeWindow(label?: string) {
     try {
       const imported = await _ensureImported()
@@ -316,6 +329,7 @@ export function useTauriWindow() {
     getCurrentMonitor,
     getPrimaryMonitor,
     setPosition,
+    getPosition,
     closeWindow,
   }
 }

apps/stage-tamagotchi/src/composables/use-rdev-mouse.ts

@@ -0,0 +1,30 @@
+import type { AiriTamagotchiEvents } from './tauri'
+
+import { createSharedComposable } from '@vueuse/core'
+import { ref } from 'vue'
+
+import { useTauriEvent } from './tauri'
+
+export const useRdevMouse = createSharedComposable(() => {
+  const mouseX = ref(0)
+  const mouseY = ref(0)
+
+  const { listen } = useTauriEvent<AiriTamagotchiEvents>()
+
+  async function setup() {
+    await listen('tauri-plugins:tauri-plugin-rdev:mousemove', (event) => {
+      if (event.payload.event_type.MouseMove) {
+        const { x, y } = event.payload.event_type.MouseMove
+        mouseX.value = x
+        mouseY.value = y
+      }
+    })
+  }
+
+  setup()
+
+  return {
+    mouseX,
+    mouseY,
+  }
+})

apps/stage-tamagotchi/src/pages/index.vue

@@ -5,13 +5,16 @@ import { WidgetStage } from '@proj-airi/stage-ui/components/scenes'
 import { useLive2d } from '@proj-airi/stage-ui/stores/live2d'
 import { useMcpStore } from '@proj-airi/stage-ui/stores/mcp'
 import { connectServer } from '@proj-airi/tauri-plugin-mcp'
+import { watchThrottled } from '@vueuse/core'
 import { storeToRefs } from 'pinia'
 import { computed, onMounted, onUnmounted, ref, watch } from 'vue'
 
 import ResourceStatusIsland from '../components/Widgets/ResourceStatusIsland/index.vue'
 
-import { useTauriCore, useTauriEvent } from '../composables/tauri'
+import { commands as passThroughCommands } from '../bindings/tauri-plugins/window-pass-through-on-hover'
+import { useTauriCore, useTauriEvent, useTauriWindow } from '../composables/tauri'
 import { useTauriGlobalShortcuts } from '../composables/tauri-global-shortcuts'
+import { useRdevMouse } from '../composables/use-rdev-mouse'
 import { useResourcesStore } from '../stores/resources'
 import { useWindowStore } from '../stores/window'
 import { useWindowControlStore } from '../stores/window-controls'
@@ -21,14 +24,117 @@ useTauriGlobalShortcuts()
 const windowControlStore = useWindowControlStore()
 const resourcesStore = useResourcesStore()
 const mcpStore = useMcpStore()
+const { getPosition } = useTauriWindow()
+const { mouseX, mouseY } = useRdevMouse()
 
 const { listen } = useTauriEvent<AiriTamagotchiEvents>()
 const { invoke } = useTauriCore()
 const { connected, serverCmd, serverArgs } = storeToRefs(mcpStore)
 const { scale, positionInPercentageString } = storeToRefs(useLive2d())
 
-const { centerPos, live2dLookAtX, live2dLookAtY, shouldHideView } = storeToRefs(useWindowStore())
+const { centerPos, live2dLookAtX, live2dLookAtY } = storeToRefs(useWindowStore())
 const live2dFocusAt = ref<Point>(centerPos.value)
+const widgetStageRef = ref<{ canvasElement: () => HTMLCanvasElement }>()
+const resourceStatusIslandRef = ref<InstanceType<typeof ResourceStatusIsland>>()
+const buttonsContainerRef = ref<HTMLDivElement>()
+const windowX = ref(0)
+const windowY = ref(0)
+const isClickThrough = ref(false)
+const isPassingThrough = ref(false)
+const isOverUI = ref(false)
+
+watchThrottled([mouseX, mouseY], async ([x, y]) => {
+  const canvas = widgetStageRef.value?.canvasElement()
+  if (!canvas)
+    return
+
+  if (windowControlStore.controlMode === WindowControlMode.RESIZE || windowControlStore.controlMode === WindowControlMode.MOVE) {
+    if (isPassingThrough.value) {
+      passThroughCommands.stopPassThrough()
+      isPassingThrough.value = false
+    }
+    return
+  }
+
+  const relativeX = x - windowX.value
+  const relativeY = y - windowY.value
+
+  const islandEl = resourceStatusIslandRef.value?.$el as HTMLElement
+  const buttonsEl = buttonsContainerRef.value
+
+  isOverUI.value = false
+  if (!windowControlStore.isIgnoringMouseEvent) {
+    if (islandEl) {
+      const rect = islandEl.getBoundingClientRect()
+      if (relativeX >= rect.left && relativeX <= rect.right && relativeY >= rect.top && relativeY <= rect.bottom)
+        isOverUI.value = true
+    }
+    if (!isOverUI.value && buttonsEl) {
+      const rect = buttonsEl.getBoundingClientRect()
+      if (relativeX >= rect.left && relativeX <= rect.right && relativeY >= rect.top && relativeY <= rect.bottom)
+        isOverUI.value = true
+    }
+
+    if (isOverUI.value) {
+      if (isPassingThrough.value) {
+        passThroughCommands.stopPassThrough()
+        isPassingThrough.value = false
+      }
+      return
+    }
+  }
+
+  let isTransparent = false
+  if (
+    !isOverUI.value
+    && relativeX >= 0
+    && relativeX < canvas.clientWidth
+    && relativeY >= 0
+    && relativeY < canvas.clientHeight
+  ) {
+    const gl = canvas.getContext('webgl2') || canvas.getContext('webgl')
+    if (gl) {
+      const pixelX = relativeX * (gl.drawingBufferWidth / canvas.clientWidth)
+      const pixelY
+        = gl.drawingBufferHeight
+          - relativeY * (gl.drawingBufferHeight / canvas.clientHeight)
+
+      const data = new Uint8Array(4)
+      gl.readPixels(
+        Math.floor(pixelX),
+        Math.floor(pixelY),
+        1,
+        1,
+        gl.RGBA,
+        gl.UNSIGNED_BYTE,
+        data,
+      )
+      isTransparent = data[3] < 100 // Use a small threshold for anti-aliasing
+    }
+  }
+  else {
+    isTransparent = true
+  }
+
+  isClickThrough.value = isTransparent
+
+  if (windowControlStore.isIgnoringMouseEvent) {
+    if (!isPassingThrough.value) {
+      passThroughCommands.startPassThrough()
+      isPassingThrough.value = true
+    }
+    return
+  }
+
+  if (isTransparent && !isPassingThrough.value) {
+    passThroughCommands.startPassThrough()
+    isPassingThrough.value = true
+  }
+  else if (!isTransparent && isPassingThrough.value) {
+    passThroughCommands.stopPassThrough()
+    isPassingThrough.value = false
+  }
+}, { throttle: 33 })
 
 watch([live2dLookAtX, live2dLookAtY], ([x, y]) => live2dFocusAt.value = { x, y }, { immediate: true })
 
@@ -82,6 +188,16 @@ async function setupWhisperModel() {
 }
 
 onMounted(async () => {
+  const pos = await getPosition()
+  if (pos) {
+    windowX.value = pos.x
+    windowY.value = pos.y
+  }
+  unListenFuncs.push(await listen('tauri://move', (event) => {
+    windowX.value = event.payload.payload.x
+    windowY.value = event.payload.payload.y
+  }))
+
   await setupVADModel()
   await setupWhisperModel()
 
@@ -118,7 +234,8 @@ if (import.meta.hot) { // For better DX
 <template>
   <div
     :class="[modeIndicatorClass, {
-      'op-0': shouldHideView,
+      'op-0': windowControlStore.isIgnoringMouseEvent && !isClickThrough,
+      'pointer-events-none': !isClickThrough,
     }]"
     max-h="[100vh]"
     max-w="[100vw]"
@@ -128,17 +245,19 @@ if (import.meta.hot) { // For better DX
   >
     <div relative h-full w-full items-end gap-2 class="view">
       <WidgetStage
+        ref="widgetStageRef"
         h-full w-full flex-1
         :focus-at="live2dFocusAt" :scale="scale"
         :x-offset="positionInPercentageString.x"
         :y-offset="positionInPercentageString.y" mb="<md:18"
       />
-      <ResourceStatusIsland />
+      <ResourceStatusIsland ref="resourceStatusIslandRef" />
       <div
+        ref="buttonsContainerRef"
         absolute bottom-4 left-4 flex gap-1 op-0 transition="opacity duration-500"
         :class="{
-          'pointer-events-none': windowControlStore.isControlActive,
-          'show-on-hover': !windowControlStore.isIgnoringMouseEvent,
+          'pointer-events-none': isClickThrough && !isOverUI,
+          'show-on-hover': !windowControlStore.isIgnoringMouseEvent && (!isClickThrough || isOverUI),
         }"
       >
         <div
@@ -212,10 +331,8 @@ if (import.meta.hot) { // For better DX
 .view {
   transition: opacity 0.5s ease-in-out;
 
-  &:hover {
-    .show-on-hover {
-      opacity: 1;
-    }
+  .show-on-hover {
+    opacity: 1;
   }
 }
 

apps/stage-tamagotchi/src/stores/shortcuts.ts

@@ -5,7 +5,6 @@ import { defineStore } from 'pinia'
 import { ref, toValue, watch } from 'vue'
 
 import { WindowControlMode } from '../types/window-controls'
-import { startClickThrough, stopClickThrough } from '../utils/windows'
 import { useWindowControlStore } from './window-controls'
 
 interface Versioned<T> { version?: string, data?: T }
@@ -96,12 +95,6 @@ export const useShortcutsStore = defineStore('shortcuts', () => {
       type: 'ignore-mouse-event',
       handle: async () => {
         windowStore.isIgnoringMouseEvent = !windowStore.isIgnoringMouseEvent
-        if (windowStore.isIgnoringMouseEvent) {
-          await startClickThrough()
-          return
-        }
-
-        await stopClickThrough()
       },
     },
   ])

apps/stage-tamagotchi/src/stores/window-controls.ts

@@ -55,12 +55,6 @@ export const useWindowMode = defineStore('window-mode', () => {
 
     unlistenFuncs.value.push(await listen('tauri-main:main:window-mode:fade-on-hover', async () => {
       windowStore.isIgnoringMouseEvent = !windowStore.isIgnoringMouseEvent
-      if (windowStore.isIgnoringMouseEvent) {
-        await startClickThrough()
-        return
-      }
-
-      await stopClickThrough()
     }))
   }
 

crates/tauri-plugin-rdev/src/lib.rs

@@ -1,14 +1,12 @@
-use std::sync::{Mutex, atomic::Ordering};
+use std::sync::{atomic::Ordering, Mutex};
 
 use log::error;
-use rdev::{Event, EventType, listen};
+use rdev::{listen, Event, EventType};
 use serde::Serialize;
 use serde_json::Value;
 use tauri::{
-  Emitter,
-  Manager,
-  Runtime,
   plugin::{Builder as PluginBuilder, TauriPlugin},
+  Emitter, Manager, Runtime,
 };
 
 #[derive(Debug, Clone, Serialize)]
@@ -22,7 +20,7 @@ pub enum DeviceKind {
 
 #[derive(Debug, Clone, Serialize)]
 pub struct DeviceEvent {
-  kind:  DeviceKind,
+  kind: DeviceKind,
   value: Value,
 }
 
@@ -49,6 +47,7 @@ fn start_listen<R: tauri::Runtime>(app: tauri::AppHandle<R>) {
       EventType::KeyRelease(_) => "tauri-plugins:tauri-plugin-rdev:keyup",
       EventType::ButtonPress(_) => "tauri-plugins:tauri-plugin-rdev:mousedown",
       EventType::ButtonRelease(_) => "tauri-plugins:tauri-plugin-rdev:mouseup",
+      EventType::MouseMove { .. } => "tauri-plugins:tauri-plugin-rdev:mousemove",
       _ => return,
     };
 
@@ -58,7 +57,7 @@ fn start_listen<R: tauri::Runtime>(app: tauri::AppHandle<R>) {
       Err(e) => {
         error!("PluginState mutex is poisoned: {}", e);
         return;
-      },
+      }
     };
 
     for label in &state.window_labels {
@@ -118,8 +117,8 @@ pub fn init<R: Runtime>() -> TauriPlugin<R> {
           state
             .window_labels
             .retain(|label| label != window_cloned.label());
-        },
-        _ => {},
+        }
+        _ => {}
       });
     })
     .build()

packages/stage-ui/src/components/Scenes/Live2D.vue

@@ -1,4 +1,6 @@
 <script setup lang="ts">
+import { ref } from 'vue'
+
 import Screen from '../Misc/Screen.vue'
 import Live2DCanvas from './Live2D/Canvas.vue'
 import Live2DModel from './Live2D/Model.vue'
@@ -21,11 +23,20 @@ withDefaults(defineProps<{
   mouthOpenSize: 0,
   scale: 1,
 })
+
+const live2dCanvasRef = ref<InstanceType<typeof Live2DCanvas>>()
+
+defineExpose({
+  canvasElement: () => {
+    return live2dCanvasRef.value?.canvasElement()
+  },
+})
 </script>
 
 <template>
   <Screen v-slot="{ width, height }" relative>
     <Live2DCanvas
+      ref="live2dCanvasRef"
       v-slot="{ app }"
       :width="width"
       :height="height"

packages/stage-ui/src/components/Scenes/Stage.vue

@@ -40,7 +40,8 @@ withDefaults(defineProps<{
 const db = ref<DuckDBWasmDrizzleDatabase>()
 // const transformersProvider = createTransformers({ embedWorkerURL })
 
-const vrmViewerRef = ref<{ setExpression: (expression: string) => void }>()
+const vrmViewerRef = ref<InstanceType<typeof VRMScene>>()
+const live2dSceneRef = ref<InstanceType<typeof Live2DScene>>()
 
 const settingsStore = useSettings()
 const { stageModelRenderer, stageViewControlsEnabled, live2dDisableFocus, stageModelSelectedUrl } = storeToRefs(settingsStore)
@@ -246,6 +247,18 @@ onMounted(async () => {
   db.value = drizzle({ connection: { bundles: getImportUrlBundles() } })
   await db.value.execute(`CREATE TABLE memory_test (vec FLOAT[768]);`)
 })
+
+function canvasElement() {
+  if (stageModelRenderer.value === 'live2d')
+    return live2dSceneRef.value?.canvasElement()
+
+  else if (stageModelRenderer.value === 'vrm')
+    return vrmViewerRef.value?.canvasElement()
+}
+
+defineExpose({
+  canvasElement,
+})
 </script>
 
 <template>
@@ -253,6 +266,7 @@ onMounted(async () => {
     <div h-full w-full>
       <Live2DScene
         v-if="stageModelRenderer === 'live2d' && showStage"
+        ref="live2dSceneRef"
         min-w="50% <lg:full" min-h="100 sm:100" h-full w-full flex-1
         :model-src="stageModelSelectedUrl"
         :focus-at="focusAt"

packages/stage-ui/src/components/Scenes/VRM.vue

@@ -1,4 +1,6 @@
 <script setup lang="ts">
+import type { TresContext } from '@tresjs/core'
+
 import { TresCanvas } from '@tresjs/core'
 import { EffectComposerPmndrs, HueSaturationPmndrs } from '@tresjs/post-processing'
 import { useElementBounding, useMouse } from '@vueuse/core'
@@ -56,6 +58,11 @@ const modelRef = ref<InstanceType<typeof VRMModel>>()
 
 const camera = shallowRef(new PerspectiveCamera())
 const controlsRef = shallowRef<InstanceType<typeof OrbitControls>>()
+const tresCanvasRef = shallowRef<TresContext>()
+
+function onTresReady(context: TresContext) {
+  tresCanvasRef.value = context
+}
 
 const effectProps = {
   saturation: 0.3,
@@ -243,6 +250,9 @@ defineExpose({
   setExpression: (expression: string) => {
     modelRef.value?.setExpression(expression)
   },
+  canvasElement: () => {
+    return tresCanvasRef.value?.renderer.value.domElement
+  },
 })
 </script>
 
@@ -256,6 +266,8 @@ defineExpose({
       :height="height"
       :tone-mapping="ACESFilmicToneMapping"
       :tone-mapping-exposure="1"
+      :preserve-drawing-buffer="true"
+      @ready="onTresReady"
     >
       <OrbitControls ref="controlsRef" />
       <Environment