diff --git a/apps/stage-tamagotchi/src/bindings/tauri-plugins/window-pass-through-on-hover.ts b/apps/stage-tamagotchi/src/bindings/tauri-plugins/window-pass-through-on-hover.ts index 924d33824..23be10dec 100644 --- a/apps/stage-tamagotchi/src/bindings/tauri-plugins/window-pass-through-on-hover.ts +++ b/apps/stage-tamagotchi/src/bindings/tauri-plugins/window-pass-through-on-hover.ts @@ -4,25 +4,40 @@ /** user-defined commands **/ +let passThroughEnabled = false; export const commands = { -async startPassThrough() : Promise> { + async startPassThrough(): Promise> { + if (passThroughEnabled) { + return { status: 'ok', data: null }; + } try { - return { status: "ok", data: await TAURI_INVOKE("plugin:window-pass-through-on-hover|start_pass_through") }; -} catch (e) { - if(e instanceof Error) throw e; - else return { status: "error", error: e as any }; -} -}, -async stopPassThrough() : Promise> { + passThroughEnabled = true; + return { status: 'ok', data: await TAURI_INVOKE('plugin:window-pass-through-on-hover|start_pass_through') }; + } + catch (e) { + passThroughEnabled = false; + if (e instanceof Error) + throw e; + else return { status: 'error', error: e as any }; + } + }, + async stopPassThrough(): Promise> { + if (!passThroughEnabled) { + return { status: 'ok', data: null }; + } try { - return { status: "ok", data: await TAURI_INVOKE("plugin:window-pass-through-on-hover|stop_pass_through") }; -} catch (e) { - if(e instanceof Error) throw e; - else return { status: "error", error: e as any }; -} -} -} + passThroughEnabled = false; + return { status: 'ok', data: await TAURI_INVOKE('plugin:window-pass-through-on-hover|stop_pass_through') }; + } + catch (e) { + passThroughEnabled = true; + if (e instanceof Error) + throw e; + else return { status: 'error', error: e as any }; + } + } +}; /** user-defined events **/ diff --git a/apps/stage-tamagotchi/src/composables/tauri-rdev.ts b/apps/stage-tamagotchi/src/composables/tauri-rdev.ts index 94ef44fdb..0b9cfb8dc 100644 --- a/apps/stage-tamagotchi/src/composables/tauri-rdev.ts +++ b/apps/stage-tamagotchi/src/composables/tauri-rdev.ts @@ -42,6 +42,17 @@ export function useTauriRdevEventTarget(): EventTarget { eventTarget.dispatchEvent(e) }) + unListenFuncs.push(await listen('tauri-plugins:tauri-plugin-rdev:mousemove', (event) => { + if (event.payload.event_type.MouseMove) { + const { x, y } = event.payload.event_type.MouseMove + const e = new MouseEvent('mousemove', { + clientX: x, + clientY: y, + }) + eventTarget.dispatchEvent(e) + } + })) + unListenFuncs.push(await listen('tauri-plugins:tauri-plugin-rdev:keyup', (event) => { if (typeof event.payload.event_type.KeyRelease === 'object' && 'Unknown' in event.payload.event_type.KeyRelease) { if (event.payload.event_type.KeyRelease.Unknown === 62) { diff --git a/apps/stage-tamagotchi/src/composables/tauri.ts b/apps/stage-tamagotchi/src/composables/tauri.ts index 4d62a40fa..9df22d01f 100644 --- a/apps/stage-tamagotchi/src/composables/tauri.ts +++ b/apps/stage-tamagotchi/src/composables/tauri.ts @@ -36,7 +36,7 @@ export interface WindowFrame { interface Events { 'tauri://resize': unknown - 'tauri://move': unknown + 'tauri://move': { payload: { x: number, y: number } } 'tauri://close-requested': unknown 'tauri://destroyed': unknown 'tauri://focus': unknown @@ -72,6 +72,7 @@ export interface AiriTamagotchiEvents extends Events { 'tauri-plugins:tauri-plugin-rdev:keyup': { time: { secs_since_epoch: number, nanos_since_epoch: number }, name: string, event_type: { KeyRelease: KeyCode | { Unknown: number } } } // similar to 'keyup' events from DOM elements 'tauri-plugins:tauri-plugin-rdev:mousedown': { time: { secs_since_epoch: number, nanos_since_epoch: number }, name: string, event_type: { ButtonPress: string } } // similar to 'mousedown' events from DOM elements 'tauri-plugins:tauri-plugin-rdev:mouseup': { time: { secs_since_epoch: number, nanos_since_epoch: number }, name: string, event_type: { ButtonRelease: string } } // similar to 'mouseup' events from DOM elements + 'tauri-plugins:tauri-plugin-rdev:mousemove': { time: { secs_since_epoch: number, nanos_since_epoch: number }, name: string, event_type: { MouseMove: { x: number, y: number } } } // similar to 'mousemove' events from DOM elements // MCP 'mcp_plugin_destroyed': undefined @@ -288,6 +289,18 @@ export function useTauriWindow() { } } + async function getPosition() { + try { + const imported = await _ensureImported() + const window = imported.getCurrentWindow() + return await window.innerPosition() + } + catch (error) { + console.error('Failed to get window position:', error) + return undefined + } + } + async function closeWindow(label?: string) { try { const imported = await _ensureImported() @@ -316,6 +329,7 @@ export function useTauriWindow() { getCurrentMonitor, getPrimaryMonitor, setPosition, + getPosition, closeWindow, } } diff --git a/apps/stage-tamagotchi/src/composables/use-rdev-mouse.ts b/apps/stage-tamagotchi/src/composables/use-rdev-mouse.ts new file mode 100644 index 000000000..1123fdbed --- /dev/null +++ b/apps/stage-tamagotchi/src/composables/use-rdev-mouse.ts @@ -0,0 +1,30 @@ +import type { AiriTamagotchiEvents } from './tauri' + +import { createSharedComposable } from '@vueuse/core' +import { ref } from 'vue' + +import { useTauriEvent } from './tauri' + +export const useRdevMouse = createSharedComposable(() => { + const mouseX = ref(0) + const mouseY = ref(0) + + const { listen } = useTauriEvent() + + async function setup() { + await listen('tauri-plugins:tauri-plugin-rdev:mousemove', (event) => { + if (event.payload.event_type.MouseMove) { + const { x, y } = event.payload.event_type.MouseMove + mouseX.value = x + mouseY.value = y + } + }) + } + + setup() + + return { + mouseX, + mouseY, + } +}) diff --git a/apps/stage-tamagotchi/src/pages/index.vue b/apps/stage-tamagotchi/src/pages/index.vue index b6468edde..772e78d08 100644 --- a/apps/stage-tamagotchi/src/pages/index.vue +++ b/apps/stage-tamagotchi/src/pages/index.vue @@ -5,13 +5,16 @@ import { WidgetStage } from '@proj-airi/stage-ui/components/scenes' import { useLive2d } from '@proj-airi/stage-ui/stores/live2d' import { useMcpStore } from '@proj-airi/stage-ui/stores/mcp' import { connectServer } from '@proj-airi/tauri-plugin-mcp' +import { watchThrottled } from '@vueuse/core' import { storeToRefs } from 'pinia' import { computed, onMounted, onUnmounted, ref, watch } from 'vue' import ResourceStatusIsland from '../components/Widgets/ResourceStatusIsland/index.vue' -import { useTauriCore, useTauriEvent } from '../composables/tauri' +import { commands as passThroughCommands } from '../bindings/tauri-plugins/window-pass-through-on-hover' +import { useTauriCore, useTauriEvent, useTauriWindow } from '../composables/tauri' import { useTauriGlobalShortcuts } from '../composables/tauri-global-shortcuts' +import { useRdevMouse } from '../composables/use-rdev-mouse' import { useResourcesStore } from '../stores/resources' import { useWindowStore } from '../stores/window' import { useWindowControlStore } from '../stores/window-controls' @@ -21,14 +24,117 @@ useTauriGlobalShortcuts() const windowControlStore = useWindowControlStore() const resourcesStore = useResourcesStore() const mcpStore = useMcpStore() +const { getPosition } = useTauriWindow() +const { mouseX, mouseY } = useRdevMouse() const { listen } = useTauriEvent() const { invoke } = useTauriCore() const { connected, serverCmd, serverArgs } = storeToRefs(mcpStore) const { scale, positionInPercentageString } = storeToRefs(useLive2d()) -const { centerPos, live2dLookAtX, live2dLookAtY, shouldHideView } = storeToRefs(useWindowStore()) +const { centerPos, live2dLookAtX, live2dLookAtY } = storeToRefs(useWindowStore()) const live2dFocusAt = ref(centerPos.value) +const widgetStageRef = ref<{ canvasElement: () => HTMLCanvasElement }>() +const resourceStatusIslandRef = ref>() +const buttonsContainerRef = ref() +const windowX = ref(0) +const windowY = ref(0) +const isClickThrough = ref(false) +const isPassingThrough = ref(false) +const isOverUI = ref(false) + +watchThrottled([mouseX, mouseY], async ([x, y]) => { + const canvas = widgetStageRef.value?.canvasElement() + if (!canvas) + return + + if (windowControlStore.controlMode === WindowControlMode.RESIZE || windowControlStore.controlMode === WindowControlMode.MOVE) { + if (isPassingThrough.value) { + passThroughCommands.stopPassThrough() + isPassingThrough.value = false + } + return + } + + const relativeX = x - windowX.value + const relativeY = y - windowY.value + + const islandEl = resourceStatusIslandRef.value?.$el as HTMLElement + const buttonsEl = buttonsContainerRef.value + + isOverUI.value = false + if (!windowControlStore.isIgnoringMouseEvent) { + if (islandEl) { + const rect = islandEl.getBoundingClientRect() + if (relativeX >= rect.left && relativeX <= rect.right && relativeY >= rect.top && relativeY <= rect.bottom) + isOverUI.value = true + } + if (!isOverUI.value && buttonsEl) { + const rect = buttonsEl.getBoundingClientRect() + if (relativeX >= rect.left && relativeX <= rect.right && relativeY >= rect.top && relativeY <= rect.bottom) + isOverUI.value = true + } + + if (isOverUI.value) { + if (isPassingThrough.value) { + passThroughCommands.stopPassThrough() + isPassingThrough.value = false + } + return + } + } + + let isTransparent = false + if ( + !isOverUI.value + && relativeX >= 0 + && relativeX < canvas.clientWidth + && relativeY >= 0 + && relativeY < canvas.clientHeight + ) { + const gl = canvas.getContext('webgl2') || canvas.getContext('webgl') + if (gl) { + const pixelX = relativeX * (gl.drawingBufferWidth / canvas.clientWidth) + const pixelY + = gl.drawingBufferHeight + - relativeY * (gl.drawingBufferHeight / canvas.clientHeight) + + const data = new Uint8Array(4) + gl.readPixels( + Math.floor(pixelX), + Math.floor(pixelY), + 1, + 1, + gl.RGBA, + gl.UNSIGNED_BYTE, + data, + ) + isTransparent = data[3] < 100 // Use a small threshold for anti-aliasing + } + } + else { + isTransparent = true + } + + isClickThrough.value = isTransparent + + if (windowControlStore.isIgnoringMouseEvent) { + if (!isPassingThrough.value) { + passThroughCommands.startPassThrough() + isPassingThrough.value = true + } + return + } + + if (isTransparent && !isPassingThrough.value) { + passThroughCommands.startPassThrough() + isPassingThrough.value = true + } + else if (!isTransparent && isPassingThrough.value) { + passThroughCommands.stopPassThrough() + isPassingThrough.value = false + } +}, { throttle: 33 }) watch([live2dLookAtX, live2dLookAtY], ([x, y]) => live2dFocusAt.value = { x, y }, { immediate: true }) @@ -82,6 +188,16 @@ async function setupWhisperModel() { } onMounted(async () => { + const pos = await getPosition() + if (pos) { + windowX.value = pos.x + windowY.value = pos.y + } + unListenFuncs.push(await listen('tauri://move', (event) => { + windowX.value = event.payload.payload.x + windowY.value = event.payload.payload.y + })) + await setupVADModel() await setupWhisperModel() @@ -118,7 +234,8 @@ if (import.meta.hot) { // For better DX