feat(stage-ui,stage-ui-three): support fade on hover for three

apps/stage-tamagotchi/package.json

@@ -59,6 +59,8 @@
     "@proj-airi/server-sdk": "workspace:*",
     "@proj-airi/stage-layouts": "workspace:^",
     "@proj-airi/stage-ui": "workspace:^",
+    "@proj-airi/stage-ui-live2d": "workspace:^",
+    "@proj-airi/stage-ui-three": "workspace:^",
     "@proj-airi/ui": "workspace:^",
     "@stdlib/string-base-kebabcase": "^0.2.2",
     "@tresjs/cientos": "^5.2.1",

apps/stage-tamagotchi/src/renderer/pages/index.vue

@@ -3,6 +3,7 @@ import type { ChatProvider } from '@xsai-ext/providers/utils'
 
 import workletUrl from '@proj-airi/stage-ui/workers/vad/process.worklet?worker&url'
 
+import { useThreeSceneIsTransparentAtPoint } from '@proj-airi/stage-ui-three'
 import { WidgetStage } from '@proj-airi/stage-ui/components/scenes'
 import { useAudioRecorder } from '@proj-airi/stage-ui/composables/audio/audio-recorder'
 import { useCanvasPixelIsTransparentAtPoint } from '@proj-airi/stage-ui/composables/canvas-alpha'
@@ -12,7 +13,7 @@ import { useLive2d } from '@proj-airi/stage-ui/stores/live2d'
 import { useConsciousnessStore } from '@proj-airi/stage-ui/stores/modules/consciousness'
 import { useHearingSpeechInputPipeline } from '@proj-airi/stage-ui/stores/modules/hearing'
 import { useProvidersStore } from '@proj-airi/stage-ui/stores/providers'
-import { useSettingsAudioDevice } from '@proj-airi/stage-ui/stores/settings'
+import { useSettings, useSettingsAudioDevice } from '@proj-airi/stage-ui/stores/settings'
 import { refDebounced, useBroadcastChannel } from '@vueuse/core'
 import { storeToRefs } from 'pinia'
 import { computed, onUnmounted, ref, toRef, watch } from 'vue'
@@ -32,7 +33,7 @@ import { useControlsIslandStore } from '../stores/controls-island'
 import { useWindowStore } from '../stores/window'
 
 const controlsIslandRef = ref<InstanceType<typeof ControlsIsland>>()
-const widgetStageRef = ref<{ canvasElement: () => HTMLCanvasElement }>()
+const widgetStageRef = ref<InstanceType<typeof WidgetStage>>()
 const stageCanvas = toRef(() => widgetStageRef.value?.canvasElement())
 const componentStateStage = ref<'pending' | 'loading' | 'mounted'>('pending')
 
@@ -47,9 +48,31 @@ const isOutsideFor250Ms = refDebounced(isOutside, 250)
 const { x: relativeMouseX, y: relativeMouseY } = useElectronRelativeMouse()
 // NOTICE: In real-world use cases of Fade on Hover feature, the cursor may move around the edge of the
 // model rapidly, causing flickering effects when checking pixel transparency strictly.
-// Here we use `regionRadius` to help to detect with look-ahead strategy to relax the pixel accurate
-// check / detection on hovered underlying canvas pixel, therefore inaccurate mouse movements won't cause flickering.
-const isTransparent = useCanvasPixelIsTransparentAtPoint(stageCanvas, relativeMouseX, relativeMouseY, { regionRadius: 25 })
+// Here we use render-target pixel sampling to keep detection aligned with the actual render output.
+const isTransparentByPixels = useCanvasPixelIsTransparentAtPoint(
+  stageCanvas,
+  relativeMouseX,
+  relativeMouseY,
+  { regionRadius: 25 },
+)
+const isTransparentByThree = useThreeSceneIsTransparentAtPoint(
+  widgetStageRef,
+  relativeMouseX,
+  relativeMouseY,
+  { regionRadius: 25 },
+)
+
+const { stageModelRenderer } = storeToRefs(useSettings())
+const isTransparent = computed(() => {
+  if (stageModelRenderer.value === 'vrm')
+    return isTransparentByThree.value
+
+  if (stageModelRenderer.value === 'live2d')
+    return isTransparentByPixels.value
+
+  return true
+})
+
 const { isNearAnyBorder: isAroundWindowBorder } = useElectronMouseAroundWindowBorder({ threshold: 30 })
 const isAroundWindowBorderFor250Ms = refDebounced(isAroundWindowBorder, 250)
 

packages/stage-ui-three/package.json

@@ -34,6 +34,7 @@
     "@tresjs/post-processing": "^3.2.0",
     "@vueuse/core": "^14.1.0",
     "culori": "^4.0.2",
+    "es-toolkit": "catalog:",
     "pinia": "^3.0.4",
     "postprocessing": "^6.38.2",
     "three": "^0.182.0",

packages/stage-ui-three/src/components/ThreeScene.vue

@@ -9,7 +9,7 @@
 
 import type { VRM } from '@pixiv/three-vrm'
 import type { TresContext } from '@tresjs/core'
-import type { DirectionalLight, SphericalHarmonics3, Texture, WebGLRenderTarget } from 'three'
+import type { DirectionalLight, SphericalHarmonics3, Texture, WebGLRenderer, WebGLRenderTarget } from 'three'
 
 import type { Vec3 } from '../stores/model-store'
 
@@ -29,9 +29,10 @@ import {
 } from 'three'
 import { onMounted, onUnmounted, ref, shallowRef, watch } from 'vue'
 
+// From stage-ui-three package
+import { useRenderTargetRegionAtClientPoint } from '../composables/render-target'
 // pinia store
 import { useModelStore } from '../stores/model-store'
-// From stage-ui-three package
 import { OrbitControls } from './Controls'
 import { SkyBox } from './Environment'
 import { VRMModel } from './Model'
@@ -99,6 +100,12 @@ const controlsRef = shallowRef<InstanceType<typeof OrbitControls>>()
 const tresCanvasRef = shallowRef<TresContext>()
 const skyBoxEnvRef = ref<InstanceType<typeof SkyBox>>()
 const dirLightRef = ref<InstanceType<typeof DirectionalLight>>()
+const { readRenderTargetRegionAtClientPoint, disposeRenderTarget } = useRenderTargetRegionAtClientPoint({
+  getRenderer: () => tresCanvasRef.value?.renderer.instance as WebGLRenderer | undefined,
+  getScene: () => tresCanvasRef.value?.scene.value,
+  getCamera: () => camera.value,
+  getCanvas: () => tresCanvasRef.value?.renderer.instance.domElement,
+})
 
 /*
   * Pinia store definition
@@ -194,7 +201,9 @@ onMounted(() => {
   }
 })
 
-onUnmounted(() => {})
+onUnmounted(() => {
+  disposeRenderTarget()
+})
 
 const effectProps = {
   saturation: 0.3,
@@ -291,6 +300,10 @@ defineExpose({
   canvasElement: () => {
     return tresCanvasRef.value?.renderer.instance.domElement
   },
+  camera: () => camera.value,
+  renderer: () => tresCanvasRef.value?.renderer.instance,
+  scene: () => modelRef.value?.scene,
+  readRenderTargetRegionAtClientPoint,
 })
 </script>
 

packages/stage-ui-three/src/composables/hit-test.ts

@@ -0,0 +1,71 @@
+import type { MaybeRefOrGetter, Ref } from 'vue'
+
+import type { RenderTargetRegionRead } from './render-target'
+
+import { toRef } from '@vueuse/core'
+import { computed } from 'vue'
+
+interface HitTestTarget {
+  readRenderTargetRegionAtClientPoint?: (clientX: number, clientY: number, radius: number) => RenderTargetRegionRead | null
+}
+
+export function useThreeSceneIsTransparentAtPoint(
+  scene: MaybeRefOrGetter<HitTestTarget | undefined>,
+  pointX: MaybeRefOrGetter<number>,
+  pointY: MaybeRefOrGetter<number>,
+  optionsOrThreshold: number | { threshold?: number, regionRadius?: number } = 10,
+): Ref<boolean> {
+  const sceneRef = toRef(scene)
+  const xRef = toRef(pointX)
+  const yRef = toRef(pointY)
+
+  const options = typeof optionsOrThreshold === 'number'
+    ? { threshold: optionsOrThreshold, regionRadius: 0 }
+    : optionsOrThreshold
+
+  const threshold = options?.threshold ?? 10
+  const radius = Math.max(0, options?.regionRadius ?? 0)
+
+  return computed(() => {
+    const instance = sceneRef.value
+    if (!instance || !instance.readRenderTargetRegionAtClientPoint)
+      return true
+
+    const result = instance.readRenderTargetRegionAtClientPoint(xRef.value, yRef.value, radius)
+    if (!result)
+      return true
+
+    const {
+      data,
+      readWidth,
+      readHeight,
+      startX,
+      startY,
+      centerX,
+      centerY,
+      scaleX,
+      scaleY,
+    } = result
+
+    const radiusSq = radius * radius
+    for (let y = 0; y < readHeight; y += 1) {
+      const gy = startY + y
+      const dy = (gy - centerY) / scaleY
+      const dySq = dy * dy
+
+      for (let x = 0; x < readWidth; x += 1) {
+        const gx = startX + x
+        const dx = (gx - centerX) / scaleX
+        if (dx * dx + dySq > radiusSq)
+          continue
+
+        const index = (y * readWidth + x) * 4
+        const alpha = data[index + 3]
+        if (alpha >= threshold)
+          return false
+      }
+    }
+
+    return true
+  })
+}

packages/stage-ui-three/src/composables/render-target.ts

@@ -0,0 +1,112 @@
+import type { Camera, Scene, WebGLRenderer } from 'three'
+
+import { clamp } from 'es-toolkit/math'
+import { Vector2, WebGLRenderTarget } from 'three'
+import { shallowRef } from 'vue'
+
+export interface RenderTargetRegionRead {
+  data: Uint8Array
+  readWidth: number
+  readHeight: number
+  startX: number
+  startY: number
+  centerX: number
+  centerY: number
+  scaleX: number
+  scaleY: number
+}
+
+export function useRenderTargetRegionAtClientPoint(context: {
+  getRenderer: () => WebGLRenderer | undefined
+  getScene: () => Scene | undefined
+  getCamera: () => Camera | undefined
+  getCanvas: () => HTMLCanvasElement | undefined
+}) {
+  const renderTargetRef = shallowRef<WebGLRenderTarget>()
+  const renderTargetSize = new Vector2()
+
+  function ensureRenderTarget(renderer: WebGLRenderer) {
+    // Match the offscreen target to the current drawing buffer size (DPI + canvas resize).
+    // Recreate the target when size changes to avoid reading stale or cropped pixels.
+    renderer.getDrawingBufferSize(renderTargetSize)
+    const width = Math.max(1, Math.floor(renderTargetSize.x))
+    const height = Math.max(1, Math.floor(renderTargetSize.y))
+
+    if (!renderTargetRef.value || renderTargetRef.value.width !== width || renderTargetRef.value.height !== height) {
+      // Dispose old target to keep GPU memory in check.
+      renderTargetRef.value?.dispose()
+      renderTargetRef.value = new WebGLRenderTarget(width, height, { depthBuffer: false })
+    }
+
+    return renderTargetRef.value
+  }
+
+  function readRenderTargetRegionAtClientPoint(clientX: number, clientY: number, radius: number): RenderTargetRegionRead | null {
+    const renderer = context.getRenderer()
+    const scene = context.getScene()
+    const camera = context.getCamera()
+    const canvas = context.getCanvas()
+    if (!renderer || !scene || !camera || !canvas)
+      return null
+
+    const rect = canvas.getBoundingClientRect()
+    const xIn = clientX - rect.left
+    const yIn = clientY - rect.top
+    const inCanvas = xIn >= 0 && yIn >= 0 && xIn < rect.width && yIn < rect.height
+    if (!inCanvas)
+      return null
+
+    const renderTarget = ensureRenderTarget(renderer)
+    const scaleX = renderTarget.width / rect.width
+    const scaleY = renderTarget.height / rect.height
+    if (!Number.isFinite(scaleX) || !Number.isFinite(scaleY))
+      return null
+
+    const centerX = Math.floor(xIn * scaleX)
+    const centerY = Math.floor(renderTarget.height - 1 - yIn * scaleY)
+
+    const radiusX = Math.ceil(radius * scaleX)
+    const radiusY = Math.ceil(radius * scaleY)
+
+    const startX = clamp(centerX - radiusX, 0, renderTarget.width - 1)
+    const endX = clamp(centerX + radiusX, 0, renderTarget.width - 1)
+    const startY = clamp(centerY - radiusY, 0, renderTarget.height - 1)
+    const endY = clamp(centerY + radiusY, 0, renderTarget.height - 1)
+
+    const readWidth = endX - startX + 1
+    const readHeight = endY - startY + 1
+    const data = new Uint8Array(readWidth * readHeight * 4)
+
+    const prevTarget = renderer.getRenderTarget()
+    // Render into our offscreen target so we can read pixels from it.
+    // Preserve the previous target to avoid breaking the main render pipeline.
+    renderer.setRenderTarget(renderTarget)
+    renderer.clear()
+    renderer.render(scene, camera)
+    renderer.readRenderTargetPixels(renderTarget, startX, startY, readWidth, readHeight, data)
+    // Restore the original target so downstream renders keep working.
+    renderer.setRenderTarget(prevTarget)
+
+    return {
+      data,
+      readWidth,
+      readHeight,
+      startX,
+      startY,
+      centerX,
+      centerY,
+      scaleX,
+      scaleY,
+    }
+  }
+
+  function disposeRenderTarget() {
+    renderTargetRef.value?.dispose()
+    renderTargetRef.value = undefined
+  }
+
+  return {
+    readRenderTargetRegionAtClientPoint,
+    disposeRenderTarget,
+  }
+}

packages/stage-ui-three/src/index.ts

@@ -1,3 +1,5 @@
 export { default as ThreeScene } from './components/ThreeScene.vue'
+export * from './composables/hit-test'
+export * from './composables/render-target'
 export { useModelStore } from './stores/model-store'
 export * from './utils/vrm-preview'

packages/stage-ui/src/components/scenes/Stage.vue

@@ -339,6 +339,13 @@ function canvasElement() {
     return vrmViewerRef.value?.canvasElement()
 }
 
+function readRenderTargetRegionAtClientPoint(clientX: number, clientY: number, radius: number) {
+  if (stageModelRenderer.value !== 'vrm')
+    return null
+
+  return vrmViewerRef.value?.readRenderTargetRegionAtClientPoint?.(clientX, clientY, radius) ?? null
+}
+
 onUnmounted(() => {
   if (lipSyncLoopId.value) {
     cancelAnimationFrame(lipSyncLoopId.value)
@@ -350,6 +357,7 @@ onUnmounted(() => {
 
 defineExpose({
   canvasElement,
+  readRenderTargetRegionAtClientPoint,
 })
 
 onPlaybackFinished(() => {

packages/stage-ui/src/composables/canvas-alpha.ts

@@ -132,7 +132,7 @@ export function useCanvasPixelAtPoint(
     const scaleY = gl.drawingBufferHeight / height.value
     const pixelX = Math.floor(xIn * scaleX)
     // Flip Y; subtract 1 to avoid top-edge off-by-one
-    const pixelY = Math.floor(gl.drawingBufferHeight - 1 - yIn * scaleY)
+    const pixelY = Math.floor(gl.drawingBufferHeight + yIn * scaleY)
 
     const data = new Uint8Array(4)
     try {
@@ -141,6 +141,7 @@ export function useCanvasPixelAtPoint(
     catch {
       return new Uint8Array([0, 0, 0, 0])
     }
+
     return data
   })
 

pnpm-lock.yaml

@@ -106,7 +106,7 @@ catalogs:
       specifier: ^8.6.0
       version: 8.6.0
     es-toolkit:
-      specifier: ^1.43.0
+      specifier: 1.43.0
       version: 1.43.0
     hono:
       specifier: 4.11.3
@@ -875,6 +875,12 @@ importers:
       '@proj-airi/stage-ui':
         specifier: workspace:^
         version: link:../../packages/stage-ui
+      '@proj-airi/stage-ui-live2d':
+        specifier: workspace:^
+        version: link:../../packages/stage-ui-live2d
+      '@proj-airi/stage-ui-three':
+        specifier: workspace:^
+        version: link:../../packages/stage-ui-three
       '@proj-airi/ui':
         specifier: workspace:^
         version: link:../../packages/ui
@@ -2600,6 +2606,9 @@ importers:
       culori:
         specifier: ^4.0.2
         version: 4.0.2
+      es-toolkit:
+        specifier: 'catalog:'
+        version: 1.43.0
       pinia:
         specifier: ^3.0.4
         version: 3.0.4(typescript@5.9.3)(vue@3.5.25(typescript@5.9.3))

pnpm-workspace.yaml

@@ -56,7 +56,7 @@ catalog:
   builder-util-runtime: ^9.3.1
   drizzle-valibot: ^0.4.2
   embla-carousel-vue: ^8.6.0
-  es-toolkit: ^1.43.0
+  es-toolkit: 1.43.0
   hono: 4.11.3
   injeca: ^0.1.5
   is-network-error: ^1.3.0