feat(stage-ui): animate idle eye saccades on VRMs (#17)

* feat(stage-ui): animate idle eye saccades on VRMs
* chore: clean up VRM upon unmounting
* fix: missing component imports
* fix: netlify

cspell.config.yaml

@@ -90,6 +90,8 @@ words:
   - pthread
   - rehype
   - rushstack
+  - Saccade
+  - saccades
   - Shadcn
   - shiki
   - shikijs

packages/moonshine-web/netlify.toml

@@ -3,7 +3,7 @@ publish = "packages/moonshine-web/dist"
 command = "pnpm run packages:stub && pnpm run build"
 
 [build.environment]
-NODE_VERSION = "22"
+NODE_VERSION = "23"
 
 [[redirects]]
 from = "/assets/*"

packages/stage-ui/src/components/Scenes/VRM.vue

@@ -6,6 +6,7 @@ import { ref } from 'vue'
 import Collapsable from '../Collapsable.vue'
 import DataGuiRange from '../DataGui/Range.vue'
 import Screen from '../Screen.vue'
+import TransitionVertical from '../TransitionVertical.vue'
 import VRMModel from '../VRM/Model.vue'
 
 const props = defineProps<{

packages/stage-ui/src/components/VRM/Model.vue

@@ -1,10 +1,11 @@
 <script setup lang="ts">
 import type { VRMCore } from '@pixiv/three-vrm-core'
+import { VRMUtils } from '@pixiv/three-vrm'
 import { useLoop, useTresContext } from '@tresjs/core'
 import { AnimationMixer } from 'three'
 
 import { onMounted, onUnmounted, ref, watch } from 'vue'
-import { clipFromVRMAnimation, loadVRMAnimation, useBlink } from '../../composables/vrm/animation'
+import { clipFromVRMAnimation, loadVRMAnimation, useBlink, useIdleEyeSaccades } from '../../composables/vrm/animation'
 import { loadVrm } from '../../composables/vrm/core'
 import { useVRMEmote } from '../../composables/vrm/expression'
 
@@ -20,11 +21,14 @@ const emit = defineEmits<{
   (e: 'error', value: unknown): void
 }>()
 
+let disposeBeforeRenderLoop: (() => void | undefined)
+
 const vrm = ref<VRMCore>()
 const vrmAnimationMixer = ref<AnimationMixer>()
 const { scene } = useTresContext()
 const { onBeforeRender } = useLoop()
 const blink = useBlink()
+const idleEyeSaccades = useIdleEyeSaccades()
 const vrmEmote = ref<ReturnType<typeof useVRMEmote>>()
 
 watch(() => props.position, ([x, y, z]) => {
@@ -63,14 +67,15 @@ onMounted(async () => {
 
     vrmEmote.value = useVRMEmote(_vrm)
 
-    onBeforeRender(({ delta }) => {
+    vrm.value = _vrm
+
+    disposeBeforeRenderLoop = onBeforeRender(({ delta }) => {
       vrmAnimationMixer.value?.update(delta)
       vrm.value?.update(delta)
       blink.update(vrm.value, delta)
+      idleEyeSaccades.update(vrm.value, delta)
       vrmEmote.value?.update(delta)
-    })
-
-    vrm.value = _vrm
+    }).off
   }
   catch (err) {
     emit('error', err)
@@ -78,9 +83,10 @@ onMounted(async () => {
 })
 
 onUnmounted(() => {
+  disposeBeforeRenderLoop?.()
   if (vrm.value) {
-    const { scene } = useTresContext()
-    scene.value.remove(vrm.value.scene)
+    vrm.value.scene.removeFromParent()
+    VRMUtils.deepDispose(vrm.value.scene)
   }
 })
 

packages/stage-ui/src/composables/vrm/animation.ts

@@ -1,6 +1,8 @@
 import type { VRMAnimation } from '@pixiv/three-vrm-animation'
 import type { VRMCore } from '@pixiv/three-vrm-core'
 import { createVRMAnimationClip } from '@pixiv/three-vrm-animation'
+import { Object3D, Vector3 } from 'three'
+import { randFloat } from 'three/src/math/MathUtils.js'
 import { ref } from 'vue'
 import { useVRMLoader } from './loader'
 
@@ -87,3 +89,76 @@ export function useBlink() {
 
   return { update }
 }
+
+/**
+ * This is to simulate idle eye saccades in a *pretty* naive way.
+ * Not using any reactivity here as it's not yet needed.
+ * Keeping it here as a composable for future extension.
+ */
+export function useIdleEyeSaccades() {
+  let nextSaccadeAfter = -1
+  let fixationTarget: Vector3 | undefined
+  let timeSinceLastSaccade = 0
+
+  const STEP = 400
+  const P = [
+    [0.075, 800],
+    [0.110, 0],
+    [0.125, 0],
+    [0.140, 0],
+    [0.125, 0],
+    [0.050, 0],
+    [0.040, 0],
+    [0.030, 0],
+    [0.020, 0],
+    [1.000, 0],
+  ]
+  for (let i = 1; i < P.length; i++) {
+    P[i][0] += P[i - 1][0]
+    P[i][1] = P[i - 1][1] + STEP
+  }
+
+  function nextSaccadeTimeMs() {
+    const r = Math.random()
+    for (let i = 0; i < P.length; i++) {
+      if (r <= P[i][0]) {
+        return P[i][1] + Math.random() * STEP
+      }
+    }
+    return P[P.length - 1][1] + Math.random() * STEP
+  }
+
+  // Just a naive vector generator - Simulating random content on a 27in monitor at 65cm distance
+  function updateFixationTarget() {
+    if (!fixationTarget) {
+      fixationTarget = new Vector3(randFloat(-0.25, 0.25), randFloat(-0.2, 0.15), -0.65)
+    }
+    else {
+      fixationTarget.set(randFloat(-0.25, 0.25), randFloat(-0.2, 0.15), -0.65)
+    }
+  }
+
+  // Function to handle idle eye saccades
+  function update(vrm: VRMCore | undefined, delta: number) {
+    if (!vrm?.expressionManager || !vrm.lookAt)
+      return
+
+    if (timeSinceLastSaccade >= nextSaccadeAfter) {
+      updateFixationTarget()
+      timeSinceLastSaccade = 0
+      nextSaccadeAfter = nextSaccadeTimeMs() / 1000
+    }
+    else if (!fixationTarget) {
+      updateFixationTarget()
+    }
+
+    if (!vrm.lookAt.target) {
+      vrm.lookAt.target = new Object3D()
+    }
+    vrm.lookAt.target.position.lerp(fixationTarget!, randFloat(0.2, 0.5))
+    vrm.lookAt?.update(delta)
+    timeSinceLastSaccade += delta
+  }
+
+  return { update }
+}

packages/stage-web/netlify.toml

@@ -3,7 +3,7 @@ publish = "packages/stage-web/dist"
 command = "pnpm run packages:stub && pnpm run build"
 
 [build.environment]
-NODE_VERSION = "22"
+NODE_VERSION = "23"
 
 [[redirects]]
 from = "/assets/*"

packages/stage-web/src/components/Layouts/InteractiveArea.vue

@@ -1,9 +1,9 @@
 <script setup lang="ts">
+import { BasicTextarea, TransitionVertical } from '@proj-airi/stage-ui/components'
 import { useMicVAD, useWhisper } from '@proj-airi/stage-ui/composables'
 
 import WhisperWorker from '@proj-airi/stage-ui/libs/workers/worker?worker&url'
 import { useAudioContext, useChatStore, useSettings } from '@proj-airi/stage-ui/stores'
-
 import { useDevicesList } from '@vueuse/core'
 import { storeToRefs } from 'pinia'
 import { ref, watch } from 'vue'

packages/whisper-webgpu/netlify.toml

@@ -3,7 +3,7 @@ publish = "packages/whisper-webgpu/dist"
 command = "pnpm run packages:stub && pnpm run build"
 
 [build.environment]
-NODE_VERSION = "22"
+NODE_VERSION = "23"
 
 [[redirects]]
 from = "/assets/*"