feat(stage-ui): animate idle eye and focus movements on Live2D models (#23)

packages/stage-ui/src/components/Live2D/Viewer.vue

@@ -10,6 +10,8 @@ import { DropShadowFilter } from 'pixi-filters'
 import { Live2DModel, MotionPreloadStrategy, MotionPriority } from 'pixi-live2d-display/cubism4'
 import { computed, onMounted, onUnmounted, ref, watch } from 'vue'
 
+import { useLive2DIdleEyeFocus } from '../../composables/live2d'
+
 const props = withDefaults(defineProps<{
   model: string
   mouthOpenSize?: number
@@ -32,6 +34,8 @@ const breakpoints = useBreakpoints(breakpointsTailwind)
 const isMobile = computed(() => breakpoints.between('sm', 'md').value || breakpoints.smaller('sm').value)
 const { height, width } = useElementBounding(containerRef, { immediate: true, windowResize: true, reset: true })
 
+const idleEyeFocus = useLive2DIdleEyeFocus()
+
 function getCoreModel() {
   return model.value!.internalModel.coreModel as any
 }
@@ -84,8 +88,34 @@ async function initLive2DPixiStage(parent: HTMLDivElement) {
       model.value.motion('tap_body')
   })
 
-  const coreModel = model.value.internalModel.coreModel as any
+  const internalModel = model.value.internalModel
+  const coreModel = internalModel.coreModel as any
+  const motionManager = internalModel.motionManager
   coreModel.setParameterValueById('ParamMouthOpenY', mouthOpenSize.value)
+
+  // Remove eye ball movements from idle motion group to prevent conflicts
+  // This is too hacky
+  if (motionManager.groups.idle) {
+    motionManager.motionGroups[motionManager.groups.idle]?.forEach((motion) => {
+      motion._motionData.curves.forEach((curve: any) => {
+        // TODO: After emotion mapper, stage editor, eye related parameters should be take cared to be dynamical instead of hardcoding
+        if (curve.id === 'ParamEyeBallX' || curve.id === 'ParamEyeBallY') {
+          curve.id = `_${curve.id}`
+        }
+      })
+    })
+  }
+
+  // This is hacky too
+  const hookedUpdate = motionManager.update
+  motionManager.update = function (model, now) {
+    hookedUpdate?.call(this, model, now)
+    // Only update eye focus when the model is idle
+    if (motionManager.state.currentGroup === motionManager.groups.idle) {
+      idleEyeFocus.update(internalModel, now)
+    }
+    return true
+  }
 }
 
 async function setMotion(motionName: string) {

packages/stage-ui/src/composables/live2d/animation.ts

@@ -0,0 +1,34 @@
+import type { InternalModel } from 'pixi-live2d-display/cubism4'
+
+import { lerp, randFloat } from 'three/src/math/MathUtils.js'
+
+import { randomSaccadeInterval } from '../../utils'
+
+/**
+ * This is to simulate idle eye saccades and focus (head) movements in a *pretty* naive way.
+ * Not using any reactivity here as it's not yet needed.
+ * Keeping it here as a composable for future extension.
+ */
+export function useLive2DIdleEyeFocus() {
+  let nextSaccadeAfter = -1
+  let focusTarget: [number, number] | undefined
+  let lastSaccadeAt = -1
+
+  // Function to handle idle eye saccades and focus (head) movements
+  function update(model: InternalModel, now: number) {
+    if (now >= nextSaccadeAfter || now < lastSaccadeAt) {
+      focusTarget = [randFloat(-1, 1), randFloat(-1, 0.7)]
+      lastSaccadeAt = now
+      nextSaccadeAfter = now + (randomSaccadeInterval() / 1000)
+      model.focusController.focus(focusTarget![0] * 0.5, focusTarget![1] * 0.5, false)
+    }
+
+    model.focusController.update(now - lastSaccadeAt)
+    const coreModel = model.coreModel as any
+    // TODO: After emotion mapper, stage editor, eye related parameters should be take cared to be dynamical instead of hardcoding
+    coreModel.setParameterValueById('ParamEyeBallX', lerp(coreModel.getParameterValueById('ParamEyeBallX'), focusTarget![0], 0.3))
+    coreModel.setParameterValueById('ParamEyeBallY', lerp(coreModel.getParameterValueById('ParamEyeBallY'), focusTarget![1], 0.3))
+  }
+
+  return { update }
+}

packages/stage-ui/src/composables/live2d/index.ts

@@ -0,0 +1 @@
+export * from './animation'

packages/stage-ui/src/composables/vrm/animation.ts

@@ -6,6 +6,7 @@ import { Object3D, Vector3 } from 'three'
 import { randFloat } from 'three/src/math/MathUtils.js'
 import { ref } from 'vue'
 
+import { randomSaccadeInterval } from '../../utils'
 import { useVRMLoader } from './loader'
 
 export interface GLTFUserdata extends Record<string, any> {
@@ -102,34 +103,6 @@ export function useIdleEyeSaccades() {
   let fixationTarget: Vector3 | undefined
   let timeSinceLastSaccade = 0
 
-  const STEP = 400
-  const P = [
-    [0.075, 800],
-    [0.110, 0],
-    [0.125, 0],
-    [0.140, 0],
-    [0.125, 0],
-    [0.050, 0],
-    [0.040, 0],
-    [0.030, 0],
-    [0.020, 0],
-    [1.000, 0],
-  ]
-  for (let i = 1; i < P.length; i++) {
-    P[i][0] += P[i - 1][0]
-    P[i][1] = P[i - 1][1] + STEP
-  }
-
-  function nextSaccadeTimeMs() {
-    const r = Math.random()
-    for (let i = 0; i < P.length; i++) {
-      if (r <= P[i][0]) {
-        return P[i][1] + Math.random() * STEP
-      }
-    }
-    return P[P.length - 1][1] + Math.random() * STEP
-  }
-
   // Just a naive vector generator - Simulating random content on a 27in monitor at 65cm distance
   function updateFixationTarget() {
     if (!fixationTarget) {
@@ -148,7 +121,7 @@ export function useIdleEyeSaccades() {
     if (timeSinceLastSaccade >= nextSaccadeAfter) {
       updateFixationTarget()
       timeSinceLastSaccade = 0
-      nextSaccadeAfter = nextSaccadeTimeMs() / 1000
+      nextSaccadeAfter = randomSaccadeInterval() / 1000
     }
     else if (!fixationTarget) {
       updateFixationTarget()

packages/stage-ui/src/utils/eye-motions.ts

@@ -0,0 +1,32 @@
+const EYE_SACCADE_INT_STEP = 400
+const EYE_SACCADE_INT_P = [
+  [0.075, 800],
+  [0.110, 0],
+  [0.125, 0],
+  [0.140, 0],
+  [0.125, 0],
+  [0.050, 0],
+  [0.040, 0],
+  [0.030, 0],
+  [0.020, 0],
+  [1.000, 0],
+]
+for (let i = 1; i < EYE_SACCADE_INT_P.length; i++) {
+  EYE_SACCADE_INT_P[i][0] += EYE_SACCADE_INT_P[i - 1][0]
+  EYE_SACCADE_INT_P[i][1] = EYE_SACCADE_INT_P[i - 1][1] + EYE_SACCADE_INT_STEP
+}
+
+/**
+ * This is a simple function to generate a random interval between eye saccades.
+ *
+ * @returns Interval in milliseconds
+ */
+export function randomSaccadeInterval(): number {
+  const r = Math.random()
+  for (let i = 0; i < EYE_SACCADE_INT_P.length; i++) {
+    if (r <= EYE_SACCADE_INT_P[i][0]) {
+      return EYE_SACCADE_INT_P[i][1] + Math.random() * EYE_SACCADE_INT_STEP
+    }
+  }
+  return EYE_SACCADE_INT_P[EYE_SACCADE_INT_P.length - 1][1] + Math.random() * EYE_SACCADE_INT_STEP
+}

packages/stage-ui/src/utils/index.ts

@@ -1 +1,2 @@
+export * from './eye-motions'
 export * from './iterator'