From b69abd2b5ab70aa1d72b5e7224f146c8426394eb Mon Sep 17 00:00:00 2001
From: Neko Ayaka <neko@ayaka.moe>
Date: Sat, 7 Dec 2024 03:49:43 +0800
Subject: [PATCH] feat: basic 3d emotion from LLM completions

---
 packages/stage/src/components/MainStage.vue   |  23 ++-
 packages/stage/src/components/ThreeDScene.vue | 179 +++++++++++-------
 packages/stage/src/components/VRMModel.vue    |  11 ++
 .../stage/src/composables/vrm/expression.ts   | 169 +++++++++++++++++
 packages/stage/src/constants/emotions.ts      |  10 +
 5 files changed, 321 insertions(+), 71 deletions(-)
 create mode 100644 packages/stage/src/composables/vrm/expression.ts

diff --git a/packages/stage/src/components/MainStage.vue b/packages/stage/src/components/MainStage.vue
index 4af4d1e9f..464a138db 100644
--- a/packages/stage/src/components/MainStage.vue
+++ b/packages/stage/src/components/MainStage.vue
@@ -11,7 +11,7 @@ import { useMarkdown } from '../composables/markdown'
 import { useQueue } from '../composables/queue'
 import { useDelayMessageQueue, useEmotionsMessageQueue, useMessageContentQueue } from '../composables/queues'
 import { llmInferenceEndToken } from '../constants'
-import { EMOTION_EmotionMotionName_value, EmotionThinkMotionName } from '../constants/emotions'
+import { EMOTION_EmotionMotionName_value, EMOTION_VRMExpressionName_value, EmotionThinkMotionName } from '../constants/emotions'
 import SystemPromptV2 from '../constants/prompts/system-v2'
 import { useLLM } from '../stores/llm'
 import { useSettings } from '../stores/settings'
@@ -25,9 +25,13 @@ import ThreeDScene from './ThreeDScene.vue'
 const nowSpeakingAvatarBorderOpacityMin = 30
 const nowSpeakingAvatarBorderOpacityMax = 100
 
-const { elevenLabsApiKey, openAiApiBaseURL, openAiApiKey } = storeToRefs(useSettings())
+const {
+  elevenLabsApiKey,
+  openAiApiBaseURL,
+  openAiApiKey,
+  stageView,
+} = storeToRefs(useSettings())
 const openAIModel = useLocalStorage<{ id: string, name?: string }>('settings/llm/openai/model', { id: 'openai/gpt-3.5-turbo', name: 'OpenAI GPT3.5 Turbo' })
-const stageView = useLocalStorage('settings/stage/view/model-renderer', '2d')
 
 const {
   streamSpeech,
@@ -39,6 +43,7 @@ const { process } = useMarkdown()
 
 const listening = ref(false)
 const live2DViewerRef = ref<{ setMotion: (motionName: string) => Promise<void> }>()
+const vrmViewerRef = ref<{ setExpression: (expression: string) => void }>()
 const supportedModels = ref<{ id: string, name?: string }[]>([])
 const messageInput = ref<string>('')
 const messages = ref<Array<Message>>([SystemPromptV2 as SystemMessage])
@@ -131,7 +136,16 @@ const messageContentQueue = useMessageContentQueue(ttsQueue)
 const emotionsQueue = useQueue<Emotion>({
   handlers: [
     async (ctx) => {
-      await live2DViewerRef.value!.setMotion(EMOTION_EmotionMotionName_value[ctx.data])
+      if (stageView.value === '3d') {
+        const value = EMOTION_VRMExpressionName_value[ctx.data]
+        if (!value)
+          return
+
+        await vrmViewerRef.value!.setExpression(value)
+      }
+      else if (stageView.value === '2d') {
+        await live2DViewerRef.value!.setMotion(EMOTION_EmotionMotionName_value[ctx.data])
+      }
     },
   ],
 })
@@ -289,6 +303,7 @@ onUnmounted(() => {
       />
       <ThreeDScene
         v-else-if="stageView === '3d'"
+        ref="vrmViewerRef"
         model="/assets/vrm/models/AvatarSample-B/AvatarSample_B.vrm"
         idle-animation="/assets/vrm/animations/idle_loop.vrma"
         w="50%" min-w="50% <lg:full" min-h="100 sm:100" h-full flex-1
diff --git a/packages/stage/src/components/ThreeDScene.vue b/packages/stage/src/components/ThreeDScene.vue
index 46c6f103b..c54f10a24 100644
--- a/packages/stage/src/components/ThreeDScene.vue
+++ b/packages/stage/src/components/ThreeDScene.vue
@@ -22,84 +22,128 @@ const cameraPositionZ = ref(-1)
 const vrmModelPositionX = ref(-0.18)
 const vrmModelPositionY = ref(-1.4)
 const vrmModelPositionZ = ref(-0.24)
+
+const modelRef = ref<{
+  setExpression: (expression: string) => void
+}>()
+
+defineExpose({
+  setExpression: (expression: string) => {
+    modelRef.value?.setExpression(expression)
+  },
+})
 </script>
 
 <template>
   <Screen v-slot="{ canvasHeight, canvasWidth }" relative>
-    <div z="10" top="2" absolute w-full gap-2 flex="~ col md:row">
-      <Collapsable h-fit w-full>
-        <template #label>
-          <span font-mono>Camera</span>
-        </template>
-        <div grid="~ cols-[20px_1fr_60px]" w-full gap-1 p-2 text-sm font-mono>
-          <div text="zinc-400 dark:zinc-500">
-            <span>X</span>
-          </div>
-          <label w-full flex items-center gap-2>
-            <DataGuiRange v-model="cameraPositionX" :min="-10" :max="10" :step="0.01" />
-          </label>
-          <div text-right>
-            <span>{{ cameraPositionX }}</span>
-          </div>
+    <div z="10" top="2" absolute w-full flex="~ col" gap-2>
+      <div w-full flex="~ col md:row" gap-2>
+        <Collapsable h-fit w-full>
+          <template #label>
+            <span font-mono>Camera</span>
+          </template>
+          <div grid="~ cols-[20px_1fr_60px]" w-full gap-1 p-2 text-sm font-mono>
+            <div text="zinc-400 dark:zinc-500">
+              <span>X</span>
+            </div>
+            <label w-full flex items-center gap-2>
+              <DataGuiRange v-model="cameraPositionX" :min="-10" :max="10" :step="0.01" />
+            </label>
+            <div text-right>
+              <span>{{ cameraPositionX }}</span>
+            </div>
 
-          <div text="zinc-400 dark:zinc-500">
-            <span>Y</span>
-          </div>
-          <label w-full flex items-center gap-2>
-            <DataGuiRange v-model="cameraPositionY" :min="-10" :max="10" :step="0.01" />
-          </label>
-          <div text-right>
-            <span>{{ cameraPositionY }}</span>
-          </div>
+            <div text="zinc-400 dark:zinc-500">
+              <span>Y</span>
+            </div>
+            <label w-full flex items-center gap-2>
+              <DataGuiRange v-model="cameraPositionY" :min="-10" :max="10" :step="0.01" />
+            </label>
+            <div text-right>
+              <span>{{ cameraPositionY }}</span>
+            </div>
 
-          <div text="zinc-400 dark:zinc-500">
-            <span>Z</span>
-          </div>
-          <label w-full flex items-center gap-2>
-            <DataGuiRange v-model="cameraPositionZ" :min="-10" :max="10" :step="0.01" />
-          </label>
-          <div text-right>
-            <span>{{ cameraPositionZ }}</span>
-          </div>
-        </div>
-      </Collapsable>
-      <Collapsable h-fit w-full>
-        <template #label>
-          <span font-mono>Model</span>
-        </template>
-        <div grid="~ cols-[20px_1fr_60px]" w-full gap-1 p-2 text-sm font-mono>
-          <div text="zinc-400 dark:zinc-500">
-            <span>X</span>
+            <div text="zinc-400 dark:zinc-500">
+              <span>Z</span>
+            </div>
+            <label w-full flex items-center gap-2>
+              <DataGuiRange v-model="cameraPositionZ" :min="-10" :max="10" :step="0.01" />
+            </label>
+            <div text-right>
+              <span>{{ cameraPositionZ }}</span>
+            </div>
           </div>
+        </Collapsable>
+        <Collapsable h-fit w-full>
+          <template #label>
+            <span font-mono>Model</span>
+          </template>
+          <div grid="~ cols-[20px_1fr_60px]" w-full gap-1 p-2 text-sm font-mono>
+            <div text="zinc-400 dark:zinc-500">
+              <span>X</span>
+            </div>
 
-          <label w-full flex items-center gap-2>
-            <DataGuiRange v-model="vrmModelPositionX" :min="-10" :max="10" :step="0.01" />
-          </label>
-          <div text-right>
-            <span>{{ vrmModelPositionX }}</span>
-          </div>
+            <label w-full flex items-center gap-2>
+              <DataGuiRange v-model="vrmModelPositionX" :min="-10" :max="10" :step="0.01" />
+            </label>
+            <div text-right>
+              <span>{{ vrmModelPositionX }}</span>
+            </div>
 
-          <div text="zinc-400 dark:zinc-500">
-            <span>Y</span>
-          </div>
-          <label w-full flex items-center gap-2>
-            <DataGuiRange v-model="vrmModelPositionY" :min="-10" :max="10" :step="0.01" />
-          </label>
-          <div text-right>
-            <span>{{ vrmModelPositionY }}</span>
-          </div>
+            <div text="zinc-400 dark:zinc-500">
+              <span>Y</span>
+            </div>
+            <label w-full flex items-center gap-2>
+              <DataGuiRange v-model="vrmModelPositionY" :min="-10" :max="10" :step="0.01" />
+            </label>
+            <div text-right>
+              <span>{{ vrmModelPositionY }}</span>
+            </div>
 
-          <div text="zinc-400 dark:zinc-500">
-            <span>Z</span>
+            <div text="zinc-400 dark:zinc-500">
+              <span>Z</span>
+            </div>
+            <label w-full flex items-center gap-2>
+              <DataGuiRange v-model="vrmModelPositionZ" :min="-10" :max="10" :step="0.01" />
+            </label>
+            <div text-right>
+              <span>{{ vrmModelPositionZ }}</span>
+            </div>
           </div>
-          <label w-full flex items-center gap-2>
-            <DataGuiRange v-model="vrmModelPositionZ" :min="-10" :max="10" :step="0.01" />
-          </label>
-          <div text-right>
-            <span>{{ vrmModelPositionZ }}</span>
-          </div>
-        </div>
-      </Collapsable>
+        </Collapsable>
+      </div>
+      <div flex="~ row" w-full gap-2>
+        <button
+          rounded-lg bg="zinc-200 dark:zinc-800/50" px-2 py-1
+          @click="modelRef?.setExpression('neutral')"
+        >
+          🙂 Neutral
+        </button>
+        <button
+          rounded-lg bg="zinc-200 dark:zinc-800/50" px-2 py-1
+          @click="modelRef?.setExpression('surprised')"
+        >
+          🤯 Surprised
+        </button>
+        <button
+          rounded-lg bg="zinc-200 dark:zinc-800/50" px-2 py-1
+          @click="modelRef?.setExpression('sad')"
+        >
+          😫 Sad
+        </button>
+        <button
+          rounded-lg bg="zinc-200 dark:zinc-800/50" px-2 py-1
+          @click="modelRef?.setExpression('angry')"
+        >
+          😠 Angry
+        </button>
+        <button
+          rounded-lg bg="zinc-200 dark:zinc-800/50" px-2 py-1
+          @click="modelRef?.setExpression('happy')"
+        >
+          😄 Happy
+        </button>
+      </div>
     </div>
     <TresCanvas :alpha="true" :antialias="true" :width="canvasWidth" :height="canvasHeight">
       <OrbitControls />
@@ -107,6 +151,7 @@ const vrmModelPositionZ = ref(-0.24)
       <TresDirectionalLight :color="0xFFFFFF" :intensity="1.2" :position="[1, 1, 1]" />
       <TresAmbientLight :color="0xFFFFFF" :intensity="1.5" />
       <VRMModel
+        ref="modelRef"
         :model="props.model"
         :idle-animation="props.idleAnimation"
         :position="[vrmModelPositionX, vrmModelPositionY, vrmModelPositionZ]"
diff --git a/packages/stage/src/components/VRMModel.vue b/packages/stage/src/components/VRMModel.vue
index c293e6ab2..7ccc34086 100644
--- a/packages/stage/src/components/VRMModel.vue
+++ b/packages/stage/src/components/VRMModel.vue
@@ -4,6 +4,7 @@ import { useLoop, useTresContext } from '@tresjs/core'
 import { AnimationMixer } from 'three'
 import { clipFromVRMAnimation, loadVRMAnimation, useBlink } from '~/composables/vrm/animation'
 import { loadVrm } from '~/composables/vrm/core'
+import { useVRMEmote } from '~/composables/vrm/expression'
 
 const props = defineProps<{
   model: string
@@ -22,6 +23,7 @@ const vrmAnimationMixer = ref<AnimationMixer>()
 const { scene } = useTresContext()
 const { onBeforeRender } = useLoop()
 const blink = useBlink()
+const vrmEmote = ref<ReturnType<typeof useVRMEmote>>()
 
 watch(() => props.position, ([x, y, z]) => {
   if (vrm.value) {
@@ -57,10 +59,13 @@ onMounted(async () => {
     vrmAnimationMixer.value = new AnimationMixer(_vrm.scene)
     vrmAnimationMixer.value.clipAction(clip).play()
 
+    vrmEmote.value = useVRMEmote(_vrm)
+
     onBeforeRender(({ delta }) => {
       vrmAnimationMixer.value?.update(delta)
       vrm.value?.update(delta)
       blink.update(vrm.value, delta)
+      vrmEmote.value?.update(delta)
     })
 
     vrm.value = _vrm
@@ -76,6 +81,12 @@ onUnmounted(() => {
     scene.value.remove(vrm.value.scene)
   }
 })
+
+defineExpose({
+  setExpression(expression: string) {
+    vrmEmote.value?.setEmotionWithResetAfter(expression, 1000)
+  },
+})
 </script>
 
 <template>
diff --git a/packages/stage/src/composables/vrm/expression.ts b/packages/stage/src/composables/vrm/expression.ts
new file mode 100644
index 000000000..8ddefda78
--- /dev/null
+++ b/packages/stage/src/composables/vrm/expression.ts
@@ -0,0 +1,169 @@
+import type { VRMCore } from '@pixiv/three-vrm-core'
+import { ref } from 'vue'
+
+interface EmotionState {
+  expression?: {
+    name: string
+    value: number
+    duration?: number
+    curve?: (t: number) => number
+  }[]
+  blendDuration?: number
+}
+
+export function useVRMEmote(vrm: VRMCore) {
+  const currentEmotion = ref<string | null>(null)
+  const isTransitioning = ref(false)
+  const transitionProgress = ref(0)
+  const currentExpressionValues = ref(new Map<string, number>())
+  const targetExpressionValues = ref(new Map<string, number>())
+  const resetTimeout = ref<number>()
+
+  // Utility functions
+  const lerp = (start: number, end: number, t: number): number => {
+    return start + (end - start) * t
+  }
+
+  const easeInOutCubic = (t: number): number => {
+    return t < 0.5 ? 4 * t * t * t : 1 - (-2 * t + 2) ** 3 / 2
+  }
+
+  // Emotion states definition
+  const emotionStates = new Map<string, EmotionState>([
+    ['happy', {
+      expression: [
+        { name: 'happy', value: 1.0, duration: 0.3 },
+        { name: 'aa', value: 0.3 },
+      ],
+      blendDuration: 0.3,
+    }],
+    ['sad', {
+      expression: [
+        { name: 'sad', value: 1.0 },
+        { name: 'oh', value: 0.2 },
+      ],
+      blendDuration: 0.3,
+    }],
+    ['angry', {
+      expression: [
+        { name: 'angry', value: 1.0 },
+        { name: 'ee', value: 0.4 },
+      ],
+      blendDuration: 0.2,
+    }],
+    ['surprised', {
+      expression: [
+        { name: 'Surprised', value: 1.0 },
+        { name: 'oh', value: 0.6 },
+      ],
+      blendDuration: 0.1,
+    }],
+    ['neutral', {
+      expression: [
+        { name: 'neutral', value: 1.0 },
+      ],
+      blendDuration: 0.5,
+    }],
+  ])
+
+  const clearResetTimeout = () => {
+    if (resetTimeout.value) {
+      clearTimeout(resetTimeout.value)
+      resetTimeout.value = undefined
+    }
+  }
+
+  const setEmotion = (emotionName: string) => {
+    clearResetTimeout()
+
+    if (!emotionStates.has(emotionName)) {
+      console.warn(`Emotion ${emotionName} not found`)
+      return
+    }
+
+    const emotionState = emotionStates.get(emotionName)!
+    currentEmotion.value = emotionName
+    isTransitioning.value = true
+    transitionProgress.value = 0
+
+    // Reset all existing expressions to 0 first
+    if (vrm.expressionManager) {
+      // Get all expression names from the VRM model
+      const expressionNames = Object.keys(vrm.expressionManager.expressionMap)
+      for (const name of expressionNames) {
+        vrm.expressionManager.setValue(name, 0)
+      }
+    }
+
+    // Store current expression values as starting point
+    currentExpressionValues.value.clear()
+    targetExpressionValues.value.clear()
+
+    // Store all current expression values
+    for (const expr of emotionState.expression || []) {
+      const currentValue = vrm.expressionManager?.getValue(expr.name) || 0
+      currentExpressionValues.value.set(expr.name, currentValue)
+      targetExpressionValues.value.set(expr.name, expr.value)
+    }
+  }
+
+  const setEmotionWithResetAfter = (emotionName: string, ms: number) => {
+    clearResetTimeout()
+    setEmotion(emotionName)
+
+    // Set timeout to reset to neutral
+    resetTimeout.value = setTimeout(() => {
+      setEmotion('neutral')
+      resetTimeout.value = undefined
+    }, ms) as unknown as number
+  }
+
+  const update = (deltaTime: number) => {
+    if (!isTransitioning.value || !currentEmotion.value)
+      return
+
+    const emotionState = emotionStates.get(currentEmotion.value)!
+    const blendDuration = emotionState.blendDuration || 0.3
+
+    transitionProgress.value += deltaTime / blendDuration
+    if (transitionProgress.value >= 1.0) {
+      transitionProgress.value = 1.0
+      isTransitioning.value = false
+    }
+
+    // Update all expressions
+    for (const [exprName, targetValue] of targetExpressionValues.value) {
+      const startValue = currentExpressionValues.value.get(exprName) || 0
+      const currentValue = lerp(
+        startValue,
+        targetValue,
+        easeInOutCubic(transitionProgress.value),
+      )
+      vrm.expressionManager?.setValue(exprName, currentValue)
+    }
+  }
+
+  const addEmotionState = (emotionName: string, state: EmotionState) => {
+    emotionStates.set(emotionName, state)
+  }
+
+  const removeEmotionState = (emotionName: string) => {
+    emotionStates.delete(emotionName)
+  }
+
+  // Cleanup function
+  const dispose = () => {
+    clearResetTimeout()
+  }
+
+  return {
+    currentEmotion,
+    isTransitioning,
+    setEmotion,
+    setEmotionWithResetAfter,
+    update,
+    addEmotionState,
+    removeEmotionState,
+    dispose,
+  }
+}
diff --git a/packages/stage/src/constants/emotions.ts b/packages/stage/src/constants/emotions.ts
index 118ccd9c3..6f1ab1634 100644
--- a/packages/stage/src/constants/emotions.ts
+++ b/packages/stage/src/constants/emotions.ts
@@ -35,3 +35,13 @@ export const EMOTION_EmotionMotionName_value = {
   [Emotion.Awkward]: EmotionAwkwardMotionName,
   [Emotion.Question]: EmotionQuestionMotionName,
 }
+
+export const EMOTION_VRMExpressionName_value = {
+  [Emotion.Happy]: 'happy',
+  [Emotion.Sad]: 'sad',
+  [Emotion.Angry]: 'angry',
+  [Emotion.Think]: undefined,
+  [Emotion.Surprise]: 'surprised',
+  [Emotion.Awkward]: undefined,
+  [Emotion.Question]: undefined,
+} satisfies Record<Emotion, string | undefined>