feat(stage-ui): vrm lip-sync (#204)

* feat(stage-ui): vrm lip-sync

* fix: update package.json

---------

Co-authored-by: Neko <neko@ayaka.moe>

cspell.config.yaml

@@ -192,6 +192,7 @@ words:
   - wavefile
   - webgpu
   - weeb
+  - wlipsync
   - worklet
   - xast
   - xastscript

packages/stage-ui/package.json

@@ -108,6 +108,7 @@
     "reka-ui": "^2.3.1",
     "unist-builder": "^4.0.0",
     "unspeech": "^0.1.7",
+    "wlipsync": "^1.2.1",
     "xast-util-to-xml": "^4.0.0",
     "xastscript": "^4.0.0"
   },

packages/stage-ui/src/composables/vrm/lip-sync.ts

@@ -0,0 +1,38 @@
+import type { VRMCore } from '@pixiv/three-vrm-core'
+import type { Profile } from 'wlipsync'
+
+import { useAsyncState } from '@vueuse/core'
+import { onUnmounted, watch } from 'vue'
+import { createWLipSyncNode } from 'wlipsync'
+
+import profile from '../../assets/lip-sync-profile.json' with { type: 'json' }
+import { useAudioContext } from '../../stores/audio'
+
+export function useVRMLipSync(audioNode: AudioNode) {
+  const { audioContext } = useAudioContext()
+  const { state: lipSyncNode, isReady } = useAsyncState(createWLipSyncNode(audioContext, profile as Profile), undefined)
+
+  // https://github.com/mrxz/wLipSync/blob/c3bc4b321dc7e1ca333d75f7aa1e9e746cbbb23a/example/index.js#L50-L66
+  const lipSyncMap = {
+    A: 'aa',
+    E: 'ee',
+    I: 'ih',
+    O: 'oh',
+    U: 'ou',
+  }
+
+  watch(isReady, () => isReady.value && audioNode.connect(lipSyncNode.value!))
+  onUnmounted(() => audioNode.disconnect())
+
+  function update(vrm: VRMCore | undefined) {
+    if (!vrm?.expressionManager || !lipSyncNode.value)
+      return
+
+    for (const key of Object.keys(lipSyncNode.value.weights)) {
+      const weight = lipSyncNode.value.weights[key] * lipSyncNode.value.volume
+      vrm.expressionManager?.setValue(lipSyncMap[key as keyof typeof lipSyncMap], weight)
+    }
+  }
+
+  return { update }
+}

pnpm-lock.yaml

@@ -970,6 +970,9 @@ importers:
       unspeech:
         specifier: ^0.1.7
         version: 0.1.7
+      wlipsync:
+        specifier: ^1.2.1
+        version: 1.2.1
       xast-util-to-xml:
         specifier: ^4.0.0
         version: 4.0.0
@@ -11760,6 +11763,9 @@ packages:
     resolution: {integrity: sha512-c9bZp7b5YtRj2wOe6dlj32MK+Bx/M/d+9VB2SHM1OtsUHR0aV0tdP6DWh/iMt0kWi1t5g1Iudu6hQRNd1A4PVA==}
     engines: {node: '>=18'}
 
+  wlipsync@1.2.1:
+    resolution: {integrity: sha512-kyT1mcuqEyX7mrhYRfhWYg6g5Ymub2v3JRE/aBzhi+JUN6p11NFtz6PxSSUv4lS/h1lt0Sbk981xEuk8aw0p1g==}
+
   wordwrapjs@5.1.0:
     resolution: {integrity: sha512-JNjcULU2e4KJwUNv6CHgI46UvDGitb6dGryHajXTDiLgg1/RiGoPSDw4kZfYnwGtEXf2ZMeIewDQgFGzkCB2Sg==}
     engines: {node: '>=12.17'}
@@ -24748,6 +24754,8 @@ snapshots:
     dependencies:
       string-width: 7.2.0
 
+  wlipsync@1.2.1: {}
+
   wordwrapjs@5.1.0: {}
 
   workbox-background-sync@7.3.0: