refactor: cleanup code

Signed-off-by: Neko Ayaka <neko@ayaka.moe>

components/MainStage.vue

@@ -17,6 +17,9 @@ interface Message {
   content: string
 }
 
+const nowSpeakingAvatarBorderOpacityMin = 30
+const nowSpeakingAvatarBorderOpacityMax = 100
+
 const llm = useLLM()
 const { audioContext, calculateVolume } = useAudioContext()
 const { process } = useMarkdown()
@@ -25,15 +28,23 @@ const openAiApiKey = useLocalStorage('openai-api-key', '')
 const openAiApiBaseURL = useLocalStorage('openai-api-base-url', 'https://api.openai.com/v1')
 const openAIModel = useLocalStorage('openai-model', '')
 
-const audioWaveformRef = ref<{ analyser: () => AnalyserNode }>()
-
-const mouthOpenSize = ref(0)
 const supportedModels = ref<OpenAI.Model[]>([])
 const messageInput = ref<string>('')
 const messages = ref<Message[]>([])
+const audioAnalyser = ref<AnalyserNode>()
+
+const mouthOpenSize = ref(0)
 const nowSpeaking = ref(false)
 const lipSyncStarted = ref(false)
 
+const nowSpeakingAvatarBorderOpacity = computed<number>(() => {
+  if (!nowSpeaking.value)
+    return nowSpeakingAvatarBorderOpacityMin
+
+  return ((nowSpeakingAvatarBorderOpacityMin
+    + (nowSpeakingAvatarBorderOpacityMax - nowSpeakingAvatarBorderOpacityMin) * mouthOpenSize.value) / 100)
+})
+
 const model = computed<string>({
   get: () => {
     if (!openAIModel.value)
@@ -65,7 +76,7 @@ const audioQueue = useQueue<{ audioBuffer: AudioBuffer, text: string }>({
         // Connect the source to the AudioContext's destination (the speakers)
         source.connect(audioContext.destination)
         // Connect the source to the analyzer
-        source.connect(audioWaveformRef.value!.analyser())
+        source.connect(audioAnalyser.value!)
 
         // Start playing the audio
         nowSpeaking.value = true
@@ -134,7 +145,7 @@ function getVolumeWithMinMaxNormalizeWithFrameUpdates() {
   if (!nowSpeaking.value)
     return
 
-  mouthOpenSize.value = calculateVolume(audioWaveformRef.value!.analyser(), 'minmax')
+  mouthOpenSize.value = calculateVolume(audioAnalyser.value!, 'linear')
 }
 
 function setupLipSync() {
@@ -145,11 +156,17 @@ function setupLipSync() {
   }
 }
 
+function setupAnalyser() {
+  if (!audioAnalyser.value)
+    audioAnalyser.value = audioContext.createAnalyser()
+}
+
 function onSendMessage(sendingMessage: string) {
   if (!sendingMessage)
     return
 
   setupLipSync()
+  setupAnalyser()
 
   const message: Message = { role: 'assistant', content: '' }
   messages.value.push({ role: 'user', content: sendingMessage })
@@ -217,16 +234,23 @@ onUnmounted(() => {
     <div flex="~ row 1" w-full items-end space-x-2>
       <div w-full>
         <Live2DViewer :mouth-open-size="mouthOpenSize" />
-        <div>
+        <!-- <div>
           <input v-model.number="mouthOpenSize" type="range" max="1" min="0" step="0.01">
           <span>{{ mouthOpenSize }}</span>
-        </div>
-        <AudioWaveform ref="audioWaveformRef" />
+        </div> -->
+        <!-- <AudioWaveform ref="audioWaveformRef" /> -->
       </div>
       <div my="2" w-full space-y-2>
         <div v-for="(message, index) in messages" :key="index">
           <div v-if="message.role === 'assistant'" flex mr="12">
-            <div h-10 min-h-10 min-w-10 w-10 overflow-hidden rounded-full border="pink solid 3" mr="2">
+            <div
+              mr-2 h-10 min-h-10 min-w-10 w-10 overflow-hidden rounded-full
+              border="solid 3"
+              transition="all ease-in-out" duration-100
+              :style="{
+                borderColor: `rgba(236, 72, 153, ${nowSpeakingAvatarBorderOpacity.toFixed(2)})`,
+              }"
+            >
               <img :src="Avatar">
             </div>
             <div flex="~ col" bg="pink-50/50 dark:pink-900/50" p="2" border="2 solid pink/10" rounded-lg>

pages/audio.vue

@@ -0,0 +1,45 @@
+<script setup lang="ts">
+import { ref } from 'vue'
+
+const containerRef = ref<HTMLDivElement>()
+const fileInputRef = ref<HTMLInputElement>()
+
+function handleFileUpload(e: Event) {
+  if (!e)
+    return
+
+  const file = fileInputRef.value?.files?.[0]
+  if (!file)
+    return
+
+  const audioElem = document.createElement('audio')
+  containerRef.value?.appendChild(audioElem)
+
+  audioElem.src = URL.createObjectURL(file)
+  audioElem.controls = true
+  audioElem.load()
+  audioElem.play()
+}
+</script>
+
+<template>
+  <div>
+    <Suspense>
+      <ClientOnly>
+        <div>
+          <div ref="containerRef" />
+          <input
+            ref="fileInputRef"
+            type="file"
+            @change="handleFileUpload"
+          >
+        </div>
+      </ClientOnly>
+      <template #fallback>
+        <div italic op50>
+          <span animate-pulse>Loading...</span>
+        </div>
+      </template>
+    </Suspense>
+  </div>
+</template>

stores/audio.ts

@@ -15,6 +15,8 @@ function calculateVolumeWithLinearNormalize(analyser: AnalyserNode) {
     .map(v => v ** 1.2)
     .reduce((acc, cur) => acc + cur, 0)
 
+  console.log('volumeSum linear', volumeSum)
+
   return (volumeSum / dataBuffer.length / 100)
 }
 
@@ -29,7 +31,7 @@ function calculateVolumeWithMinMaxNormalize(analyser: AnalyserNode) {
   // The volume changes are so flatten, and the volume is so low, so we need to amplify it
   // We can apply a power function to amplify the volume, for example
   // v ** 1.2 will amplify the volume by 1.2 times
-  const amplifiedVolumeVector = dataBuffer.map(v => v ** 1.2)
+  const amplifiedVolumeVector = dataBuffer.map(v => v ** 1.5)
 
   // Normalize the amplified values using Min-Max scaling
   const min = Math.min(...amplifiedVolumeVector)
@@ -47,6 +49,7 @@ function calculateVolumeWithMinMaxNormalize(analyser: AnalyserNode) {
 
   // Aggregate the volume values
   const volumeSum = normalizedVolumeVector.reduce((acc, cur) => acc + cur, 0)
+  console.log('volumeSum minmax', volumeSum)
 
   // Average the volume values
   return volumeSum / dataBuffer.length