refactor: cleanup code

Signed-off-by: Neko Ayaka <neko@ayaka.moe>

components/MainStage.vue

@@ -1,19 +1,16 @@
 <script setup lang="ts">
-import { computed, onMounted, ref, watch } from 'vue'
+import { computed, onMounted, ref, watch, watchEffect } from 'vue'
 import { useLocalStorage } from '@vueuse/core'
 import type { OpenAI } from 'openai'
-import { unified } from 'unified'
-import RemarkRehype from 'remark-rehype'
-import RemarkParse from 'remark-parse'
-import RehypeStringify from 'rehype-stringify'
-import { ofetch } from 'ofetch'
 
 import Avatar from '../assets/live2d/models/hiyori_free_zh/avatar.png'
-import Live2DViewer from '../components/Live2DViewer.vue'
-import BasicTextarea from '../components/BasicTextarea.vue'
 import { useLLM } from '../stores/llm'
 import { useQueue } from '../composables/queue'
+import { useMarkdown } from '../composables/markdown'
+
 import AudioWaveform from './AudioWaveform.vue'
+import Live2DViewer from './Live2DViewer.vue'
+import BasicTextarea from './BasicTextarea.vue'
 
 interface Message {
   role: 'system' | 'assistant' | 'user'
@@ -21,19 +18,21 @@ interface Message {
 }
 
 const llm = useLLM()
-const { audioContext } = useAudioContext()
+const { audioContext, calculateVolume } = useAudioContext()
+const { process } = useMarkdown()
 
 const openAiApiKey = useLocalStorage('openai-api-key', '')
 const openAiApiBaseURL = useLocalStorage('openai-api-base-url', 'https://api.openai.com/v1')
 const openAIModel = useLocalStorage('openai-model', '')
 
-const mouthOpenSize = ref(0)
-const models = ref<OpenAI.Model[]>([])
-const input = ref<string>('')
-const messages = ref<Message[]>([])
 const audioWaveformRef = ref<{ analyser: () => AnalyserNode }>()
-const speaking = ref(false)
-const speakingLipSyncStarted = ref(false)
+
+const mouthOpenSize = ref(0)
+const supportedModels = ref<OpenAI.Model[]>([])
+const messageInput = ref<string>('')
+const messages = ref<Message[]>([])
+const nowSpeaking = ref(false)
+const lipSyncStarted = ref(false)
 
 const model = computed<string>({
   get: () => {
@@ -43,7 +42,7 @@ const model = computed<string>({
     return (JSON.parse(openAIModel.value) as OpenAI.Model).id
   },
   set: (value) => {
-    const found = models.value.find(m => m.id === value)
+    const found = supportedModels.value.find(m => m.id === value)
     if (!found) {
       openAIModel.value = ''
       return
@@ -69,10 +68,10 @@ const audioQueue = useQueue<{ audioBuffer: AudioBuffer, text: string }>({
         source.connect(audioWaveformRef.value!.analyser())
 
         // Start playing the audio
-        speaking.value = true
+        nowSpeaking.value = true
         source.start(0)
         source.onended = () => {
-          speaking.value = false
+          nowSpeaking.value = false
           resolve()
         }
       })
@@ -83,7 +82,9 @@ const audioQueue = useQueue<{ audioBuffer: AudioBuffer, text: string }>({
 const ttsQueue = useQueue<string>({
   handlers: [
     async (ctx) => {
-      const audioBuffer = await streamSpeech(ctx.data)
+      const res = await llm.streamSpeech(ctx.data)
+      // Decode the ArrayBuffer into an AudioBuffer
+      const audioBuffer = await audioContext.decodeAudioData(res)
       audioQueue.add({ audioBuffer, text: ctx.data })
     },
   ],
@@ -128,115 +129,36 @@ const messageContentQueue = useQueue<string>({
   ],
 })
 
-async function streamSpeech(text: string) {
-  const res = await ofetch('/api/v1/llm/voice/text-to-speech', {
-    body: {
-      text,
-    },
-    method: 'POST',
-    cache: 'no-cache',
-    responseType: 'arrayBuffer',
-  })
-
-  // Decode the ArrayBuffer into an AudioBuffer
-  return await audioContext.decodeAudioData(res)
-}
-
-function getVolumeWithLinearNormalize() {
-  requestAnimationFrame(getVolumeWithLinearNormalize)
-  if (!speaking.value)
+function getVolumeWithMinMaxNormalizeWithFrameUpdates() {
+  requestAnimationFrame(getVolumeWithMinMaxNormalizeWithFrameUpdates)
+  if (!nowSpeaking.value)
     return
 
-  const analyser = audioWaveformRef.value!.analyser()
-  const dataBuffer = new Uint8Array(analyser.frequencyBinCount)
-  analyser.getByteFrequencyData(dataBuffer)
-
-  const volumeVector = []
-  for (let i = 0; i < 700; i += 80)
-    volumeVector.push(dataBuffer[i])
-
-  const volumeSum = dataBuffer
-    // The volume changes are so flatten, and the volume is so low, so we need to amplify it
-    // We can apply a power function to amplify the volume, for example
-    // v ** 1.2 will amplify the volume by 1.2 times
-    .map(v => v ** 1.2)
-    .reduce((acc, cur) => acc + cur, 0)
-
-  mouthOpenSize.value = (volumeSum / dataBuffer.length / 100)
-}
-
-function getVolumeWithMinMaxNormalize() {
-  requestAnimationFrame(getVolumeWithLinearNormalize)
-  if (!speaking.value)
-    return
-
-  const analyser = audioWaveformRef.value!.analyser()
-  const dataBuffer = new Uint8Array(analyser.frequencyBinCount)
-  analyser.getByteFrequencyData(dataBuffer)
-
-  const volumeVector = []
-  for (let i = 0; i < 700; i += 80)
-    volumeVector.push(dataBuffer[i])
-
-  // The volume changes are so flatten, and the volume is so low, so we need to amplify it
-  // We can apply a power function to amplify the volume, for example
-  // v ** 1.2 will amplify the volume by 1.2 times
-  const amplifiedVolumeVector = dataBuffer.map(v => v ** 1.2)
-
-  // Normalize the amplified values using Min-Max scaling
-  const min = Math.min(...amplifiedVolumeVector)
-  const max = Math.max(...amplifiedVolumeVector)
-  const range = max - min
-
-  let normalizedVolumeVector
-  if (range === 0) {
-    // If range is zero, all values are the same, so normalization is not needed
-    normalizedVolumeVector = amplifiedVolumeVector.map(() => 0) // or any default value
-  }
-  else {
-    normalizedVolumeVector = amplifiedVolumeVector.map(v => (v - min) / range)
-  }
-
-  // Aggregate the volume values
-  const volumeSum = normalizedVolumeVector.reduce((acc, cur) => acc + cur, 0)
-
-  // Average the volume values
-  mouthOpenSize.value = volumeSum / dataBuffer.length
+  mouthOpenSize.value = calculateVolume(audioWaveformRef.value!.analyser(), 'minmax')
 }
 
 function onSendMessage(sendingMessage: string) {
-  if (!speakingLipSyncStarted.value) {
-    getVolumeWithMinMaxNormalize()
+  if (!lipSyncStarted.value) {
+    getVolumeWithMinMaxNormalizeWithFrameUpdates()
     audioContext.resume()
-    speakingLipSyncStarted.value = true
+    lipSyncStarted.value = true
   }
 
   const message: Message = { role: 'assistant', content: '' }
   messages.value.push({ role: 'user', content: sendingMessage })
   messages.value.push(message)
   const index = messages.value.length - 1
-  const textParts: string[] = []
 
   llm.stream(model.value, sendingMessage).then(async (res) => {
     for await (const textPart of res.textStream) {
       messages.value[index].content += textPart
       messageContentQueue.add(textPart)
-      textParts.push(textPart)
     }
 
     messageContentQueue.add('|<llm_inference_end>|')
   })
 
-  input.value = ''
-}
-
-function fromMarkdownToHTML(markdown: string) {
-  return unified()
-    .use(RemarkParse)
-    .use(RemarkRehype)
-    .use(RehypeStringify)
-    .processSync(markdown)
-    .toString()
+  messageInput.value = ''
 }
 
 watch(openAiApiKey, (value) => {
@@ -256,11 +178,11 @@ onMounted(async () => {
   })
 
   const fetchedModels = await llm.models()
-  models.value = fetchedModels.data
+  supportedModels.value = fetchedModels.data
 })
 
 onUnmounted(() => {
-  speakingLipSyncStarted.value = false
+  lipSyncStarted.value = false
 })
 </script>
 
@@ -301,7 +223,7 @@ onUnmounted(() => {
               <div>
                 <span font-semibold>Neuro</span>
               </div>
-              <div v-html="fromMarkdownToHTML(message.content)" />
+              <div v-html="process(message.content)" />
             </div>
           </div>
           <div v-else-if="message.role === 'user'" flex="~ row-reverse" ml="12">
@@ -309,10 +231,10 @@ onUnmounted(() => {
               <div i-carbon:user-avatar-filled text="purple" h-full w-full p="0" m="0" />
             </div>
             <div flex="~ col" bg="purple-50/50 dark:purple-900/50" p="2" border="2 solid pink/10" rounded-lg>
-              <div self-end>
+              <div>
                 <span font-semibold>You</span>
               </div>
-              <div v-html="fromMarkdownToHTML(message.content)" />
+              <div v-html="process(message.content)" />
             </div>
           </div>
         </div>
@@ -329,12 +251,12 @@ onUnmounted(() => {
           <option value="">
             Select a model
           </option>
-          <option v-for="m in models" :key="m.id" :value="m.id">
+          <option v-for="m in supportedModels" :key="m.id" :value="m.id">
             {{ 'name' in m ? `${m.name} (${m.id})` : m.id }}
           </option>
         </select>
         <BasicTextarea
-          v-model="input"
+          v-model="messageInput"
           placeholder="Message"
           p="2" bg="zinc-100 dark:zinc-800"
           w-full rounded-lg outline-none

composables/markdown.ts

@@ -0,0 +1,18 @@
+import { unified } from 'unified'
+import RemarkRehype from 'remark-rehype'
+import RemarkParse from 'remark-parse'
+import RehypeStringify from 'rehype-stringify'
+
+export function useMarkdown() {
+  const instance = unified()
+    .use(RemarkParse)
+    .use(RemarkRehype)
+    .use(RehypeStringify)
+  return {
+    process: (markdown: string): string => {
+      return instance
+        .processSync(markdown)
+        .toString()
+    },
+  }
+}

cspell.config.yaml

@@ -4,10 +4,13 @@ dictionaryDefinitions: []
 dictionaries: []
 words:
   - composables
+  - elevenlabs
   - hiyori
+  - Myriam
   - Neuro
   - ofetch
   - openai
+  - pinia
   - pixi
   - rehype
   - vueuse

server/api/v1/llm/voice/text-to-speech.ts

@@ -1,4 +1,4 @@
-import { ElevenLabsClient, stream } from 'elevenlabs'
+import { ElevenLabsClient } from 'elevenlabs'
 
 export default defineEventHandler(async (event) => {
   const body = await readBody<{ text: string }>(event)

stores/audio.ts

@@ -1,9 +1,71 @@
 import { defineStore } from 'pinia'
 
+function calculateVolumeWithLinearNormalize(analyser: AnalyserNode) {
+  const dataBuffer = new Uint8Array(analyser.frequencyBinCount)
+  analyser.getByteFrequencyData(dataBuffer)
+
+  const volumeVector = []
+  for (let i = 0; i < 700; i += 80)
+    volumeVector.push(dataBuffer[i])
+
+  const volumeSum = dataBuffer
+    // The volume changes are so flatten, and the volume is so low, so we need to amplify it
+    // We can apply a power function to amplify the volume, for example
+    // v ** 1.2 will amplify the volume by 1.2 times
+    .map(v => v ** 1.2)
+    .reduce((acc, cur) => acc + cur, 0)
+
+  return (volumeSum / dataBuffer.length / 100)
+}
+
+function calculateVolumeWithMinMaxNormalize(analyser: AnalyserNode) {
+  const dataBuffer = new Uint8Array(analyser.frequencyBinCount)
+  analyser.getByteFrequencyData(dataBuffer)
+
+  const volumeVector = []
+  for (let i = 0; i < 700; i += 80)
+    volumeVector.push(dataBuffer[i])
+
+  // The volume changes are so flatten, and the volume is so low, so we need to amplify it
+  // We can apply a power function to amplify the volume, for example
+  // v ** 1.2 will amplify the volume by 1.2 times
+  const amplifiedVolumeVector = dataBuffer.map(v => v ** 1.2)
+
+  // Normalize the amplified values using Min-Max scaling
+  const min = Math.min(...amplifiedVolumeVector)
+  const max = Math.max(...amplifiedVolumeVector)
+  const range = max - min
+
+  let normalizedVolumeVector
+  if (range === 0) {
+    // If range is zero, all values are the same, so normalization is not needed
+    normalizedVolumeVector = amplifiedVolumeVector.map(() => 0) // or any default value
+  }
+  else {
+    normalizedVolumeVector = amplifiedVolumeVector.map(v => (v - min) / range)
+  }
+
+  // Aggregate the volume values
+  const volumeSum = normalizedVolumeVector.reduce((acc, cur) => acc + cur, 0)
+
+  // Average the volume values
+  return volumeSum / dataBuffer.length
+}
+
+function calculateVolume(analyser: AnalyserNode, mode: 'linear' | 'minmax' = 'linear') {
+  switch (mode) {
+    case 'linear':
+      return calculateVolumeWithLinearNormalize(analyser)
+    case 'minmax':
+      return calculateVolumeWithMinMaxNormalize(analyser)
+  }
+}
+
 export const useAudioContext = defineStore('AudioContext', () => {
   const audioContext = new AudioContext()
 
   return {
     audioContext,
+    calculateVolume,
   }
 })

stores/llm.ts

@@ -3,6 +3,7 @@ import { streamText } from 'ai'
 import { type OpenAIProvider, type OpenAIProviderSettings, createOpenAI } from '@ai-sdk/openai'
 import { OpenAI } from 'openai'
 import { ref } from 'vue'
+import { ofetch } from 'ofetch'
 
 export const useLLM = defineStore('llm', () => {
   const openAI = ref<OpenAI>()
@@ -38,10 +39,22 @@ export const useLLM = defineStore('llm', () => {
     return await openAI.value.models.list()
   }
 
+  async function streamSpeech(text: string) {
+    return await ofetch('/api/v1/llm/voice/text-to-speech', {
+      body: {
+        text,
+      },
+      method: 'POST',
+      cache: 'no-cache',
+      responseType: 'arrayBuffer',
+    })
+  }
+
   return {
     setupOpenAI,
     openAI,
     models,
     stream,
+    streamSpeech,
   }
 })