diff --git a/components/MainStage.vue b/components/MainStage.vue
index 0209c6eea..e241dd551 100644
--- a/components/MainStage.vue
+++ b/components/MainStage.vue
@@ -1,19 +1,16 @@
@@ -301,7 +223,7 @@ onUnmounted(() => {
Neuro
-
+
@@ -309,10 +231,10 @@ onUnmounted(() => {
@@ -329,12 +251,12 @@ onUnmounted(() => {
-
{
+ return instance
+ .processSync(markdown)
+ .toString()
+ },
+ }
+}
diff --git a/cspell.config.yaml b/cspell.config.yaml
index 8e4c17304..14b683623 100644
--- a/cspell.config.yaml
+++ b/cspell.config.yaml
@@ -4,10 +4,13 @@ dictionaryDefinitions: []
dictionaries: []
words:
- composables
+ - elevenlabs
- hiyori
+ - Myriam
- Neuro
- ofetch
- openai
+ - pinia
- pixi
- rehype
- vueuse
diff --git a/server/api/v1/llm/voice/text-to-speech.ts b/server/api/v1/llm/voice/text-to-speech.ts
index 95512b8c2..90380d06f 100644
--- a/server/api/v1/llm/voice/text-to-speech.ts
+++ b/server/api/v1/llm/voice/text-to-speech.ts
@@ -1,4 +1,4 @@
-import { ElevenLabsClient, stream } from 'elevenlabs'
+import { ElevenLabsClient } from 'elevenlabs'
export default defineEventHandler(async (event) => {
const body = await readBody<{ text: string }>(event)
diff --git a/stores/audio.ts b/stores/audio.ts
index 833efa7ea..c3d842a82 100644
--- a/stores/audio.ts
+++ b/stores/audio.ts
@@ -1,9 +1,71 @@
import { defineStore } from 'pinia'
+function calculateVolumeWithLinearNormalize(analyser: AnalyserNode) {
+ const dataBuffer = new Uint8Array(analyser.frequencyBinCount)
+ analyser.getByteFrequencyData(dataBuffer)
+
+ const volumeVector = []
+ for (let i = 0; i < 700; i += 80)
+ volumeVector.push(dataBuffer[i])
+
+ const volumeSum = dataBuffer
+ // The volume changes are so flatten, and the volume is so low, so we need to amplify it
+ // We can apply a power function to amplify the volume, for example
+ // v ** 1.2 will amplify the volume by 1.2 times
+ .map(v => v ** 1.2)
+ .reduce((acc, cur) => acc + cur, 0)
+
+ return (volumeSum / dataBuffer.length / 100)
+}
+
+function calculateVolumeWithMinMaxNormalize(analyser: AnalyserNode) {
+ const dataBuffer = new Uint8Array(analyser.frequencyBinCount)
+ analyser.getByteFrequencyData(dataBuffer)
+
+ const volumeVector = []
+ for (let i = 0; i < 700; i += 80)
+ volumeVector.push(dataBuffer[i])
+
+ // The volume changes are so flatten, and the volume is so low, so we need to amplify it
+ // We can apply a power function to amplify the volume, for example
+ // v ** 1.2 will amplify the volume by 1.2 times
+ const amplifiedVolumeVector = dataBuffer.map(v => v ** 1.2)
+
+ // Normalize the amplified values using Min-Max scaling
+ const min = Math.min(...amplifiedVolumeVector)
+ const max = Math.max(...amplifiedVolumeVector)
+ const range = max - min
+
+ let normalizedVolumeVector
+ if (range === 0) {
+ // If range is zero, all values are the same, so normalization is not needed
+ normalizedVolumeVector = amplifiedVolumeVector.map(() => 0) // or any default value
+ }
+ else {
+ normalizedVolumeVector = amplifiedVolumeVector.map(v => (v - min) / range)
+ }
+
+ // Aggregate the volume values
+ const volumeSum = normalizedVolumeVector.reduce((acc, cur) => acc + cur, 0)
+
+ // Average the volume values
+ return volumeSum / dataBuffer.length
+}
+
+function calculateVolume(analyser: AnalyserNode, mode: 'linear' | 'minmax' = 'linear') {
+ switch (mode) {
+ case 'linear':
+ return calculateVolumeWithLinearNormalize(analyser)
+ case 'minmax':
+ return calculateVolumeWithMinMaxNormalize(analyser)
+ }
+}
+
export const useAudioContext = defineStore('AudioContext', () => {
const audioContext = new AudioContext()
return {
audioContext,
+ calculateVolume,
}
})
diff --git a/stores/llm.ts b/stores/llm.ts
index 2bbe406e1..26ac92494 100644
--- a/stores/llm.ts
+++ b/stores/llm.ts
@@ -3,6 +3,7 @@ import { streamText } from 'ai'
import { type OpenAIProvider, type OpenAIProviderSettings, createOpenAI } from '@ai-sdk/openai'
import { OpenAI } from 'openai'
import { ref } from 'vue'
+import { ofetch } from 'ofetch'
export const useLLM = defineStore('llm', () => {
const openAI = ref()
@@ -38,10 +39,22 @@ export const useLLM = defineStore('llm', () => {
return await openAI.value.models.list()
}
+ async function streamSpeech(text: string) {
+ return await ofetch('/api/v1/llm/voice/text-to-speech', {
+ body: {
+ text,
+ },
+ method: 'POST',
+ cache: 'no-cache',
+ responseType: 'arrayBuffer',
+ })
+ }
+
return {
setupOpenAI,
openAI,
models,
stream,
+ streamSpeech,
}
})