diff --git a/components/MainStage.vue b/components/MainStage.vue index 0209c6eea..e241dd551 100644 --- a/components/MainStage.vue +++ b/components/MainStage.vue @@ -1,19 +1,16 @@ @@ -301,7 +223,7 @@ onUnmounted(() => {
Neuro
-
+
@@ -309,10 +231,10 @@ onUnmounted(() => {
-
+
You
-
+
@@ -329,12 +251,12 @@ onUnmounted(() => { - { + return instance + .processSync(markdown) + .toString() + }, + } +} diff --git a/cspell.config.yaml b/cspell.config.yaml index 8e4c17304..14b683623 100644 --- a/cspell.config.yaml +++ b/cspell.config.yaml @@ -4,10 +4,13 @@ dictionaryDefinitions: [] dictionaries: [] words: - composables + - elevenlabs - hiyori + - Myriam - Neuro - ofetch - openai + - pinia - pixi - rehype - vueuse diff --git a/server/api/v1/llm/voice/text-to-speech.ts b/server/api/v1/llm/voice/text-to-speech.ts index 95512b8c2..90380d06f 100644 --- a/server/api/v1/llm/voice/text-to-speech.ts +++ b/server/api/v1/llm/voice/text-to-speech.ts @@ -1,4 +1,4 @@ -import { ElevenLabsClient, stream } from 'elevenlabs' +import { ElevenLabsClient } from 'elevenlabs' export default defineEventHandler(async (event) => { const body = await readBody<{ text: string }>(event) diff --git a/stores/audio.ts b/stores/audio.ts index 833efa7ea..c3d842a82 100644 --- a/stores/audio.ts +++ b/stores/audio.ts @@ -1,9 +1,71 @@ import { defineStore } from 'pinia' +function calculateVolumeWithLinearNormalize(analyser: AnalyserNode) { + const dataBuffer = new Uint8Array(analyser.frequencyBinCount) + analyser.getByteFrequencyData(dataBuffer) + + const volumeVector = [] + for (let i = 0; i < 700; i += 80) + volumeVector.push(dataBuffer[i]) + + const volumeSum = dataBuffer + // The volume changes are so flatten, and the volume is so low, so we need to amplify it + // We can apply a power function to amplify the volume, for example + // v ** 1.2 will amplify the volume by 1.2 times + .map(v => v ** 1.2) + .reduce((acc, cur) => acc + cur, 0) + + return (volumeSum / dataBuffer.length / 100) +} + +function calculateVolumeWithMinMaxNormalize(analyser: AnalyserNode) { + const dataBuffer = new Uint8Array(analyser.frequencyBinCount) + analyser.getByteFrequencyData(dataBuffer) + + const volumeVector = [] + for (let i = 0; i < 700; i += 80) + volumeVector.push(dataBuffer[i]) + + // The volume changes are so flatten, and the volume is so low, so we need to amplify it + // We can apply a power function to amplify the volume, for example + // v ** 1.2 will amplify the volume by 1.2 times + const amplifiedVolumeVector = dataBuffer.map(v => v ** 1.2) + + // Normalize the amplified values using Min-Max scaling + const min = Math.min(...amplifiedVolumeVector) + const max = Math.max(...amplifiedVolumeVector) + const range = max - min + + let normalizedVolumeVector + if (range === 0) { + // If range is zero, all values are the same, so normalization is not needed + normalizedVolumeVector = amplifiedVolumeVector.map(() => 0) // or any default value + } + else { + normalizedVolumeVector = amplifiedVolumeVector.map(v => (v - min) / range) + } + + // Aggregate the volume values + const volumeSum = normalizedVolumeVector.reduce((acc, cur) => acc + cur, 0) + + // Average the volume values + return volumeSum / dataBuffer.length +} + +function calculateVolume(analyser: AnalyserNode, mode: 'linear' | 'minmax' = 'linear') { + switch (mode) { + case 'linear': + return calculateVolumeWithLinearNormalize(analyser) + case 'minmax': + return calculateVolumeWithMinMaxNormalize(analyser) + } +} + export const useAudioContext = defineStore('AudioContext', () => { const audioContext = new AudioContext() return { audioContext, + calculateVolume, } }) diff --git a/stores/llm.ts b/stores/llm.ts index 2bbe406e1..26ac92494 100644 --- a/stores/llm.ts +++ b/stores/llm.ts @@ -3,6 +3,7 @@ import { streamText } from 'ai' import { type OpenAIProvider, type OpenAIProviderSettings, createOpenAI } from '@ai-sdk/openai' import { OpenAI } from 'openai' import { ref } from 'vue' +import { ofetch } from 'ofetch' export const useLLM = defineStore('llm', () => { const openAI = ref() @@ -38,10 +39,22 @@ export const useLLM = defineStore('llm', () => { return await openAI.value.models.list() } + async function streamSpeech(text: string) { + return await ofetch('/api/v1/llm/voice/text-to-speech', { + body: { + text, + }, + method: 'POST', + cache: 'no-cache', + responseType: 'arrayBuffer', + }) + } + return { setupOpenAI, openAI, models, stream, + streamSpeech, } })