diff --git a/components/MainStage.vue b/components/MainStage.vue index afcdd4879..4dc0e6f14 100644 --- a/components/MainStage.vue +++ b/components/MainStage.vue @@ -17,6 +17,9 @@ interface Message { content: string } +const nowSpeakingAvatarBorderOpacityMin = 30 +const nowSpeakingAvatarBorderOpacityMax = 100 + const llm = useLLM() const { audioContext, calculateVolume } = useAudioContext() const { process } = useMarkdown() @@ -25,15 +28,23 @@ const openAiApiKey = useLocalStorage('openai-api-key', '') const openAiApiBaseURL = useLocalStorage('openai-api-base-url', 'https://api.openai.com/v1') const openAIModel = useLocalStorage('openai-model', '') -const audioWaveformRef = ref<{ analyser: () => AnalyserNode }>() - -const mouthOpenSize = ref(0) const supportedModels = ref([]) const messageInput = ref('') const messages = ref([]) +const audioAnalyser = ref() + +const mouthOpenSize = ref(0) const nowSpeaking = ref(false) const lipSyncStarted = ref(false) +const nowSpeakingAvatarBorderOpacity = computed(() => { + if (!nowSpeaking.value) + return nowSpeakingAvatarBorderOpacityMin + + return ((nowSpeakingAvatarBorderOpacityMin + + (nowSpeakingAvatarBorderOpacityMax - nowSpeakingAvatarBorderOpacityMin) * mouthOpenSize.value) / 100) +}) + const model = computed({ get: () => { if (!openAIModel.value) @@ -65,7 +76,7 @@ const audioQueue = useQueue<{ audioBuffer: AudioBuffer, text: string }>({ // Connect the source to the AudioContext's destination (the speakers) source.connect(audioContext.destination) // Connect the source to the analyzer - source.connect(audioWaveformRef.value!.analyser()) + source.connect(audioAnalyser.value!) // Start playing the audio nowSpeaking.value = true @@ -134,7 +145,7 @@ function getVolumeWithMinMaxNormalizeWithFrameUpdates() { if (!nowSpeaking.value) return - mouthOpenSize.value = calculateVolume(audioWaveformRef.value!.analyser(), 'minmax') + mouthOpenSize.value = calculateVolume(audioAnalyser.value!, 'linear') } function setupLipSync() { @@ -145,11 +156,17 @@ function setupLipSync() { } } +function setupAnalyser() { + if (!audioAnalyser.value) + audioAnalyser.value = audioContext.createAnalyser() +} + function onSendMessage(sendingMessage: string) { if (!sendingMessage) return setupLipSync() + setupAnalyser() const message: Message = { role: 'assistant', content: '' } messages.value.push({ role: 'user', content: sendingMessage }) @@ -217,16 +234,23 @@ onUnmounted(() => {
-
+ +
-
+
diff --git a/pages/audio.vue b/pages/audio.vue new file mode 100644 index 000000000..6d5af8527 --- /dev/null +++ b/pages/audio.vue @@ -0,0 +1,45 @@ + + + diff --git a/stores/audio.ts b/stores/audio.ts index c3d842a82..be6246f04 100644 --- a/stores/audio.ts +++ b/stores/audio.ts @@ -15,6 +15,8 @@ function calculateVolumeWithLinearNormalize(analyser: AnalyserNode) { .map(v => v ** 1.2) .reduce((acc, cur) => acc + cur, 0) + console.log('volumeSum linear', volumeSum) + return (volumeSum / dataBuffer.length / 100) } @@ -29,7 +31,7 @@ function calculateVolumeWithMinMaxNormalize(analyser: AnalyserNode) { // The volume changes are so flatten, and the volume is so low, so we need to amplify it // We can apply a power function to amplify the volume, for example // v ** 1.2 will amplify the volume by 1.2 times - const amplifiedVolumeVector = dataBuffer.map(v => v ** 1.2) + const amplifiedVolumeVector = dataBuffer.map(v => v ** 1.5) // Normalize the amplified values using Min-Max scaling const min = Math.min(...amplifiedVolumeVector) @@ -47,6 +49,7 @@ function calculateVolumeWithMinMaxNormalize(analyser: AnalyserNode) { // Aggregate the volume values const volumeSum = normalizedVolumeVector.reduce((acc, cur) => acc + cur, 0) + console.log('volumeSum minmax', volumeSum) // Average the volume values return volumeSum / dataBuffer.length