diff --git a/apps/stage-tamagotchi/package.json b/apps/stage-tamagotchi/package.json index c1e1edffe..c6dd5555f 100644 --- a/apps/stage-tamagotchi/package.json +++ b/apps/stage-tamagotchi/package.json @@ -35,6 +35,7 @@ "@tauri-apps/plugin-window-state": "^2.3.0", "@tresjs/cientos": "^4.3.1", "@tresjs/core": "^4.3.6", + "@types/audioworklet": "^0.0.77", "@vueuse/core": "^13.4.0", "@vueuse/shared": "^13.4.0", "@xsai-ext/providers-cloud": "catalog:", diff --git a/apps/stage-tamagotchi/src-tauri/src/app/models/mod.rs b/apps/stage-tamagotchi/src-tauri/src/app/models/mod.rs index 0ecac62d5..550ebc88e 100644 --- a/apps/stage-tamagotchi/src-tauri/src/app/models/mod.rs +++ b/apps/stage-tamagotchi/src-tauri/src/app/models/mod.rs @@ -1,7 +1,6 @@ pub mod silero_vad; pub mod whisper; -use candle_core::Device; use log::info; use tauri::Runtime; @@ -29,5 +28,5 @@ pub fn new_silero_vad_processor( window: tauri::WebviewWindow ) -> anyhow::Result { info!("Loading VAD model"); - silero_vad::Processor::new(Device::Cpu, window) + silero_vad::Processor::new(window) } diff --git a/apps/stage-tamagotchi/src-tauri/src/app/models/silero_vad/mod.rs b/apps/stage-tamagotchi/src-tauri/src/app/models/silero_vad/mod.rs index 4d09d9406..32e198a51 100644 --- a/apps/stage-tamagotchi/src-tauri/src/app/models/silero_vad/mod.rs +++ b/apps/stage-tamagotchi/src-tauri/src/app/models/silero_vad/mod.rs @@ -1,9 +1,8 @@ -use std::path::PathBuf; +use std::{path::PathBuf, sync::Arc}; use anyhow::Result; use hf_hub::Repo; use log::info; -use ndarray::{Array2, Array3}; use ort::{ execution_providers::{ CPUExecutionProvider, @@ -12,28 +11,33 @@ use ort::{ DirectMLExecutionProvider, }, session::{Session, builder::GraphOptimizationLevel}, + util::Mutex, value::Tensor, }; +use serde::{Deserialize, Serialize}; use tauri::Runtime; use crate::helpers::huggingface::create_progress_emitter; -/// Main Silero VAD model wrapper with hardware acceleration support +#[derive(Serialize, Deserialize, Clone)] +pub struct VADInferenceResult { + pub output: Vec, // Speech probability output + pub state: Vec, // Updated state for next inference +} + +#[derive(Serialize, Deserialize, Clone)] +pub struct VADInferenceInput { + pub input: Vec, // Audio input buffer + pub sr: i64, // Sample rate + pub state: Vec, // Current state +} + pub struct Processor { - session: Session, - context: Array2, - state: ndarray::Array3, - last_batch_size: usize, - frame_size: usize, - context_size: usize, - sample_rate: i64, + session: Arc>, } impl Processor { - pub fn new( - _device: candle_core::Device, - window: tauri::WebviewWindow, - ) -> Result { + pub fn new(window: tauri::WebviewWindow) -> Result { let model_id = "onnx-community/silero-vad"; let revision = "main"; @@ -61,16 +65,9 @@ impl Processor { }; let session = Self::create_optimized_session(model_path.clone())?; - let (frame_size, context_size) = (512, 64); Ok(Self { - session, - context: Array2::zeros((1, context_size)), - state: Array3::zeros((2, 1, 128)), - last_batch_size: 0, - frame_size, - context_size, - sample_rate: 16000, + session: Arc::new(Mutex::new(session)), }) } @@ -97,87 +94,52 @@ impl Processor { Ok(session) } - /// Reset the model's internal state - fn reset_states( - &mut self, - batch_size: usize, - ) { - self.context = Array2::zeros((batch_size, self.context_size)); - self.state = Array3::zeros((2, batch_size, 128)); - } - - /// Validate input audio chunk - fn validate_input( + /// Stateless inference that matches JavaScript interface + /// Returns both output and updated state like the JS version + pub fn inference( &self, - x: &[f32], - ) -> Result<()> { - if x.len() != self.frame_size { + input_data: VADInferenceInput, + ) -> Result { + // Validate input dimensions + if input_data.state.len() != 2 * 1 * 128 { return Err(anyhow::anyhow!( - "Input chunk must be {} samples, got {}", - self.frame_size, - x.len() + "State must have 256 elements (2*1*128), got {}", + input_data.state.len() )); } - Ok(()) - } - - /// Process a single audio chunk and return speech probability - /// This is the main API used by hearing.vue via the audio_vad plugin function - pub fn process_chunk( - &mut self, - chunk: &[f32], - ) -> Result { - self.validate_input(chunk)?; - - let batch_size = 1; - if self.last_batch_size != batch_size { - self.reset_states(batch_size); - } - - // Prepare input tensor by concatenating context and new audio data - let mut input_data = Vec::with_capacity(self.context_size + chunk.len()); - input_data.extend_from_slice(self.context.row(0).as_slice().unwrap()); - input_data.extend_from_slice(chunk); - - let input_shape = vec![batch_size, input_data.len()]; // Create input tensors for the ONNX model - // Silero VAD requires audio input, sample rate, and state let inputs = vec![ ( "input", - Tensor::from_array((input_shape.clone(), input_data.clone()))?.into_dyn(), + Tensor::from_array((vec![1, input_data.input.len()], input_data.input.clone()))?.into_dyn(), ), ( "sr", - Tensor::from_array(([1], vec![self.sample_rate]))?.into_dyn(), + Tensor::from_array(([1], vec![input_data.sr]))?.into_dyn(), ), ( "state", - Tensor::from_array((vec![2, 1, 128], self.state.as_slice().unwrap().to_vec()))?.into_dyn(), + Tensor::from_array((vec![2, 1, 128], input_data.state.clone()))?.into_dyn(), ), ]; - // Run inference - let outputs = self.session.run(inputs)?; + // Run inference and extract data while session is still locked + let (state_data, speech_data) = { + let mut session = self.session.lock(); + let outputs = session.run(inputs)?; - // Update context from the last portion of the input - let context_start = input_data.len() - self.context_size; - let new_context_data = input_data[context_start..].to_vec(); - self.context = Array2::from_shape_vec((batch_size, self.context_size), new_context_data) - .map_err(|e| anyhow::anyhow!("Failed to update context: {}", e))?; + // Extract and clone the data immediately while session is locked + let (_state_shape, state_slice) = outputs[1].try_extract_tensor::()?; + let (_speech_shape, speech_slice) = outputs[0].try_extract_tensor::()?; - // Update state from model output - let (_state_shape, state_data) = outputs[1].try_extract_tensor::()?; - self.state = Array3::from_shape_vec((2, 1, 128), state_data.to_vec()) - .map_err(|e| anyhow::anyhow!("Failed to update state: {}", e))?; + // Clone the data to owned vectors before the session lock is released + (state_slice.to_vec(), speech_slice.to_vec()) + }; - self.last_batch_size = batch_size; - - // Extract speech probability - let (_shape, data) = outputs[0].try_extract_tensor::()?; - let speech_prob = data[0]; - - Ok(speech_prob) + Ok(VADInferenceResult { + output: speech_data, + state: state_data, + }) } } diff --git a/apps/stage-tamagotchi/src-tauri/src/plugins/audio_vad/mod.rs b/apps/stage-tamagotchi/src-tauri/src/plugins/audio_vad/mod.rs index fbcf39eed..0ac90afad 100644 --- a/apps/stage-tamagotchi/src-tauri/src/plugins/audio_vad/mod.rs +++ b/apps/stage-tamagotchi/src-tauri/src/plugins/audio_vad/mod.rs @@ -7,7 +7,10 @@ use tauri::{ plugin::{Builder as PluginBuilder, TauriPlugin}, }; -use crate::app::models::new_silero_vad_processor; +use crate::app::models::{ + new_silero_vad_processor, + silero_vad::{VADInferenceInput, VADInferenceResult}, +}; #[derive(Default)] struct AppDataSileroVadProcessor { @@ -51,27 +54,18 @@ pub async fn load_model_silero_vad( #[tauri::command] pub async fn audio_vad( app: tauri::AppHandle, - chunk: Vec, -) -> Result { + input_data: VADInferenceInput, +) -> Result { let data = app.state::>(); + let data = data.lock().unwrap(); - // Check if processor exists first - { - let data = data.lock().unwrap(); - if data.silero_vad_processor.is_none() { - return Err("Silero VAD model is not loaded".to_string()); - } + if let Some(processor) = &data.silero_vad_processor { + processor + .inference(input_data) + .map_err(|e| e.to_string()) + } else { + Err("Silero VAD model is not loaded".to_string()) } - - // Then mutable borrow - let mut data = data.lock().unwrap(); - let processor = data.silero_vad_processor.as_mut().unwrap(); - - let speech_prob = processor - .process_chunk(chunk.as_slice()) - .map_err(|e| e.to_string())?; - - Ok(speech_prob) } pub fn init() -> TauriPlugin { diff --git a/apps/stage-tamagotchi/src/composables/audio/analysis-vad.ts b/apps/stage-tamagotchi/src/composables/audio/analysis-vad.ts deleted file mode 100644 index eaffaf850..000000000 --- a/apps/stage-tamagotchi/src/composables/audio/analysis-vad.ts +++ /dev/null @@ -1,397 +0,0 @@ -import type { AudioChunkCallback } from '@proj-airi/audio/vue' - -import { onUnmounted, readonly, ref, watch } from 'vue' - -import { useTauriCore } from '../tauri' - -export interface VADSegment { - id: string - audioData: Float32Array - startTime: number - endTime: number - probability: number - isComplete: boolean -} - -export interface VADConfig { - threshold: number - silenceGapMs: number // How long silence before ending segment - minSpeechDurationMs: number // Minimum speech duration to be valid - maxSpeechDurationMs: number // Maximum speech duration before forced split - overlapMs: number // Overlap between segments for better transcription - bufferSizeMs: number // How much audio to keep in memory -} - -export function useVADAnalysis(config: Partial = {}) { - const { invoke } = useTauriCore() - - const isModelLoaded = ref(false) - const isLoading = ref(false) - const error = ref('') - const isEnabled = ref(true) - - const probability = ref(0) - const history = ref([]) - const maxHistory = 50 - - // Configuration with defaults - const vadConfig = ref({ - threshold: 0.5, - silenceGapMs: 500, // 500ms of silence to end segment - minSpeechDurationMs: 300, // Minimum 300ms speech - maxSpeechDurationMs: 30000, // Max 30s per segment - overlapMs: 200, // 200ms overlap - bufferSizeMs: 60000, // Keep 60s of audio in memory - ...config, - }) - - // Audio buffering - const audioBuffer = ref(new Float32Array(0)) - const chunkSize = 512 - const targetSampleRate = 16000 - let processingInterval: number | null = null - - // Segment management - const currentSegment = ref(null) - const completedSegments = ref([]) - const maxCompletedSegments = 10 // Keep last 10 completed segments - - // State tracking - const isSpeaking = ref(false) - const lastSpeechTime = ref(0) - const lastSilenceTime = ref(0) - const segmentStartTime = ref(0) - const segmentAudioBuffer = ref(new Float32Array(0)) - - // Callbacks for segment events - const segmentCallbacks = new Set<(segment: VADSegment) => void>() - - async function loadModel() { - if (isModelLoaded.value || isLoading.value) - return - - isLoading.value = true - error.value = '' - - try { - await invoke('plugin:proj-airi-tauri-plugin-audio-vad|load_model_silero_vad') - isModelLoaded.value = true - } - catch (err) { - error.value = err instanceof Error ? err.message : String(err) - console.error('Failed to load VAD model:', err) - } - finally { - isLoading.value = false - } - } - - function resampleIfNeeded(chunk: Float32Array, sourceSampleRate: number): Float32Array { - if (sourceSampleRate === targetSampleRate) { - return chunk - } - - const ratio = targetSampleRate / sourceSampleRate - const outputLength = Math.floor(chunk.length * ratio) - const resampled = new Float32Array(outputLength) - - for (let i = 0; i < outputLength; i++) { - const sourceIndex = i / ratio - const index = Math.floor(sourceIndex) - const fraction = sourceIndex - index - - if (index + 1 < chunk.length) { - resampled[i] = chunk[index] * (1 - fraction) + chunk[index + 1] * fraction - } - else { - resampled[i] = chunk[index] || 0 - } - } - - return resampled - } - - async function processChunk(chunk: Float32Array) { - if (!isModelLoaded.value || chunk.length !== chunkSize) - return - - try { - const chunkArray = Array.from(chunk) - const prob = await invoke('plugin:proj-airi-tauri-plugin-audio-vad|audio_vad', { - chunk: chunkArray, - }) - - if (typeof prob === 'number') { - const now = performance.now() - probability.value = prob - - // Update history - history.value.push(prob) - if (history.value.length > maxHistory) { - history.value.shift() - } - - // Process speech detection with hysteresis - const wasSpeaking = isSpeaking.value - const currentlySpeaking = prob > vadConfig.value.threshold - - if (currentlySpeaking) { - lastSpeechTime.value = now - } - else { - lastSilenceTime.value = now - } - - // Handle speech state transitions - if (!wasSpeaking && currentlySpeaking) { - // Start of speech - startSpeechSegment(now, chunk) - isSpeaking.value = true - } - else if (wasSpeaking && !currentlySpeaking) { - // Potential end of speech - wait for silence gap - const silenceDuration = now - lastSpeechTime.value - if (silenceDuration >= vadConfig.value.silenceGapMs) { - await endSpeechSegment(now) - isSpeaking.value = false - } - } - else if (wasSpeaking && currentlySpeaking) { - // Continuing speech - add to current segment - addToCurrentSegment(chunk) - - // Check for max duration - const segmentDuration = now - segmentStartTime.value - if (segmentDuration >= vadConfig.value.maxSpeechDurationMs) { - await endSpeechSegment(now, true) // Force end - startSpeechSegment(now, chunk) // Start new segment - } - } - - // Always add audio to segment buffer when speaking or recently speaking - const timeSinceLastSpeech = now - lastSpeechTime.value - if (timeSinceLastSpeech <= vadConfig.value.silenceGapMs + vadConfig.value.overlapMs) { - addToCurrentSegment(chunk) - } - } - } - catch (err) { - error.value = err instanceof Error ? err.message : String(err) - console.error('VAD processing error:', err) - } - } - - function startSpeechSegment(timestamp: number, initialChunk: Float32Array) { - segmentStartTime.value = timestamp - - // Include some pre-speech audio for context (overlap) - const overlapSamples = Math.floor((vadConfig.value.overlapMs / 1000) * targetSampleRate) - const totalBufferLength = audioBuffer.value.length - const startIndex = Math.max(0, totalBufferLength - overlapSamples) - - const preAudio = audioBuffer.value.slice(startIndex) - segmentAudioBuffer.value = new Float32Array(preAudio.length + initialChunk.length) - segmentAudioBuffer.value.set(preAudio, 0) - segmentAudioBuffer.value.set(initialChunk, preAudio.length) - - currentSegment.value = { - id: `segment_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`, - audioData: new Float32Array(0), // Will be set when segment ends - startTime: timestamp, - endTime: 0, - probability: probability.value, - isComplete: false, - } - } - - function addToCurrentSegment(chunk: Float32Array) { - if (!currentSegment.value) - return - - // Append to segment buffer - const oldBuffer = segmentAudioBuffer.value - const newBuffer = new Float32Array(oldBuffer.length + chunk.length) - newBuffer.set(oldBuffer, 0) - newBuffer.set(chunk, oldBuffer.length) - segmentAudioBuffer.value = newBuffer - } - - async function endSpeechSegment(timestamp: number, forced = false) { - if (!currentSegment.value) - return - - const segmentDuration = timestamp - segmentStartTime.value - - // Check minimum duration requirement - if (!forced && segmentDuration < vadConfig.value.minSpeechDurationMs) { - // Too short, discard - currentSegment.value = null - segmentAudioBuffer.value = new Float32Array(0) - return - } - - // Add some post-speech audio for context - const overlapSamples = Math.floor((vadConfig.value.overlapMs / 1000) * targetSampleRate) - const postAudioLength = Math.min(overlapSamples, audioBuffer.value.length) - const postAudio = audioBuffer.value.slice(-postAudioLength) - - // Final segment audio - const finalAudioBuffer = new Float32Array(segmentAudioBuffer.value.length + postAudio.length) - finalAudioBuffer.set(segmentAudioBuffer.value, 0) - finalAudioBuffer.set(postAudio, segmentAudioBuffer.value.length) - - // Complete the segment - const completedSegment: VADSegment = { - ...currentSegment.value, - audioData: finalAudioBuffer, - endTime: timestamp, - isComplete: true, - } - - // Add to completed segments - completedSegments.value.push(completedSegment) - if (completedSegments.value.length > maxCompletedSegments) { - completedSegments.value.shift() - } - - // Notify callbacks - segmentCallbacks.forEach((callback) => { - try { - callback(completedSegment) - } - catch (err) { - console.error('Segment callback error:', err) - } - }) - - // Reset for next segment - currentSegment.value = null - segmentAudioBuffer.value = new Float32Array(0) - } - - function startProcessing() { - if (processingInterval) - return - - const intervalMs = (chunkSize / targetSampleRate) * 1000 - - processingInterval = window.setInterval(async () => { - if (audioBuffer.value.length >= chunkSize) { - const chunk = audioBuffer.value.slice(0, chunkSize) - await processChunk(chunk) - - const remaining = audioBuffer.value.slice(chunkSize) - audioBuffer.value = remaining.length > 0 ? remaining : new Float32Array(0) - } - }, Math.max(1, intervalMs / 2)) - } - - function stopProcessing() { - if (processingInterval) { - clearInterval(processingInterval) - processingInterval = null - } - - // End current segment if active - if (currentSegment.value) { - endSpeechSegment(performance.now(), true) - } - - audioBuffer.value = new Float32Array(0) - probability.value = 0 - history.value = [] - } - - // Audio chunk callback - const audioChunkCallback: AudioChunkCallback = (chunk: Float32Array, sampleRate: number) => { - if (!isEnabled.value || !isModelLoaded.value) - return - - const processedChunk = resampleIfNeeded(chunk, sampleRate) - - // Add to main buffer - const currentBuffer = audioBuffer.value - const newBuffer = new Float32Array(currentBuffer.length + processedChunk.length) - newBuffer.set(currentBuffer, 0) - newBuffer.set(processedChunk, currentBuffer.length) - - // Trim buffer to max size - const maxBufferSamples = (vadConfig.value.bufferSizeMs / 1000) * targetSampleRate - if (newBuffer.length > maxBufferSamples) { - const trimAmount = newBuffer.length - maxBufferSamples - audioBuffer.value = newBuffer.slice(trimAmount) - } - else { - audioBuffer.value = newBuffer - } - } - - // Segment callback management - function onSegmentComplete(callback: (segment: VADSegment) => void) { - segmentCallbacks.add(callback) - return () => segmentCallbacks.delete(callback) - } - - // Manual segment control - async function forceEndCurrentSegment() { - if (currentSegment.value) { - await endSpeechSegment(performance.now(), true) - isSpeaking.value = false - } - } - - function clearCompletedSegments() { - completedSegments.value = [] - } - - // Get segment by ID - function getSegment(id: string): VADSegment | undefined { - return completedSegments.value.find(s => s.id === id) - } - - // Auto-start processing when enabled and model is loaded - watch([isEnabled, isModelLoaded], ([enabled, loaded]) => { - if (enabled && loaded) { - startProcessing() - } - else { - stopProcessing() - } - }) - - onUnmounted(() => { - stopProcessing() - }) - - return { - // State - isModelLoaded: readonly(isModelLoaded), - isLoading: readonly(isLoading), - error: readonly(error), - isEnabled, - - // VAD data - probability: readonly(probability), - history: readonly(history), - isSpeaking: readonly(isSpeaking), - - // Configuration - config: vadConfig, - - // Segments - currentSegment: readonly(currentSegment), - completedSegments: readonly(completedSegments), - - // Controls - loadModel, - forceEndCurrentSegment, - clearCompletedSegments, - getSegment, - - // Events - onSegmentComplete, - - // For audio stream integration - audioChunkCallback, - } -} diff --git a/apps/stage-tamagotchi/src/composables/audio/analysis-volume.ts b/apps/stage-tamagotchi/src/composables/audio/analysis-volume.ts deleted file mode 100644 index 3ac76cb9c..000000000 --- a/apps/stage-tamagotchi/src/composables/audio/analysis-volume.ts +++ /dev/null @@ -1,46 +0,0 @@ -import type { AudioAnalysisCallback } from '@proj-airi/audio/vue' - -import { computed, readonly, ref } from 'vue' - -export function useVolumeAnalysis() { - const level = ref(0) - const threshold = ref(25) - const history = ref([]) - const maxHistory = 100 - const isEnabled = ref(true) - - const isSpeaking = computed(() => level.value > threshold.value) - - const audioAnalysisCallback: AudioAnalysisCallback = (data) => { - if (!isEnabled.value) - return - - level.value = data.volumeLevel - - // Update history - history.value.push(data.volumeLevel) - if (history.value.length > maxHistory) { - history.value.shift() - } - } - - function reset() { - level.value = 0 - history.value = [] - } - - return { - // State - level: readonly(level), - threshold, - history: readonly(history), - isSpeaking, - isEnabled, - - // Controls - reset, - - // For audio stream integration - audioAnalysisCallback, - } -} diff --git a/apps/stage-tamagotchi/src/composables/audio/extract-whisper.ts b/apps/stage-tamagotchi/src/composables/audio/extract-whisper.ts index 12e068b1a..263b7f92f 100644 --- a/apps/stage-tamagotchi/src/composables/audio/extract-whisper.ts +++ b/apps/stage-tamagotchi/src/composables/audio/extract-whisper.ts @@ -1,5 +1,3 @@ -import type { VADSegment } from './analysis-vad' - import { computed, readonly, ref } from 'vue' import { useTauriCore } from '../tauri' @@ -13,6 +11,15 @@ export interface WhisperConfig { bestOf: number } +export interface VADSegment { + id: string + audioData: Float32Array + startTime: number + endTime: number + probability: number + isComplete: boolean +} + export interface TranscriptionResult { id: string segmentId: string diff --git a/apps/stage-tamagotchi/src/composables/audio/manager.ts b/apps/stage-tamagotchi/src/composables/audio/manager.ts deleted file mode 100644 index 71208ef58..000000000 --- a/apps/stage-tamagotchi/src/composables/audio/manager.ts +++ /dev/null @@ -1,171 +0,0 @@ -import type { AudioStreamConfig } from '@proj-airi/audio/vue' -import type { MaybeRefOrGetter } from 'vue' - -import { cleanupAudioContext, getAudioContext, getAudioContextState, isAudioContextReady } from '@proj-airi/audio/audio-context' -import { useAudioPlayback, useAudioStream } from '@proj-airi/audio/vue' -import { computed, readonly, shallowRef, toRef } from 'vue' - -import { useVADAnalysis } from './analysis-vad' -import { useVolumeAnalysis } from './analysis-volume' -import { useWhisperTranscription } from './extract-whisper' - -export interface AudioSource { - id: string - name: string - deviceId: string - type: 'microphone' | 'screen' | 'system' - config: AudioStreamConfig -} - -export function useAudioManager(locale: MaybeRefOrGetter = 'en') { - const audioContext = getAudioContext() - - const sources = shallowRef(new Map()) - const activeStreams = shallowRef(new Map>()) - const vadAnalyzers = shallowRef(new Map>()) - const volumeAnalyzers = shallowRef(new Map>()) - const playbackControllers = shallowRef(new Map>()) - const whisperTranscribers = shallowRef(new Map>()) - - const audioContextState = toRef(() => getAudioContextState()) - const localeRef = toRef(locale) - - function addSource(source: AudioSource) { - sources.value.set(source.id, source) - - // Create composables - const configRef = computed(() => sources.value.get(source.id)?.config) - const stream = useAudioStream(configRef) - const vadAnalyzer = useVADAnalysis() - const volumeAnalyzer = useVolumeAnalysis() - const whisperTranscriber = useWhisperTranscription({ modelSize: 'medium', temperature: 0.0 }) - - whisperTranscriber.loadModel() - - // Store the composables FIRST - activeStreams.value.set(source.id, stream) - vadAnalyzers.value.set(source.id, vadAnalyzer) - volumeAnalyzers.value.set(source.id, volumeAnalyzer) - whisperTranscribers.value.set(source.id, whisperTranscriber) - - // NOW create the playback with the stream reference - const streamRef = computed(() => activeStreams.value.get(source.id)?.mediaStream.value) - const playback = useAudioPlayback(streamRef) - playbackControllers.value.set(source.id, playback) - - // Connect analyzers - stream.addChunkCallback(vadAnalyzer.audioChunkCallback) - stream.addAnalysisCallback(volumeAnalyzer.audioAnalysisCallback) - - vadAnalyzer.onSegmentComplete((segment) => { - whisperTranscriber.queueSegment(segment, localeRef.value) - }) - - return source.id - } - - function removeSource(sourceId: string) { - const stream = activeStreams.value.get(sourceId) - if (stream) { - stream.stop() - activeStreams.value.delete(sourceId) - } - - sources.value.delete(sourceId) - vadAnalyzers.value.delete(sourceId) - volumeAnalyzers.value.delete(sourceId) - playbackControllers.value.delete(sourceId) - } - - function getSourceData(sourceId: string) { - return { - source: sources.value.get(sourceId), - stream: activeStreams.value.get(sourceId), - vad: vadAnalyzers.value.get(sourceId), - volume: volumeAnalyzers.value.get(sourceId), - playback: playbackControllers.value.get(sourceId), - whisper: whisperTranscribers.value.get(sourceId), - } - } - - async function startSource(sourceId: string) { - const stream = activeStreams.value.get(sourceId) - const vadAnalyzer = vadAnalyzers.value.get(sourceId) - - if (stream) { - await stream.start() - } - - if (vadAnalyzer) { - vadAnalyzer.loadModel() - } - } - - async function stopSource(sourceId: string) { - const stream = activeStreams.value.get(sourceId) - if (stream) { - await stream.stop() - } - } - - // Global audio context controls - async function suspendGlobalAudio() { - await audioContext?.suspend() - } - - async function resumeGlobalAudio() { - await audioContext?.resume() - } - - async function cleanupGlobalAudio() { - // Stop all sources first - for (const sourceId of Object.keys(sources.value)) { - await stopSource(sourceId) - } - - // Cleanup global context - await cleanupAudioContext() - } - - // Convenience methods - function addMicrophone(deviceId: string, name: string = 'Primary Microphone') { - return addSource({ - id: `mic-${deviceId}`, - name, - deviceId, - type: 'microphone', - config: { - deviceId, - sampleRate: 16000, - echoCancellation: true, - noiseSuppression: false, - autoGainControl: false, - }, - }) - } - - return { - // State - sources: readonly(sources.value), - audioContextState: { - isReady: isAudioContextReady, - sampleRate: audioContext?.sampleRate ?? 0, - error: audioContextState.value.error, - }, - - // Management - addSource, - removeSource, - startSource, - stopSource, - getSourceData, - - // Global controls - suspendGlobalAudio, - resumeGlobalAudio, - cleanupGlobalAudio, - - // Convenience - addMicrophone, - } -} diff --git a/apps/stage-tamagotchi/src/composables/tauri.ts b/apps/stage-tamagotchi/src/composables/tauri.ts index 5d91e37fa..591ded253 100644 --- a/apps/stage-tamagotchi/src/composables/tauri.ts +++ b/apps/stage-tamagotchi/src/composables/tauri.ts @@ -2,6 +2,8 @@ import type { InvokeArgs, InvokeOptions } from '@tauri-apps/api/core' import type { EventCallback, EventName, UnlistenFn } from '@tauri-apps/api/event' import type { Monitor } from '@tauri-apps/api/window' +import type { InvokeMethods, InvokeMethodShape } from '../tauri/invoke' + import { withRetry } from '@moeru/std' import { computedAsync, until } from '@vueuse/core' @@ -128,39 +130,6 @@ export function useTauriEvent() { } } -export interface InvokeMethods { - // app windows - 'open_settings_window': { args: undefined, options: undefined, returns: void } - 'open_chat_window': { args: undefined, options: undefined, returns: void } - - // Plugin - Audio Transcription - 'plugin:proj-airi-tauri-plugin-audio-transcription|load_model_whisper': { args: { modelType: 'base' | 'largev3' | 'tiny' | 'medium' }, options: undefined, returns: void } - 'plugin:proj-airi-tauri-plugin-audio-transcription|audio_transcription': { args: { chunk: number[], language: string }, options: undefined, returns: [string, string] } - - // Plugin - Audio VAD - 'plugin:proj-airi-tauri-plugin-audio-vad|load_model_silero_vad': { args: undefined, options: undefined, returns: void } - 'plugin:proj-airi-tauri-plugin-audio-vad|audio_vad': { args: { chunk: number[] }, options: undefined, returns: number } - - // Plugin - Window Pass through on hover - 'plugin:proj-airi-tauri-plugin-window-pass-through-on-hover|start_monitor': { args: undefined, options: undefined, returns: void } - 'plugin:proj-airi-tauri-plugin-window-pass-through-on-hover|stop_monitor': { args: undefined, options: undefined, returns: void } - 'plugin:proj-airi-tauri-plugin-window-pass-through-on-hover|start_pass_through': { args: undefined, options: undefined, returns: void } - 'plugin:proj-airi-tauri-plugin-window-pass-through-on-hover|stop_pass_through': { args: undefined, options: undefined, returns: void } - - // Plugin - WindowRouterLink - 'plugin:proj-airi-tauri-plugin-window-router-link|go': { - args: { route: string, windowLabel?: string } | undefined - options: undefined - returns: void - } -} - -interface InvokeMethodShape { - args: InvokeArgs | undefined - options: InvokeOptions | undefined - returns: any -} - export function useTauriCore = InvokeMethods>() { const { platform, isInitialized } = useAppRuntime() @@ -172,6 +141,14 @@ export function useTauriCore = In } }) + const tauriCoreApiInvoke = computedAsync(async () => { + await until(isInitialized).toBeTruthy() + + if (platform.value !== 'web') { + return untilImported(() => import('../tauri/invoke'), console.warn) + } + }) + async function invoke( command: C, args?: IM[C]['args'], @@ -184,13 +161,13 @@ export function useTauriCore = In return } - await until(tauriCoreApi).toBeTruthy() - const imported = await tauriCoreApi.value + await until(tauriCoreApiInvoke).toBeTruthy() + const imported = await tauriCoreApiInvoke.value if (!imported) { throw new Error('Tauri core API not available') } - return await imported.invoke(command as string, args as InvokeArgs | undefined, options as InvokeOptions | undefined) + return await imported.invoke(command, args as InvokeArgs | undefined, options as InvokeOptions | undefined) } return { diff --git a/apps/stage-tamagotchi/src/pages/settings/modules/hearing.vue b/apps/stage-tamagotchi/src/pages/settings/modules/hearing.vue index 66627e241..2c64ec531 100644 --- a/apps/stage-tamagotchi/src/pages/settings/modules/hearing.vue +++ b/apps/stage-tamagotchi/src/pages/settings/modules/hearing.vue @@ -3,84 +3,288 @@ import { LevelMeter, ThresholdMeter, TimeSeriesChart } from '@proj-airi/stage-ui import { FieldCheckbox, FieldRange, FieldSelect } from '@proj-airi/ui' import { useDevicesList } from '@vueuse/core' import { computed, nextTick, onMounted, onUnmounted, ref, watch } from 'vue' -import { useI18n } from 'vue-i18n' -import { useAudioManager } from '../../../composables/audio/manager' +import workletUrl from '../../../tauri/vad/process.worklet?worker&url' + +import { createVAD, VADAudioManager } from '../../../tauri/vad' const devices = useDevicesList({ constraints: { audio: true } }) const audioInputs = computed(() => devices.audioInputs.value) -const i18n = useI18n() -// Initialize audio manager -const audioManager = useAudioManager(i18n.locale) +const selectedAudioInput = ref(devices.audioInputs.value[0]?.deviceId || '') -const selectedAudioInput = ref('') -const selectedAudioInputSourceId = ref('') +const isMonitoring = ref(false) +const enablePlayback = ref(false) -const enabledMonitoring = ref(false) -const enabledPlayback = ref(false) -const monitorVolume = ref(50) -const useVADModel = ref(true) +// Audio processing state +const audioContext = ref() +const mediaStream = ref() +const analyser = ref() +const gainNode = ref() +const dataArray = ref() +const animationFrame = ref() -// Get current source data reactively -const currentSource = computed(() => { - return selectedAudioInputSourceId.value ? audioManager.getSourceData(selectedAudioInputSourceId.value) : null -}) +// Audio levels and indicators +const volumeLevel = ref(0) // 0-100 +const isSpeaking = ref(false) +const speakingThreshold = ref(25) // 0-100 (for volume-based fallback) +const monitorVolume = ref(50) // 0-100 -// Extract reactive values from the current source -const volumeLevel = computed(() => currentSource.value?.volume?.level.value ?? 0) -const vadProbability = computed(() => currentSource.value?.vad?.probability.value ?? 0) -const vadThreshold = computed({ - get: () => currentSource.value?.vad?.config.value.threshold ?? 0.5, - set: (value) => { - if (currentSource.value?.vad?.config.value.threshold) { - currentSource.value.vad.config.value.threshold = value - } - }, -}) +// VAD integration +const vadManager = ref() +const isVADModelLoaded = ref(false) +const isLoadingVADModel = ref(false) +const vadModelError = ref('') +const useVADModel = ref(true) // Toggle between VAD and volume-based detection +const vadProbability = ref(0) // Raw VAD probability +const vadThreshold = ref(0.5) // VAD probability threshold for speech detection -const speakingThreshold = computed({ - get: () => currentSource.value?.volume?.threshold.value ?? 25, - set: (value) => { - if (currentSource.value?.volume?.threshold) { - currentSource.value.volume.threshold.value = value - } - }, -}) +// VAD visualization +const vadHistory = ref([]) // History for chart visualization +const maxVadHistory = 50 // Keep 50 samples (~1.6 seconds at 32ms intervals) -const isVADModelLoaded = computed(() => currentSource.value?.vad?.isModelLoaded.value ?? false) -const isLoadingVADModel = computed(() => currentSource.value?.vad?.isLoading.value ?? false) -const vadModelError = computed(() => currentSource.value?.vad?.error.value ?? '') -const vadHistory = computed(() => currentSource.value?.vad?.history.value ?? []) +// VAD functions +async function loadVADModel() { + if (isVADModelLoaded.value || isLoadingVADModel.value) + return -// Speaking detection - prioritize VAD if enabled and loaded -const isSpeaking = computed(() => { - if (useVADModel.value && isVADModelLoaded.value) { - return currentSource.value?.vad?.isSpeaking.value ?? false + isLoadingVADModel.value = true + vadModelError.value = '' + + try { + // Create and initialize the VAD + const vad = await createVAD({ + sampleRate: 16000, + speechThreshold: vadThreshold.value, + exitThreshold: vadThreshold.value * 0.3, + minSilenceDurationMs: 400, + }) + + // Set up event handlers + vad.on('speech-start', () => { + isSpeaking.value = true + }) + + vad.on('speech-end', () => { + isSpeaking.value = false + }) + + vad.on('debug', ({ data }) => { + if (data?.probability !== undefined) { + vadProbability.value = data.probability + + // Update VAD history for visualization + vadHistory.value.push(data.probability) + if (vadHistory.value.length > maxVadHistory) { + vadHistory.value.shift() + } + } + }) + + vad.on('status', ({ type, message }) => { + if (type === 'error') { + vadModelError.value = message + } + }) + + // Create and initialize audio manager + const manager = new VADAudioManager(vad, { + minChunkSize: 512, + audioContextOptions: { + sampleRate: 16000, + latencyHint: 'interactive', + }, + }) + + await manager.initialize(workletUrl) + vadManager.value = manager + isVADModelLoaded.value = true } - return currentSource.value?.volume?.isSpeaking.value ?? false -}) + catch (error) { + vadModelError.value = error instanceof Error ? error.message : String(error) + console.error('Failed to load VAD model:', error) + } + finally { + isLoadingVADModel.value = false + } +} -// Playback controls -const playbackEnabled = computed({ - get: () => currentSource.value?.playback?.isEnabled.value ?? false, - set: (value) => { - if (currentSource.value?.playback?.isEnabled) { - currentSource.value.playback.isEnabled.value = value +// Audio monitoring +async function setupAudioMonitoring() { + try { + if (!selectedAudioInput.value) { + console.warn('No audio input device selected') + return } - }, -}) -const playbackVolume = computed({ - get: () => currentSource.value?.playback?.volume.value ?? 50, - set: (value) => { - if (currentSource.value?.playback?.volume) { - currentSource.value.playback.volume.value = value + // Clean up existing connections + await stopAudioMonitoring() + + // Get user media with selected device + mediaStream.value = await navigator.mediaDevices.getUserMedia({ + audio: { + deviceId: selectedAudioInput.value, + echoCancellation: true, + noiseSuppression: true, + autoGainControl: true, + }, + }) + + // Create audio context + audioContext.value = new AudioContext() + const source = audioContext.value.createMediaStreamSource(mediaStream.value) + + // Create analyser for volume detection + analyser.value = audioContext.value.createAnalyser() + analyser.value.fftSize = 256 + analyser.value.smoothingTimeConstant = 0.3 + + // Create gain node for playback volume control + gainNode.value = audioContext.value.createGain() + gainNode.value.gain.value = enablePlayback.value ? (monitorVolume.value / 100) : 0 + + // Connect audio graph + source.connect(analyser.value) + + if (enablePlayback.value) { + source.connect(gainNode.value) + gainNode.value.connect(audioContext.value.destination) } - }, + + // Set up data array for analysis + const bufferLength = analyser.value.frequencyBinCount + dataArray.value = new Uint8Array(bufferLength) + + // Start audio analysis loop + startAudioAnalysis() + + // Load VAD model and start VAD processing if enabled + if (useVADModel.value) { + await loadVADModel() + if (vadManager.value) { + await vadManager.value.startMicrophone() + } + } + } + catch (error) { + console.error('Error setting up audio monitoring:', error) + vadModelError.value = error instanceof Error ? error.message : String(error) + } +} + +async function stopAudioMonitoring() { + // Stop animation frame + if (animationFrame.value) { + cancelAnimationFrame(animationFrame.value) + animationFrame.value = undefined + } + + // Stop VAD manager + if (vadManager.value) { + await vadManager.value.stopMicrophone() + } + + // Stop media stream + if (mediaStream.value) { + mediaStream.value.getTracks().forEach(track => track.stop()) + mediaStream.value = undefined + } + + // Close audio context + if (audioContext.value) { + await audioContext.value.close() + audioContext.value = undefined + } + + analyser.value = undefined + gainNode.value = undefined + dataArray.value = undefined + volumeLevel.value = 0 + isSpeaking.value = false + vadProbability.value = 0 + vadHistory.value = [] +} + +function startAudioAnalysis() { + const analyze = () => { + if (!analyser.value || !dataArray.value) + return + + // Get frequency data for volume visualization + analyser.value.getByteFrequencyData(dataArray.value) + + // Calculate RMS volume level + let sum = 0 + for (let i = 0; i < dataArray.value.length; i++) { + sum += dataArray.value[i] * dataArray.value[i] + } + const rms = Math.sqrt(sum / dataArray.value.length) + volumeLevel.value = Math.min(100, (rms / 255) * 100 * 3) // Amplify for better visualization + + // Fallback speaking detection (when VAD model is not used) + if (!useVADModel.value || !isVADModelLoaded.value) { + isSpeaking.value = volumeLevel.value > speakingThreshold.value + } + + animationFrame.value = requestAnimationFrame(analyze) + } + analyze() +} + +// Update playback routing when playback setting changes +async function updatePlayback() { + if (!audioContext.value || !gainNode.value) + return + + if (enablePlayback.value) { + gainNode.value.gain.value = monitorVolume.value / 100 + gainNode.value.connect(audioContext.value.destination) + } + else { + gainNode.value.gain.value = 0 + gainNode.value.disconnect() + } +} + +// Watchers +watch(selectedAudioInput, async () => { + if (isMonitoring.value) { + await setupAudioMonitoring() + } }) -// Speaking indicator styling +watch(enablePlayback, updatePlayback) +watch(monitorVolume, () => { + if (gainNode.value && enablePlayback.value) { + gainNode.value.gain.value = monitorVolume.value / 100 + } +}) + +watch(audioInputs, () => { + if (!selectedAudioInput.value && audioInputs.value.length > 0) { + selectedAudioInput.value = audioInputs.value[0]?.deviceId + } +}) + +watch(vadThreshold, () => { + // Update VAD threshold if model is loaded + if (vadManager.value && isVADModelLoaded.value) { + // Note: We would need to add an updateConfig method to VADAudioManager + // For now, this is a placeholder + } +}) + +// Monitoring toggle +async function toggleMonitoring() { + if (isMonitoring.value) { + await setupAudioMonitoring() + } + else { + await stopAudioMonitoring() + } +} + +// Speaking indicator with enhanced VAD visualization const speakingIndicatorClass = computed(() => { if (!useVADModel.value || !isVADModelLoaded.value) { // Volume-based: simple green/white @@ -94,79 +298,32 @@ const speakingIndicatorClass = computed(() => { const threshold = vadThreshold.value if (prob > threshold) { - return 'bg-green-500 shadow-lg shadow-green-500/50' + // Speaking: green (could add intensity in future) + return `bg-green-500 shadow-lg shadow-green-500/50` } else if (prob > threshold * 0.5) { + // Close to threshold: yellow return 'bg-yellow-500 shadow-lg shadow-yellow-500/30' } else { + // Low probability: neutral return 'bg-white dark:bg-neutral-900 border-2 border-neutral-300 dark:border-neutral-600' } }) -// Setup primary microphone source when device changes -watch(selectedAudioInput, async (newDeviceId) => { - // Remove existing source - if (selectedAudioInputSourceId.value) { - await audioManager.stopSource(selectedAudioInputSourceId.value) - audioManager.removeSource(selectedAudioInputSourceId.value) - selectedAudioInputSourceId.value = '' - } - - // Add new source if device selected - if (newDeviceId) { - const device = audioInputs.value.find(d => d.deviceId === newDeviceId) - selectedAudioInputSourceId.value = audioManager.addMicrophone( - newDeviceId, - device?.label || 'Primary Microphone', - ) - } +// Lifecycle +onMounted(() => { + devices.ensurePermissions().then(() => nextTick()).then(() => { + if (audioInputs.value.length > 0 && !selectedAudioInput.value) { + selectedAudioInput.value = audioInputs.value[0]?.deviceId + } + }) }) -// Watch for VAD model toggle -watch([useVADModel, currentSource], ([enabled, source]) => { - if (source?.vad?.isEnabled) { - source.vad.isEnabled.value = enabled - } -}) - -// Sync playback settings -watch(enabledPlayback, (enabled) => { - playbackEnabled.value = enabled -}) - -watch(monitorVolume, (volume) => { - playbackVolume.value = volume -}) - -// Monitoring toggle -async function toggleMonitoring() { - if (!selectedAudioInputSourceId.value) - return - - if (enabledMonitoring.value) { - await audioManager.startSource(selectedAudioInputSourceId.value) - } - else { - await audioManager.stopSource(selectedAudioInputSourceId.value) - } -} - -// Initialize with first available device -onMounted(async () => { - await devices.ensurePermissions() - await nextTick() - - if (audioInputs.value.length > 0 && !selectedAudioInput.value) { - selectedAudioInput.value = audioInputs.value[0]?.deviceId - } -}) - -// Cleanup on unmount -onUnmounted(async () => { - if (selectedAudioInputSourceId.value) { - await audioManager.stopSource(selectedAudioInputSourceId.value) - audioManager.removeSource(selectedAudioInputSourceId.value) +onUnmounted(() => { + stopAudioMonitoring() + if (vadManager.value) { + vadManager.value.dispose() } }) @@ -176,11 +333,16 @@ onUnmounted(async () => {
@@ -194,43 +356,59 @@ onUnmounted(async () => {
-
+
-
+
{{ isSpeaking ? 'Speaking Detected' : 'Silence' }} @@ -242,7 +420,8 @@ onUnmounted(async () => {
@@ -253,10 +432,8 @@ onUnmounted(async () => { Loading...
-
+
+
Inference error: {{ vadModelError }}
@@ -272,34 +449,43 @@ onUnmounted(async () => {
-
+
-
+
-
+
Audio feedback warning diff --git a/apps/stage-tamagotchi/src/tauri/invoke.ts b/apps/stage-tamagotchi/src/tauri/invoke.ts new file mode 100644 index 000000000..ae888a073 --- /dev/null +++ b/apps/stage-tamagotchi/src/tauri/invoke.ts @@ -0,0 +1,44 @@ +import type { InvokeArgs, InvokeOptions } from '@tauri-apps/api/core' + +import { invoke as tauriInvoke } from '@tauri-apps/api/core' + +export interface InvokeMethods { + // app windows + 'open_settings_window': { args: undefined, options: undefined, returns: void } + 'open_chat_window': { args: undefined, options: undefined, returns: void } + + // Plugin - Audio Transcription + 'plugin:proj-airi-tauri-plugin-audio-transcription|load_model_whisper': { args: { modelType: 'base' | 'largev3' | 'tiny' | 'medium' }, options: undefined, returns: void } + 'plugin:proj-airi-tauri-plugin-audio-transcription|audio_transcription': { args: { chunk: number[], language: string }, options: undefined, returns: [string, string] } + + // Plugin - Audio VAD + 'plugin:proj-airi-tauri-plugin-audio-vad|load_model_silero_vad': { args: undefined, options: undefined, returns: void } + 'plugin:proj-airi-tauri-plugin-audio-vad|audio_vad': { args: { inputData: { input: number[], sr: number, state: number[] } }, options: undefined, returns: number } + + // Plugin - Window Pass through on hover + 'plugin:proj-airi-tauri-plugin-window-pass-through-on-hover|start_monitor': { args: undefined, options: undefined, returns: void } + 'plugin:proj-airi-tauri-plugin-window-pass-through-on-hover|stop_monitor': { args: undefined, options: undefined, returns: void } + 'plugin:proj-airi-tauri-plugin-window-pass-through-on-hover|start_pass_through': { args: undefined, options: undefined, returns: void } + 'plugin:proj-airi-tauri-plugin-window-pass-through-on-hover|stop_pass_through': { args: undefined, options: undefined, returns: void } + + // Plugin - WindowRouterLink + 'plugin:proj-airi-tauri-plugin-window-router-link|go': { + args: { route: string, windowLabel?: string } | undefined + options: undefined + returns: void + } +} + +export interface InvokeMethodShape { + args: InvokeArgs | undefined + options: InvokeOptions | undefined + returns: any +} + +export async function invoke = InvokeMethods>( + command: C, + args?: IM[C]['args'], + options?: IM[C]['options'], +): Promise { + return await tauriInvoke(command as string, args as InvokeArgs | undefined, options as InvokeOptions | undefined) +} diff --git a/apps/stage-tamagotchi/src/tauri/vad/index.ts b/apps/stage-tamagotchi/src/tauri/vad/index.ts new file mode 100644 index 000000000..2591dd3da --- /dev/null +++ b/apps/stage-tamagotchi/src/tauri/vad/index.ts @@ -0,0 +1,4 @@ +export { VADAudioManager } from './manager' +export type { VADAudioOptions } from './manager' +export { createVAD, VAD } from './vad' +export type { VADConfig, VADEventCallback, VADEvents } from './vad' diff --git a/apps/stage-tamagotchi/src/tauri/vad/manager.ts b/apps/stage-tamagotchi/src/tauri/vad/manager.ts new file mode 100644 index 000000000..47ea4e4a1 --- /dev/null +++ b/apps/stage-tamagotchi/src/tauri/vad/manager.ts @@ -0,0 +1,171 @@ +import type { VAD, VADConfig } from './vad' + +export interface VADAudioOptions { + /** + * Audio context options + */ + audioContextOptions?: AudioContextOptions + + /** + * The minimum size of audio chunks to process + */ + minChunkSize?: number + + /** + * VAD configuration options + */ + vadConfig?: Partial +} + +/** + * Manages audio input and worklet processing for the VAD module + */ +export class VADAudioManager { + private audioContext: AudioContext | null = null + private audioWorkletNode: AudioWorkletNode | null = null + private mediaStream: MediaStream | null = null + private sourceNode: MediaStreamAudioSourceNode | null = null + private vad: VAD // Updated type + private workletInitialized: boolean = false + + /** + * Create a new VAD audio manager + */ + constructor(vad: VAD, options: VADAudioOptions = {}) { // Updated parameter type + this.vad = vad + + // Create audio context with user options or defaults + this.audioContext = new AudioContext(options.audioContextOptions || { + sampleRate: 16000, // Match the VAD sample rate + latencyHint: 'interactive', + }) + } + + /** + * Initialize the audio worklet and connect to microphone + */ + public async initialize(workletUrl: string): Promise { + if (!this.audioContext) { + throw new Error('Audio context not created') + } + + try { + if (!this.workletInitialized) { + await this.audioContext.audioWorklet.addModule(workletUrl) + URL.revokeObjectURL(workletUrl) + this.workletInitialized = true + } + + // Create the worklet node + this.audioWorkletNode = new AudioWorkletNode(this.audioContext, 'vad-processor') + + // Set up message handling from the worklet + this.audioWorkletNode.port.onmessage = async (event) => { + const { buffer } = event.data + if (buffer && buffer.length > 0) { + await this.vad.processAudio(new Float32Array(buffer)) + } + } + } + catch (error) { + console.error('Failed to initialize audio worklet:', error) + throw error + } + } + + /** + * Start capturing audio from the microphone + */ + public async startMicrophone(): Promise { + if (!this.audioContext || !this.audioWorkletNode) { + throw new Error('Audio system not initialized. Call initialize() first.') + } + + try { + if (this.audioContext.state === 'suspended') { + await this.audioContext.resume() + } + + // Request microphone access + this.mediaStream = await navigator.mediaDevices.getUserMedia({ + audio: { + echoCancellation: true, + noiseSuppression: true, + autoGainControl: true, + sampleRate: this.audioContext.sampleRate, + }, + }) + + // Create source node and connect to worklet + this.sourceNode = this.audioContext.createMediaStreamSource(this.mediaStream) + this.sourceNode.connect(this.audioWorkletNode) + + // Connect worklet to a silent destination (to keep the audio graph active) + // Using a GainNode with gain=0 to ensure no sound is output + const silentGain = this.audioContext.createGain() + silentGain.gain.value = 0 + this.audioWorkletNode.connect(silentGain) + silentGain.connect(this.audioContext.destination) + } + catch (error) { + console.error('Failed to start microphone:', error) + throw error + } + } + + public async stopMicrophone(): Promise { + if (this.mediaStream) { + this.mediaStream.getTracks().forEach(track => track.stop()) + this.mediaStream = null + } + + if (this.sourceNode) { + this.sourceNode.disconnect() + this.sourceNode = null + } + + this.audioContext?.suspend() + this.audioWorkletNode?.disconnect() + } + + /** + * Stop capturing audio + */ + public stop(): void { + // Disconnect nodes + if (this.sourceNode && this.audioWorkletNode) { + this.sourceNode.disconnect() + this.audioWorkletNode.disconnect() + } + + // Stop all tracks in the media stream + if (this.mediaStream) { + this.mediaStream.getTracks().forEach(track => track.stop()) + this.mediaStream = null + } + + // Suspend the audio context rather than closing it + // This allows us to reuse it later + if (this.audioContext && this.audioContext.state !== 'closed') { + this.audioContext.suspend() + } + + this.sourceNode = null + this.audioWorkletNode = null + } + + /** + * Clean up all resources + */ + public dispose(): void { + this.stop() + + // Now fully close the audio context + if (this.audioContext && this.audioContext.state !== 'closed') { + this.audioContext.close() + this.audioContext = null + } + + this.workletInitialized = false + } +} diff --git a/apps/stage-tamagotchi/src/tauri/vad/process.worklet.ts b/apps/stage-tamagotchi/src/tauri/vad/process.worklet.ts new file mode 100644 index 000000000..899d933fe --- /dev/null +++ b/apps/stage-tamagotchi/src/tauri/vad/process.worklet.ts @@ -0,0 +1,53 @@ +// vad-worklet-processor.ts +// This file needs to be registered as an AudioWorklet + +/** + * Minimum chunk size for processing audio + */ +const MIN_CHUNK_SIZE = 512 + +/** + * Global state for audio buffer accumulation + */ +let globalPointer = 0 +const globalBuffer = new Float32Array(MIN_CHUNK_SIZE) + +/** + * VAD AudioWorklet Processor - processes audio chunks and sends them to the main thread + */ +class VADProcessor extends AudioWorkletProcessor { + process(inputs: Float32Array[][], _outputs: Float32Array[][], _parameters: Record) { + const buffer = inputs[0][0] + if (!buffer) + return true // buffer is null when the stream ends + + if (buffer.length > MIN_CHUNK_SIZE) { + // If the buffer is larger than the minimum chunk size, send the entire buffer + this.port.postMessage({ buffer }) + } + else { + const remaining = MIN_CHUNK_SIZE - globalPointer + if (buffer.length >= remaining) { + // If the buffer is larger than (or equal to) the remaining space in the global buffer, copy the remaining space + globalBuffer.set(buffer.subarray(0, remaining), globalPointer) + + // Send the global buffer + this.port.postMessage({ buffer: globalBuffer }) + + // Reset the global buffer and set the remaining buffer + globalBuffer.fill(0) + globalBuffer.set(buffer.subarray(remaining), 0) + globalPointer = buffer.length - remaining + } + else { + // If the buffer is smaller than the remaining space in the global buffer, copy the buffer to the global buffer + globalBuffer.set(buffer, globalPointer) + globalPointer += buffer.length + } + } + + return true + } +} + +registerProcessor('vad-processor', VADProcessor) diff --git a/apps/stage-tamagotchi/src/tauri/vad/vad.ts b/apps/stage-tamagotchi/src/tauri/vad/vad.ts new file mode 100644 index 000000000..ae8e8f802 --- /dev/null +++ b/apps/stage-tamagotchi/src/tauri/vad/vad.ts @@ -0,0 +1,236 @@ +import { invoke } from '../invoke' + +export interface VADConfig { + sampleRate: number + speechThreshold: number + exitThreshold: number + minSilenceDurationMs: number + speechPadMs: number + minSpeechDurationMs: number + maxBufferDuration: number + newBufferSize: number +} + +export interface VADEvents { + 'speech-start': void + 'speech-end': void + 'speech-ready': { buffer: Float32Array, duration: number } + 'status': { type: string, message: string } + 'debug': { message: string, data?: any } +} + +export type VADEventCallback = (event: VADEvents[K]) => void + +export class VAD { + private config: VADConfig + private state: Float32Array = new Float32Array(2 * 1 * 128) // 2, 1, 128 + private buffer: Float32Array + private bufferPointer: number = 0 + private isRecording: boolean = false + private postSpeechSamples: number = 0 + private prevBuffers: Float32Array[] = [] + private inferenceChain: Promise = Promise.resolve() + private eventListeners: Partial[]>> = {} + private isReady: boolean = false + + constructor(userConfig: Partial = {}) { + const defaultConfig: VADConfig = { + sampleRate: 16000, + speechThreshold: 0.3, + exitThreshold: 0.1, + minSilenceDurationMs: 400, + speechPadMs: 80, + minSpeechDurationMs: 250, + maxBufferDuration: 30, + newBufferSize: 512, + } + + this.config = { ...defaultConfig, ...userConfig } + this.buffer = new Float32Array(this.config.maxBufferDuration * this.config.sampleRate) + } + + public async initialize(): Promise { + try { + this.emit('status', { type: 'info', message: 'Loading VAD model...' }) + + await invoke('plugin:proj-airi-tauri-plugin-audio-vad|load_model_silero_vad') + this.isReady = true + + this.emit('status', { type: 'info', message: 'VAD model loaded successfully' }) + } + catch (error) { + this.emit('status', { type: 'error', message: `Failed to load VAD model: ${error}` }) + throw error + } + } + + public on(event: K, callback: VADEventCallback): void { + if (!this.eventListeners[event]) { + this.eventListeners[event] = [] + } + this.eventListeners[event]!.push(callback as any) + } + + public off(event: K, callback: VADEventCallback): void { + if (!this.eventListeners[event]) + return + this.eventListeners[event] = this.eventListeners[event]!.filter(cb => cb !== callback) + } + + private emit(event: K, data: VADEvents[K]): void { + if (!this.eventListeners[event]) + return + for (const callback of this.eventListeners[event]!) { + callback(data) + } + } + + public async processAudio(inputBuffer: Float32Array): Promise { + if (!this.isReady) { + throw new Error('VAD model is not initialized. Call initialize() first.') + } + + const wasRecording = this.isRecording + + // Perform VAD using Rust backend + const isSpeech = await this.detectSpeech(inputBuffer) + + // The rest of the logic remains the same as your original implementation + const sampleRateMs = this.config.sampleRate / 1000 + const minSilenceDurationSamples = this.config.minSilenceDurationMs * sampleRateMs + const speechPadSamples = this.config.speechPadMs * sampleRateMs + const minSpeechDurationSamples = this.config.minSpeechDurationMs * sampleRateMs + const maxPrevBuffers = Math.ceil(speechPadSamples / this.config.newBufferSize) + + if (!wasRecording && !isSpeech) { + if (this.prevBuffers.length >= maxPrevBuffers) { + this.prevBuffers.shift() + } + this.prevBuffers.push(inputBuffer.slice(0)) + return + } + + const remaining = this.buffer.length - this.bufferPointer + if (inputBuffer.length >= remaining) { + this.buffer.set(inputBuffer.subarray(0, remaining), this.bufferPointer) + this.bufferPointer += remaining + + const overflow = inputBuffer.subarray(remaining) + this.processSpeechSegment(overflow) + return + } + else { + this.buffer.set(inputBuffer, this.bufferPointer) + this.bufferPointer += inputBuffer.length + } + + if (isSpeech) { + if (!this.isRecording) { + this.emit('speech-start', undefined) + this.emit('status', { type: 'info', message: 'Speech detected' }) + } + + this.isRecording = true + this.postSpeechSamples = 0 + return + } + + this.postSpeechSamples += inputBuffer.length + + if (this.postSpeechSamples >= minSilenceDurationSamples) { + if (this.bufferPointer < minSpeechDurationSamples) { + this.reset() + return + } + + this.processSpeechSegment() + } + } + + private async detectSpeech(buffer: Float32Array): Promise { + // Use Rust backend for inference + const result = await (this.inferenceChain = this.inferenceChain.then(() => + invoke('plugin:proj-airi-tauri-plugin-audio-vad|audio_vad', { + inputData: { + input: Array.from(buffer), + sr: this.config.sampleRate, + state: Array.from(this.state), + }, + }), + )) as { output: number[], state: number[] } + + // Update the state + this.state = new Float32Array(result.state) + + // Get the speech probability + const speechProb = result.output[0] + + this.emit('debug', { + message: 'VAD score', + data: { probability: speechProb }, + }) + + // Apply thresholds + return ( + speechProb > this.config.speechThreshold + || (this.isRecording && speechProb >= this.config.exitThreshold) + ) + } + + private processSpeechSegment(overflow?: Float32Array): void { + const sampleRateMs = this.config.sampleRate / 1000 + const speechPadSamples = this.config.speechPadMs * sampleRateMs + + const duration = (this.bufferPointer / this.config.sampleRate) * 1000 + const overflowLength = overflow?.length ?? 0 + + const prevLength = this.prevBuffers.reduce((acc, b) => acc + b.length, 0) + const finalBuffer = new Float32Array(prevLength + this.bufferPointer + speechPadSamples) + + let offset = 0 + for (const prev of this.prevBuffers) { + finalBuffer.set(prev, offset) + offset += prev.length + } + + finalBuffer.set(this.buffer.slice(0, this.bufferPointer + speechPadSamples), offset) + + this.emit('speech-end', undefined) + this.emit('speech-ready', { + buffer: finalBuffer, + duration, + }) + + if (overflow) { + this.buffer.set(overflow, 0) + } + this.reset(overflowLength) + } + + private reset(offset: number = 0): void { + this.buffer.fill(0, offset) + this.bufferPointer = offset + this.isRecording = false + this.postSpeechSamples = 0 + this.prevBuffers = [] + } + + public updateConfig(newConfig: Partial): void { + this.config = { ...this.config, ...newConfig } + + if (newConfig.maxBufferDuration || newConfig.sampleRate) { + this.buffer = new Float32Array(this.config.maxBufferDuration * this.config.sampleRate) + this.bufferPointer = 0 + } + } + + public isCurrentlyRecording(): boolean { + return this.isRecording + } +} + +export async function createVAD(config?: Partial): Promise { + const vad = new VAD(config) + await vad.initialize() + return vad +} diff --git a/apps/stage-tamagotchi/tsconfig.json b/apps/stage-tamagotchi/tsconfig.json index f5fe46eeb..5f75dd763 100644 --- a/apps/stage-tamagotchi/tsconfig.json +++ b/apps/stage-tamagotchi/tsconfig.json @@ -17,7 +17,8 @@ "vite/client", "vite-plugin-vue-layouts/client", "unplugin-vue-macros/macros-global", - "unplugin-vue-router/client" + "unplugin-vue-router/client", + "@types/audioworklet" ], "allowJs": true, "strict": true, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ef6d3e4cb..4cb72aa0e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -321,6 +321,9 @@ importers: '@tresjs/core': specifier: ^4.3.6 version: 4.3.6(three@0.177.0)(typescript@5.8.3)(vue@3.5.17(typescript@5.8.3)) + '@types/audioworklet': + specifier: ^0.0.77 + version: 0.0.77 '@vueuse/core': specifier: ^13.4.0 version: 13.4.0(vue@3.5.17(typescript@5.8.3))