feat(stage-tamagotchi): realtime whisper

apps/stage-tamagotchi/src-tauri/src/app/models/mod.rs

@@ -8,10 +8,19 @@ use tauri::Runtime;
 use crate::{app::models::whisper::WhichWhisperModel, helpers::huggingface::load_device};
 
 pub fn new_whisper_processor<R: Runtime>(
-  window: tauri::WebviewWindow<R>
+  window: tauri::WebviewWindow<R>,
+  model_type: Option<WhichWhisperModel>,
 ) -> anyhow::Result<whisper::Processor> {
   let device = load_device().map_err(|err| anyhow::anyhow!("Failed to load device: {}", err))?;
-  let whisper_model = WhichWhisperModel::Tiny;
+  let whisper_model = model_type.unwrap_or_else(|| {
+    if device.is_cuda() {
+      WhichWhisperModel::LargeV3
+    } else if device.is_metal() {
+      WhichWhisperModel::Base
+    } else {
+      WhichWhisperModel::Tiny
+    }
+  });
   info!("Loading whisper model: {:?}", whisper_model);
   whisper::Processor::new(whisper_model, device.clone(), window)
 }

apps/stage-tamagotchi/src-tauri/src/plugins/audio_transcription/mod.rs

@@ -1,5 +1,6 @@
 use std::sync::Mutex;
 
+use clap::ValueEnum;
 use log::info;
 use tauri::{
   Manager,
@@ -7,7 +8,7 @@ use tauri::{
   plugin::{Builder as PluginBuilder, TauriPlugin},
 };
 
-use crate::app::models::new_whisper_processor;
+use crate::app::models::{new_whisper_processor, whisper::WhichWhisperModel};
 
 #[derive(Default)]
 struct AppDataWhisperProcessor {
@@ -18,6 +19,7 @@ struct AppDataWhisperProcessor {
 pub async fn load_model_whisper<R: Runtime>(
   app: tauri::AppHandle<R>,
   window: tauri::WebviewWindow<R>,
+  model_type: Option<String>,
 ) -> Result<(), String> {
   info!("Loading models...");
 
@@ -31,7 +33,15 @@ pub async fn load_model_whisper<R: Runtime>(
   }
 
   // Load the traditional whisper models first
-  match new_whisper_processor(window) {
+  match new_whisper_processor(
+    window,
+    Some(WhichWhisperModel::from_str(
+      model_type
+        .unwrap_or_else(|| "medium".to_string())
+        .as_str(),
+      true,
+    )?),
+  ) {
     Ok(p) => {
       let data = app.state::<Mutex<AppDataWhisperProcessor>>();
       let mut data = data.lock().unwrap();

apps/stage-tamagotchi/src/composables/audio/analysis-vad.ts

@@ -0,0 +1,397 @@
+import type { AudioChunkCallback } from '@proj-airi/audio/vue'
+
+import { onUnmounted, readonly, ref, watch } from 'vue'
+
+import { useTauriCore } from '../tauri'
+
+export interface VADSegment {
+  id: string
+  audioData: Float32Array
+  startTime: number
+  endTime: number
+  probability: number
+  isComplete: boolean
+}
+
+export interface VADConfig {
+  threshold: number
+  silenceGapMs: number // How long silence before ending segment
+  minSpeechDurationMs: number // Minimum speech duration to be valid
+  maxSpeechDurationMs: number // Maximum speech duration before forced split
+  overlapMs: number // Overlap between segments for better transcription
+  bufferSizeMs: number // How much audio to keep in memory
+}
+
+export function useVADAnalysis(config: Partial<VADConfig> = {}) {
+  const { invoke } = useTauriCore()
+
+  const isModelLoaded = ref(false)
+  const isLoading = ref(false)
+  const error = ref('')
+  const isEnabled = ref(true)
+
+  const probability = ref(0)
+  const history = ref<number[]>([])
+  const maxHistory = 50
+
+  // Configuration with defaults
+  const vadConfig = ref<VADConfig>({
+    threshold: 0.5,
+    silenceGapMs: 500, // 500ms of silence to end segment
+    minSpeechDurationMs: 300, // Minimum 300ms speech
+    maxSpeechDurationMs: 30000, // Max 30s per segment
+    overlapMs: 200, // 200ms overlap
+    bufferSizeMs: 60000, // Keep 60s of audio in memory
+    ...config,
+  })
+
+  // Audio buffering
+  const audioBuffer = ref<Float32Array>(new Float32Array(0))
+  const chunkSize = 512
+  const targetSampleRate = 16000
+  let processingInterval: number | null = null
+
+  // Segment management
+  const currentSegment = ref<VADSegment | null>(null)
+  const completedSegments = ref<VADSegment[]>([])
+  const maxCompletedSegments = 10 // Keep last 10 completed segments
+
+  // State tracking
+  const isSpeaking = ref(false)
+  const lastSpeechTime = ref(0)
+  const lastSilenceTime = ref(0)
+  const segmentStartTime = ref(0)
+  const segmentAudioBuffer = ref<Float32Array>(new Float32Array(0))
+
+  // Callbacks for segment events
+  const segmentCallbacks = new Set<(segment: VADSegment) => void>()
+
+  async function loadModel() {
+    if (isModelLoaded.value || isLoading.value)
+      return
+
+    isLoading.value = true
+    error.value = ''
+
+    try {
+      await invoke('plugin:proj-airi-tauri-plugin-audio-vad|load_model_silero_vad')
+      isModelLoaded.value = true
+    }
+    catch (err) {
+      error.value = err instanceof Error ? err.message : String(err)
+      console.error('Failed to load VAD model:', err)
+    }
+    finally {
+      isLoading.value = false
+    }
+  }
+
+  function resampleIfNeeded(chunk: Float32Array, sourceSampleRate: number): Float32Array {
+    if (sourceSampleRate === targetSampleRate) {
+      return chunk
+    }
+
+    const ratio = targetSampleRate / sourceSampleRate
+    const outputLength = Math.floor(chunk.length * ratio)
+    const resampled = new Float32Array(outputLength)
+
+    for (let i = 0; i < outputLength; i++) {
+      const sourceIndex = i / ratio
+      const index = Math.floor(sourceIndex)
+      const fraction = sourceIndex - index
+
+      if (index + 1 < chunk.length) {
+        resampled[i] = chunk[index] * (1 - fraction) + chunk[index + 1] * fraction
+      }
+      else {
+        resampled[i] = chunk[index] || 0
+      }
+    }
+
+    return resampled
+  }
+
+  async function processChunk(chunk: Float32Array) {
+    if (!isModelLoaded.value || chunk.length !== chunkSize)
+      return
+
+    try {
+      const chunkArray = Array.from(chunk)
+      const prob = await invoke('plugin:proj-airi-tauri-plugin-audio-vad|audio_vad', {
+        chunk: chunkArray,
+      })
+
+      if (typeof prob === 'number') {
+        const now = performance.now()
+        probability.value = prob
+
+        // Update history
+        history.value.push(prob)
+        if (history.value.length > maxHistory) {
+          history.value.shift()
+        }
+
+        // Process speech detection with hysteresis
+        const wasSpeaking = isSpeaking.value
+        const currentlySpeaking = prob > vadConfig.value.threshold
+
+        if (currentlySpeaking) {
+          lastSpeechTime.value = now
+        }
+        else {
+          lastSilenceTime.value = now
+        }
+
+        // Handle speech state transitions
+        if (!wasSpeaking && currentlySpeaking) {
+          // Start of speech
+          startSpeechSegment(now, chunk)
+          isSpeaking.value = true
+        }
+        else if (wasSpeaking && !currentlySpeaking) {
+          // Potential end of speech - wait for silence gap
+          const silenceDuration = now - lastSpeechTime.value
+          if (silenceDuration >= vadConfig.value.silenceGapMs) {
+            await endSpeechSegment(now)
+            isSpeaking.value = false
+          }
+        }
+        else if (wasSpeaking && currentlySpeaking) {
+          // Continuing speech - add to current segment
+          addToCurrentSegment(chunk)
+
+          // Check for max duration
+          const segmentDuration = now - segmentStartTime.value
+          if (segmentDuration >= vadConfig.value.maxSpeechDurationMs) {
+            await endSpeechSegment(now, true) // Force end
+            startSpeechSegment(now, chunk) // Start new segment
+          }
+        }
+
+        // Always add audio to segment buffer when speaking or recently speaking
+        const timeSinceLastSpeech = now - lastSpeechTime.value
+        if (timeSinceLastSpeech <= vadConfig.value.silenceGapMs + vadConfig.value.overlapMs) {
+          addToCurrentSegment(chunk)
+        }
+      }
+    }
+    catch (err) {
+      error.value = err instanceof Error ? err.message : String(err)
+      console.error('VAD processing error:', err)
+    }
+  }
+
+  function startSpeechSegment(timestamp: number, initialChunk: Float32Array) {
+    segmentStartTime.value = timestamp
+
+    // Include some pre-speech audio for context (overlap)
+    const overlapSamples = Math.floor((vadConfig.value.overlapMs / 1000) * targetSampleRate)
+    const totalBufferLength = audioBuffer.value.length
+    const startIndex = Math.max(0, totalBufferLength - overlapSamples)
+
+    const preAudio = audioBuffer.value.slice(startIndex)
+    segmentAudioBuffer.value = new Float32Array(preAudio.length + initialChunk.length)
+    segmentAudioBuffer.value.set(preAudio, 0)
+    segmentAudioBuffer.value.set(initialChunk, preAudio.length)
+
+    currentSegment.value = {
+      id: `segment_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`,
+      audioData: new Float32Array(0), // Will be set when segment ends
+      startTime: timestamp,
+      endTime: 0,
+      probability: probability.value,
+      isComplete: false,
+    }
+  }
+
+  function addToCurrentSegment(chunk: Float32Array) {
+    if (!currentSegment.value)
+      return
+
+    // Append to segment buffer
+    const oldBuffer = segmentAudioBuffer.value
+    const newBuffer = new Float32Array(oldBuffer.length + chunk.length)
+    newBuffer.set(oldBuffer, 0)
+    newBuffer.set(chunk, oldBuffer.length)
+    segmentAudioBuffer.value = newBuffer
+  }
+
+  async function endSpeechSegment(timestamp: number, forced = false) {
+    if (!currentSegment.value)
+      return
+
+    const segmentDuration = timestamp - segmentStartTime.value
+
+    // Check minimum duration requirement
+    if (!forced && segmentDuration < vadConfig.value.minSpeechDurationMs) {
+      // Too short, discard
+      currentSegment.value = null
+      segmentAudioBuffer.value = new Float32Array(0)
+      return
+    }
+
+    // Add some post-speech audio for context
+    const overlapSamples = Math.floor((vadConfig.value.overlapMs / 1000) * targetSampleRate)
+    const postAudioLength = Math.min(overlapSamples, audioBuffer.value.length)
+    const postAudio = audioBuffer.value.slice(-postAudioLength)
+
+    // Final segment audio
+    const finalAudioBuffer = new Float32Array(segmentAudioBuffer.value.length + postAudio.length)
+    finalAudioBuffer.set(segmentAudioBuffer.value, 0)
+    finalAudioBuffer.set(postAudio, segmentAudioBuffer.value.length)
+
+    // Complete the segment
+    const completedSegment: VADSegment = {
+      ...currentSegment.value,
+      audioData: finalAudioBuffer,
+      endTime: timestamp,
+      isComplete: true,
+    }
+
+    // Add to completed segments
+    completedSegments.value.push(completedSegment)
+    if (completedSegments.value.length > maxCompletedSegments) {
+      completedSegments.value.shift()
+    }
+
+    // Notify callbacks
+    segmentCallbacks.forEach((callback) => {
+      try {
+        callback(completedSegment)
+      }
+      catch (err) {
+        console.error('Segment callback error:', err)
+      }
+    })
+
+    // Reset for next segment
+    currentSegment.value = null
+    segmentAudioBuffer.value = new Float32Array(0)
+  }
+
+  function startProcessing() {
+    if (processingInterval)
+      return
+
+    const intervalMs = (chunkSize / targetSampleRate) * 1000
+
+    processingInterval = window.setInterval(async () => {
+      if (audioBuffer.value.length >= chunkSize) {
+        const chunk = audioBuffer.value.slice(0, chunkSize)
+        await processChunk(chunk)
+
+        const remaining = audioBuffer.value.slice(chunkSize)
+        audioBuffer.value = remaining.length > 0 ? remaining : new Float32Array(0)
+      }
+    }, Math.max(1, intervalMs / 2))
+  }
+
+  function stopProcessing() {
+    if (processingInterval) {
+      clearInterval(processingInterval)
+      processingInterval = null
+    }
+
+    // End current segment if active
+    if (currentSegment.value) {
+      endSpeechSegment(performance.now(), true)
+    }
+
+    audioBuffer.value = new Float32Array(0)
+    probability.value = 0
+    history.value = []
+  }
+
+  // Audio chunk callback
+  const audioChunkCallback: AudioChunkCallback = (chunk: Float32Array, sampleRate: number) => {
+    if (!isEnabled.value || !isModelLoaded.value)
+      return
+
+    const processedChunk = resampleIfNeeded(chunk, sampleRate)
+
+    // Add to main buffer
+    const currentBuffer = audioBuffer.value
+    const newBuffer = new Float32Array(currentBuffer.length + processedChunk.length)
+    newBuffer.set(currentBuffer, 0)
+    newBuffer.set(processedChunk, currentBuffer.length)
+
+    // Trim buffer to max size
+    const maxBufferSamples = (vadConfig.value.bufferSizeMs / 1000) * targetSampleRate
+    if (newBuffer.length > maxBufferSamples) {
+      const trimAmount = newBuffer.length - maxBufferSamples
+      audioBuffer.value = newBuffer.slice(trimAmount)
+    }
+    else {
+      audioBuffer.value = newBuffer
+    }
+  }
+
+  // Segment callback management
+  function onSegmentComplete(callback: (segment: VADSegment) => void) {
+    segmentCallbacks.add(callback)
+    return () => segmentCallbacks.delete(callback)
+  }
+
+  // Manual segment control
+  async function forceEndCurrentSegment() {
+    if (currentSegment.value) {
+      await endSpeechSegment(performance.now(), true)
+      isSpeaking.value = false
+    }
+  }
+
+  function clearCompletedSegments() {
+    completedSegments.value = []
+  }
+
+  // Get segment by ID
+  function getSegment(id: string): VADSegment | undefined {
+    return completedSegments.value.find(s => s.id === id)
+  }
+
+  // Auto-start processing when enabled and model is loaded
+  watch([isEnabled, isModelLoaded], ([enabled, loaded]) => {
+    if (enabled && loaded) {
+      startProcessing()
+    }
+    else {
+      stopProcessing()
+    }
+  })
+
+  onUnmounted(() => {
+    stopProcessing()
+  })
+
+  return {
+    // State
+    isModelLoaded: readonly(isModelLoaded),
+    isLoading: readonly(isLoading),
+    error: readonly(error),
+    isEnabled,
+
+    // VAD data
+    probability: readonly(probability),
+    history: readonly(history),
+    isSpeaking: readonly(isSpeaking),
+
+    // Configuration
+    config: vadConfig,
+
+    // Segments
+    currentSegment: readonly(currentSegment),
+    completedSegments: readonly(completedSegments),
+
+    // Controls
+    loadModel,
+    forceEndCurrentSegment,
+    clearCompletedSegments,
+    getSegment,
+
+    // Events
+    onSegmentComplete,
+
+    // For audio stream integration
+    audioChunkCallback,
+  }
+}

apps/stage-tamagotchi/src/composables/audio/analysis-volume.ts

@@ -0,0 +1,46 @@
+import type { AudioAnalysisCallback } from '@proj-airi/audio/vue'
+
+import { computed, readonly, ref } from 'vue'
+
+export function useVolumeAnalysis() {
+  const level = ref(0)
+  const threshold = ref(25)
+  const history = ref<number[]>([])
+  const maxHistory = 100
+  const isEnabled = ref(true)
+
+  const isSpeaking = computed(() => level.value > threshold.value)
+
+  const audioAnalysisCallback: AudioAnalysisCallback = (data) => {
+    if (!isEnabled.value)
+      return
+
+    level.value = data.volumeLevel
+
+    // Update history
+    history.value.push(data.volumeLevel)
+    if (history.value.length > maxHistory) {
+      history.value.shift()
+    }
+  }
+
+  function reset() {
+    level.value = 0
+    history.value = []
+  }
+
+  return {
+    // State
+    level: readonly(level),
+    threshold,
+    history: readonly(history),
+    isSpeaking,
+    isEnabled,
+
+    // Controls
+    reset,
+
+    // For audio stream integration
+    audioAnalysisCallback,
+  }
+}

apps/stage-tamagotchi/src/composables/audio/extract-whisper-languages.ts

@@ -0,0 +1,112 @@
+// Language constants
+const LANGUAGES: Record<string, string>[] = [
+  { en: 'english' },
+  { zh: 'chinese' },
+  { 'zh-Hans': 'chinese' },
+  { 'zh-Hant': 'chinese' },
+  { 'zh-CN': 'chinese' },
+  { 'zh-TW': 'chinese' },
+  { 'zh-HK': 'chinese' },
+  { de: 'german' },
+  { es: 'spanish' },
+  { ru: 'russian' },
+  { ko: 'korean' },
+  { fr: 'french' },
+  { ja: 'japanese' },
+  { pt: 'portuguese' },
+  { tr: 'turkish' },
+  { pl: 'polish' },
+  { ca: 'catalan' },
+  { nl: 'dutch' },
+  { ar: 'arabic' },
+  { sv: 'swedish' },
+  { it: 'italian' },
+  { id: 'indonesian' },
+  { hi: 'hindi' },
+  { fi: 'finnish' },
+  { vi: 'vietnamese' },
+  { he: 'hebrew' },
+  { uk: 'ukrainian' },
+  { el: 'greek' },
+  { ms: 'malay' },
+  { cs: 'czech' },
+  { ro: 'romanian' },
+  { da: 'danish' },
+  { hu: 'hungarian' },
+  { ta: 'tamil' },
+  { no: 'norwegian' },
+  { th: 'thai' },
+  { ur: 'urdu' },
+  { hr: 'croatian' },
+  { bg: 'bulgarian' },
+  { lt: 'lithuanian' },
+  { la: 'latin' },
+  { mi: 'maori' },
+  { ml: 'malayalam' },
+  { cy: 'welsh' },
+  { sk: 'slovak' },
+  { te: 'telugu' },
+  { fa: 'persian' },
+  { lv: 'latvian' },
+  { bn: 'bengali' },
+  { sr: 'serbian' },
+  { az: 'azerbaijani' },
+  { sl: 'slovenian' },
+  { kn: 'kannada' },
+  { et: 'estonian' },
+  { mk: 'macedonian' },
+  { br: 'breton' },
+  { eu: 'basque' },
+  { is: 'icelandic' },
+  { hy: 'armenian' },
+  { ne: 'nepali' },
+  { mn: 'mongolian' },
+  { bs: 'bosnian' },
+  { kk: 'kazakh' },
+  { sq: 'albanian' },
+  { sw: 'swahili' },
+  { gl: 'galician' },
+  { mr: 'marathi' },
+  { pa: 'punjabi' },
+  { si: 'sinhala' },
+  { km: 'khmer' },
+  { sn: 'shona' },
+  { yo: 'yoruba' },
+  { so: 'somali' },
+  { af: 'afrikaans' },
+  { oc: 'occitan' },
+  { ka: 'georgian' },
+  { be: 'belarusian' },
+  { tg: 'tajik' },
+  { sd: 'sindhi' },
+  { gu: 'gujarati' },
+  { am: 'amharic' },
+  { yi: 'yiddish' },
+  { lo: 'lao' },
+  { uz: 'uzbek' },
+  { fo: 'faroese' },
+  { ht: 'haitian creole' },
+  { ps: 'pashto' },
+  { tk: 'turkmen' },
+  { nn: 'nynorsk' },
+  { mt: 'maltese' },
+  { sa: 'sanskrit' },
+  { lb: 'luxembourgish' },
+  { my: 'myanmar' },
+  { bo: 'tibetan' },
+  { tl: 'tagalog' },
+  { mg: 'malagasy' },
+  { as: 'assamese' },
+  { tt: 'tatar' },
+  { haw: 'hawaiian' },
+  { ln: 'lingala' },
+  { ha: 'hausa' },
+  { ba: 'bashkir' },
+  { jw: 'javanese' },
+  { su: 'sundanese' },
+]
+
+export function mapLanguageCodeToName(code: string): string {
+  const lang = LANGUAGES.find(lang => lang[code])
+  return lang ? lang[code] : 'Unknown Language'
+}

apps/stage-tamagotchi/src/composables/audio/extract-whisper.ts

@@ -0,0 +1,206 @@
+import type { VADSegment } from './analysis-vad'
+
+import { computed, readonly, ref } from 'vue'
+
+import { useTauriCore } from '../tauri'
+import { mapLanguageCodeToName } from './extract-whisper-languages'
+
+export interface WhisperConfig {
+  modelSize: 'tiny' | 'base' | 'small' | 'medium' | 'large'
+  language?: string
+  temperature: number
+  beamSize: number
+  bestOf: number
+}
+
+export interface TranscriptionResult {
+  id: string
+  segmentId: string
+  text: string
+  language?: string
+  processingTimeMs: number
+  timestamp: number
+}
+
+export function useWhisperTranscription(config: Partial<WhisperConfig> = {}) {
+  const { invoke } = useTauriCore()
+
+  const isModelLoaded = ref(false)
+  const isLoading = ref(false)
+  const error = ref('')
+  const isProcessing = ref(false)
+
+  const whisperConfig = ref<WhisperConfig>({
+    modelSize: 'base',
+    language: undefined, // Auto-detect
+    temperature: 0.0,
+    beamSize: 5,
+    bestOf: 5,
+    ...config,
+  })
+
+  const transcriptionQueue = ref<VADSegment[]>([])
+  const transcriptionResults = ref<TranscriptionResult[]>([])
+  const maxResults = 50
+
+  const onTranscriptionResultHooks = ref<((result: TranscriptionResult) => void)[]>([])
+
+  const currentTranscription = computed(() =>
+    transcriptionResults.value[transcriptionResults.value.length - 1],
+  )
+
+  async function loadModel(modelType: 'base' | 'largev3' | 'tiny' | 'medium' = 'base') {
+    if (isModelLoaded.value || isLoading.value)
+      return
+
+    isLoading.value = true
+    error.value = ''
+
+    try {
+      await invoke('plugin:proj-airi-tauri-plugin-audio-transcription|load_model_whisper', { modelType })
+      isModelLoaded.value = true
+    }
+    catch (err) {
+      error.value = err instanceof Error ? err.message : String(err)
+      console.error('Failed to load Whisper model:', err)
+    }
+    finally {
+      isLoading.value = false
+    }
+  }
+
+  async function transcribeSegment(segment: VADSegment, locale: string): Promise<TranscriptionResult | null> {
+    if (!isModelLoaded.value || !segment.isComplete)
+      return null
+
+    const startTime = performance.now()
+
+    try {
+      isProcessing.value = true
+
+      const audioArray = Array.from(segment.audioData)
+      const [result, language] = await invoke('plugin:proj-airi-tauri-plugin-audio-transcription|audio_transcription', {
+        chunk: audioArray,
+        language: mapLanguageCodeToName(locale),
+      }) || ['', '']
+
+      const transcription: TranscriptionResult = {
+        id: `transcription_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`,
+        segmentId: segment.id,
+        text: result?.trim() || '',
+        language: language?.trim() || '',
+        processingTimeMs: performance.now() - startTime,
+        timestamp: Date.now(),
+      }
+
+      // Add to results
+      transcriptionResults.value.push(transcription)
+      if (transcriptionResults.value.length > maxResults) {
+        transcriptionResults.value.shift()
+      }
+
+      // Notify hooks
+      onTranscriptionResultHooks.value.forEach(cb => cb(transcription))
+
+      return transcription
+    }
+    catch (err) {
+      error.value = err instanceof Error ? err.message : String(err)
+      console.error('Transcription error:', err)
+      return null
+    }
+    finally {
+      isProcessing.value = false
+    }
+  }
+
+  async function transcribeAudioData(audioData: Float32Array, locale: string): Promise<TranscriptionResult | null> {
+    if (!isModelLoaded.value)
+      return null
+
+    const fakeSegment: VADSegment = {
+      id: `manual_${Date.now()}`,
+      audioData,
+      startTime: 0,
+      endTime: (audioData.length / 16000) * 1000,
+      probability: 1.0,
+      isComplete: true,
+    }
+
+    return await transcribeSegment(fakeSegment, locale)
+  }
+
+  function queueSegment(segment: VADSegment, locale: string) {
+    if (segment.isComplete) {
+      transcriptionQueue.value.push(segment)
+      processQueue(locale)
+    }
+  }
+
+  async function processQueue(locale: string) {
+    if (isProcessing.value || transcriptionQueue.value.length === 0)
+      return
+
+    const segment = transcriptionQueue.value.shift()
+    if (segment) {
+      await transcribeSegment(segment, locale)
+      // Process next item in queue
+      if (transcriptionQueue.value.length > 0) {
+        setTimeout(() => processQueue(locale), 100)
+      }
+    }
+  }
+
+  async function onTranscriptionResult(cb: (result: TranscriptionResult) => void) {
+    onTranscriptionResultHooks.value.push(cb)
+
+    // Immediately call with current results
+    transcriptionResults.value.forEach(result => cb(result))
+
+    // Return a cleanup function
+    return () => {
+      const index = onTranscriptionResultHooks.value.indexOf(cb)
+      if (index !== -1) {
+        onTranscriptionResultHooks.value.splice(index, 1)
+      }
+    }
+  }
+
+  function clearResults() {
+    transcriptionResults.value = []
+  }
+
+  function clearQueue() {
+    transcriptionQueue.value = []
+  }
+
+  function getResultBySegmentId(segmentId: string): TranscriptionResult | undefined {
+    return transcriptionResults.value.find(r => r.segmentId === segmentId)
+  }
+
+  return {
+    // State
+    isModelLoaded: readonly(isModelLoaded),
+    isLoading: readonly(isLoading),
+    isProcessing: readonly(isProcessing),
+    error: readonly(error),
+
+    // Configuration
+    config: whisperConfig,
+
+    // Results
+    transcriptionResults: readonly(transcriptionResults),
+    currentTranscription,
+    queueLength: computed(() => transcriptionQueue.value.length),
+
+    // Controls
+    loadModel,
+    transcribeSegment,
+    transcribeAudioData,
+    queueSegment,
+    clearResults,
+    clearQueue,
+    getResultBySegmentId,
+    onTranscriptionResult,
+  }
+}

apps/stage-tamagotchi/src/composables/audio/manager.ts

@@ -0,0 +1,171 @@
+import type { AudioStreamConfig } from '@proj-airi/audio/vue'
+import type { MaybeRefOrGetter } from 'vue'
+
+import { cleanupAudioContext, getAudioContext, getAudioContextState, isAudioContextReady } from '@proj-airi/audio/audio-context'
+import { useAudioPlayback, useAudioStream } from '@proj-airi/audio/vue'
+import { computed, readonly, shallowRef, toRef } from 'vue'
+
+import { useVADAnalysis } from './analysis-vad'
+import { useVolumeAnalysis } from './analysis-volume'
+import { useWhisperTranscription } from './extract-whisper'
+
+export interface AudioSource {
+  id: string
+  name: string
+  deviceId: string
+  type: 'microphone' | 'screen' | 'system'
+  config: AudioStreamConfig
+}
+
+export function useAudioManager(locale: MaybeRefOrGetter<string> = 'en') {
+  const audioContext = getAudioContext()
+
+  const sources = shallowRef(new Map<string, AudioSource>())
+  const activeStreams = shallowRef(new Map<string, ReturnType<typeof useAudioStream>>())
+  const vadAnalyzers = shallowRef(new Map<string, ReturnType<typeof useVADAnalysis>>())
+  const volumeAnalyzers = shallowRef(new Map<string, ReturnType<typeof useVolumeAnalysis>>())
+  const playbackControllers = shallowRef(new Map<string, ReturnType<typeof useAudioPlayback>>())
+  const whisperTranscribers = shallowRef(new Map<string, ReturnType<typeof useWhisperTranscription>>())
+
+  const audioContextState = toRef(() => getAudioContextState())
+  const localeRef = toRef(locale)
+
+  function addSource(source: AudioSource) {
+    sources.value.set(source.id, source)
+
+    // Create composables
+    const configRef = computed(() => sources.value.get(source.id)?.config)
+    const stream = useAudioStream(configRef)
+    const vadAnalyzer = useVADAnalysis()
+    const volumeAnalyzer = useVolumeAnalysis()
+    const whisperTranscriber = useWhisperTranscription({ modelSize: 'medium', temperature: 0.0 })
+
+    whisperTranscriber.loadModel()
+
+    // Store the composables FIRST
+    activeStreams.value.set(source.id, stream)
+    vadAnalyzers.value.set(source.id, vadAnalyzer)
+    volumeAnalyzers.value.set(source.id, volumeAnalyzer)
+    whisperTranscribers.value.set(source.id, whisperTranscriber)
+
+    // NOW create the playback with the stream reference
+    const streamRef = computed(() => activeStreams.value.get(source.id)?.mediaStream.value)
+    const playback = useAudioPlayback(streamRef)
+    playbackControllers.value.set(source.id, playback)
+
+    // Connect analyzers
+    stream.addChunkCallback(vadAnalyzer.audioChunkCallback)
+    stream.addAnalysisCallback(volumeAnalyzer.audioAnalysisCallback)
+
+    vadAnalyzer.onSegmentComplete((segment) => {
+      whisperTranscriber.queueSegment(segment, localeRef.value)
+    })
+
+    return source.id
+  }
+
+  function removeSource(sourceId: string) {
+    const stream = activeStreams.value.get(sourceId)
+    if (stream) {
+      stream.stop()
+      activeStreams.value.delete(sourceId)
+    }
+
+    sources.value.delete(sourceId)
+    vadAnalyzers.value.delete(sourceId)
+    volumeAnalyzers.value.delete(sourceId)
+    playbackControllers.value.delete(sourceId)
+  }
+
+  function getSourceData(sourceId: string) {
+    return {
+      source: sources.value.get(sourceId),
+      stream: activeStreams.value.get(sourceId),
+      vad: vadAnalyzers.value.get(sourceId),
+      volume: volumeAnalyzers.value.get(sourceId),
+      playback: playbackControllers.value.get(sourceId),
+      whisper: whisperTranscribers.value.get(sourceId),
+    }
+  }
+
+  async function startSource(sourceId: string) {
+    const stream = activeStreams.value.get(sourceId)
+    const vadAnalyzer = vadAnalyzers.value.get(sourceId)
+
+    if (stream) {
+      await stream.start()
+    }
+
+    if (vadAnalyzer) {
+      vadAnalyzer.loadModel()
+    }
+  }
+
+  async function stopSource(sourceId: string) {
+    const stream = activeStreams.value.get(sourceId)
+    if (stream) {
+      await stream.stop()
+    }
+  }
+
+  // Global audio context controls
+  async function suspendGlobalAudio() {
+    await audioContext?.suspend()
+  }
+
+  async function resumeGlobalAudio() {
+    await audioContext?.resume()
+  }
+
+  async function cleanupGlobalAudio() {
+    // Stop all sources first
+    for (const sourceId of Object.keys(sources.value)) {
+      await stopSource(sourceId)
+    }
+
+    // Cleanup global context
+    await cleanupAudioContext()
+  }
+
+  // Convenience methods
+  function addMicrophone(deviceId: string, name: string = 'Primary Microphone') {
+    return addSource({
+      id: `mic-${deviceId}`,
+      name,
+      deviceId,
+      type: 'microphone',
+      config: {
+        deviceId,
+        sampleRate: 16000,
+        echoCancellation: true,
+        noiseSuppression: false,
+        autoGainControl: false,
+      },
+    })
+  }
+
+  return {
+    // State
+    sources: readonly(sources.value),
+    audioContextState: {
+      isReady: isAudioContextReady,
+      sampleRate: audioContext?.sampleRate ?? 0,
+      error: audioContextState.value.error,
+    },
+
+    // Management
+    addSource,
+    removeSource,
+    startSource,
+    stopSource,
+    getSourceData,
+
+    // Global controls
+    suspendGlobalAudio,
+    resumeGlobalAudio,
+    cleanupGlobalAudio,
+
+    // Convenience
+    addMicrophone,
+  }
+}

apps/stage-tamagotchi/src/composables/tauri.ts

@@ -134,8 +134,8 @@ export interface InvokeMethods {
   'open_chat_window': { args: undefined, options: undefined, returns: void }
 
   // Plugin - Audio Transcription
-  'plugin:proj-airi-tauri-plugin-audio-transcription|load_model_whisper': { args: undefined, options: undefined, returns: void }
-  'plugin:proj-airi-tauri-plugin-audio-transcription|audio_transcription': { args: { chunk: number[] }, options: undefined, returns: string }
+  'plugin:proj-airi-tauri-plugin-audio-transcription|load_model_whisper': { args: { modelType: 'base' | 'largev3' | 'tiny' | 'medium' }, options: undefined, returns: void }
+  'plugin:proj-airi-tauri-plugin-audio-transcription|audio_transcription': { args: { chunk: number[], language: string }, options: undefined, returns: [string, string] }
 
   // Plugin - Audio VAD
   'plugin:proj-airi-tauri-plugin-audio-vad|load_model_silero_vad': { args: undefined, options: undefined, returns: void }

apps/stage-tamagotchi/src/pages/index.vue

@@ -83,7 +83,7 @@ onMounted(async () => {
   }))
 
   // Load models
-  invoke('plugin:proj-airi-tauri-plugin-audio-transcription|load_model_whisper')
+  invoke('plugin:proj-airi-tauri-plugin-audio-transcription|load_model_whisper', { modelType: 'medium' })
   invoke('plugin:proj-airi-tauri-plugin-audio-vad|load_model_silero_vad')
 
   if (connected.value)

apps/stage-tamagotchi/src/pages/settings/modules/hearing.vue

@@ -3,327 +3,84 @@ import { LevelMeter, ThresholdMeter, TimeSeriesChart } from '@proj-airi/stage-ui
 import { FieldCheckbox, FieldRange, FieldSelect } from '@proj-airi/ui'
 import { useDevicesList } from '@vueuse/core'
 import { computed, nextTick, onMounted, onUnmounted, ref, watch } from 'vue'
+import { useI18n } from 'vue-i18n'
 
-import { useTauriCore } from '../../../composables/tauri'
+import { useAudioManager } from '../../../composables/audio/manager'
 
 const devices = useDevicesList({ constraints: { audio: true } })
 const audioInputs = computed(() => devices.audioInputs.value)
-const { invoke } = useTauriCore()
 
-const selectedAudioInput = ref<string>(devices.audioInputs.value[0]?.deviceId || '')
+const i18n = useI18n()
+// Initialize audio manager
+const audioManager = useAudioManager(i18n.locale)
 
-const isMonitoring = ref(false)
-const enablePlayback = ref(false)
+const selectedAudioInput = ref<string>('')
+const selectedAudioInputSourceId = ref<string>('')
 
-// Audio processing state
-const audioContext = ref<AudioContext>()
-const mediaStream = ref<MediaStream>()
-const analyser = ref<AnalyserNode>()
-const gainNode = ref<GainNode>()
-const dataArray = ref<Uint8Array>()
-const animationFrame = ref<number>()
+const enabledMonitoring = ref(false)
+const enabledPlayback = ref(false)
+const monitorVolume = ref(50)
+const useVADModel = ref(true)
 
-// Audio levels and indicators
-const volumeLevel = ref(0) // 0-100
-const isSpeaking = ref(false)
-const speakingThreshold = ref(25) // 0-100 (for volume-based fallback)
-const monitorVolume = ref(50) // 0-100
-
-// Tauri VAD integration
-const isVADModelLoaded = ref(false)
-const isLoadingVADModel = ref(false)
-const vadModelError = ref('')
-const useVADModel = ref(true) // Toggle between Tauri VAD and volume-based detection
-const vadProbability = ref(0) // Raw VAD probability from Tauri
-const vadThreshold = ref(0.5) // VAD probability threshold for speech detection
-
-// Audio chunk buffering for Tauri VAD
-const audioChunkBuffer = ref<Float32Array>(new Float32Array(0))
-const chunkSize = 512 // Exactly 512 samples for 16kHz as expected by VAD model
-const vadProcessingInterval = ref<number | null>(null)
-const sampleRate = 16000 // Fixed sample rate for VAD
-
-// VAD visualization
-const vadHistory = ref<number[]>([]) // History for chart visualization
-const maxVadHistory = 50 // Keep 50 samples (~1.6 seconds at 32ms intervals)
-
-// Tauri VAD functions
-async function loadVADModel() {
-  if (isVADModelLoaded.value || isLoadingVADModel.value)
-    return
-
-  isLoadingVADModel.value = true
-  vadModelError.value = ''
-
-  try {
-    await invoke('plugin:proj-airi-tauri-plugin-audio-vad|load_model_silero_vad')
-    isVADModelLoaded.value = true
-  }
-  catch (error) {
-    vadModelError.value = error as string
-    console.error('Failed to load VAD model:', error)
-  }
-  finally {
-    isLoadingVADModel.value = false
-  }
-}
-
-async function processAudioChunkWithVAD(audioData: Float32Array) {
-  if (!isVADModelLoaded.value)
-    return
-
-  try {
-    // Ensure we have exactly 512 samples as expected by the VAD model
-    if (audioData.length !== chunkSize) {
-      console.warn(`VAD received ${audioData.length} samples, expected ${chunkSize}`)
-      return
-    }
-
-    // Convert Float32Array to regular array for Tauri
-    const chunk = Array.from(audioData)
-    const probability = await invoke('plugin:proj-airi-tauri-plugin-audio-vad|audio_vad', { chunk })
-
-    if (probability != null && typeof probability === 'number') {
-      vadProbability.value = probability
-
-      // Update VAD history for visualization
-      vadHistory.value.push(probability)
-      if (vadHistory.value.length > maxVadHistory) {
-        vadHistory.value.shift()
-      }
-
-      // Update speaking detection based on VAD
-      if (useVADModel.value) {
-        isSpeaking.value = vadProbability.value > vadThreshold.value
-      }
-    }
-  }
-  catch (error) {
-    console.error('VAD processing error:', error)
-    vadModelError.value = error as string
-    // Fall back to volume-based detection on error
-    if (useVADModel.value) {
-      isSpeaking.value = volumeLevel.value > speakingThreshold.value
-    }
-  }
-}
-
-function startVADProcessing() {
-  if (vadProcessingInterval.value)
-    return
-
-  // Process chunks immediately when buffer has enough samples
-  vadProcessingInterval.value = window.setInterval(async () => {
-    if (audioChunkBuffer.value.length >= chunkSize) {
-      // Process the chunk with Tauri VAD (exactly 512 samples)
-      const chunk = audioChunkBuffer.value.slice(0, chunkSize)
-      await processAudioChunkWithVAD(chunk)
-
-      // Remove processed samples from buffer
-      const remaining = audioChunkBuffer.value.slice(chunkSize)
-      audioChunkBuffer.value = remaining.length > 0 ? remaining : new Float32Array(0)
-    }
-  }, 10) // Check every 10ms for available chunks
-}
-
-function stopVADProcessing() {
-  if (vadProcessingInterval.value) {
-    clearInterval(vadProcessingInterval.value)
-    vadProcessingInterval.value = null
-  }
-  audioChunkBuffer.value = new Float32Array(0)
-  vadProbability.value = 0
-  vadHistory.value = []
-}
-
-// Audio monitoring
-async function setupAudioMonitoring() {
-  try {
-    if (!selectedAudioInput.value) {
-      console.warn('No audio input device selected')
-      return
-    }
-
-    // Clean up existing connections
-    await stopAudioMonitoring()
-
-    // Get user media with selected device
-    mediaStream.value = await navigator.mediaDevices.getUserMedia({
-      audio: {
-        deviceId: selectedAudioInput.value,
-        echoCancellation: true,
-        noiseSuppression: true,
-        autoGainControl: true,
-        sampleRate, // Explicitly request 16kHz
-      },
-    })
-
-    // Create audio context with fixed sample rate for VAD
-    audioContext.value = new AudioContext({ sampleRate })
-    const source = audioContext.value.createMediaStreamSource(mediaStream.value)
-
-    // Create analyser for volume detection
-    analyser.value = audioContext.value.createAnalyser()
-    analyser.value.fftSize = 512 // Match our chunk size for better alignment
-    analyser.value.smoothingTimeConstant = 0.1 // Less smoothing for better real-time response
-
-    // Create gain node for playback volume control
-    gainNode.value = audioContext.value.createGain()
-    gainNode.value.gain.value = enablePlayback.value ? (monitorVolume.value / 100) : 0
-
-    // Connect audio graph
-    source.connect(analyser.value)
-
-    if (enablePlayback.value) {
-      source.connect(gainNode.value)
-      gainNode.value.connect(audioContext.value.destination)
-    }
-
-    // Set up data array for analysis
-    const bufferLength = analyser.value.frequencyBinCount
-    dataArray.value = new Uint8Array(bufferLength)
-
-    // Start audio analysis loop
-    startAudioAnalysis()
-
-    // Load VAD model and start VAD processing if enabled
-    if (useVADModel.value) {
-      await loadVADModel()
-      if (isVADModelLoaded.value) {
-        audioChunkBuffer.value = new Float32Array(0)
-        startVADProcessing()
-      }
-    }
-  }
-  catch (error) {
-    console.error('Error setting up audio monitoring:', error)
-  }
-}
-
-async function stopAudioMonitoring() {
-  // Stop animation frame
-  if (animationFrame.value) {
-    cancelAnimationFrame(animationFrame.value)
-    animationFrame.value = undefined
-  }
-
-  // Stop media stream
-  if (mediaStream.value) {
-    mediaStream.value.getTracks().forEach(track => track.stop())
-    mediaStream.value = undefined
-  }
-
-  // Close audio context
-  if (audioContext.value) {
-    await audioContext.value.close()
-    audioContext.value = undefined
-  }
-
-  analyser.value = undefined
-  gainNode.value = undefined
-  dataArray.value = undefined
-  volumeLevel.value = 0
-  isSpeaking.value = false
-
-  // Stop VAD processing
-  stopVADProcessing()
-}
-
-function startAudioAnalysis() {
-  const analyze = () => {
-    if (!analyser.value || !dataArray.value)
-      return
-
-    // Get frequency data for volume visualization
-    analyser.value.getByteFrequencyData(dataArray.value)
-
-    // Calculate RMS volume level
-    let sum = 0
-    for (let i = 0; i < dataArray.value.length; i++) {
-      sum += dataArray.value[i] * dataArray.value[i]
-    }
-    const rms = Math.sqrt(sum / dataArray.value.length)
-    volumeLevel.value = Math.min(100, (rms / 255) * 100 * 3) // Amplify for better visualization
-
-    // Fallback speaking detection (when VAD model is not used)
-    if (!useVADModel.value || !isVADModelLoaded.value) {
-      isSpeaking.value = volumeLevel.value > speakingThreshold.value
-    }
-
-    // Collect audio samples for VAD processing
-    if (useVADModel.value && isVADModelLoaded.value) {
-      // Get time domain data for VAD (raw audio samples)
-      // Use smaller buffer size for more frequent updates
-      const bufferSize = 128 // Smaller chunks for better real-time processing
-      const timeDataArray = new Float32Array(bufferSize)
-      analyser.value.getFloatTimeDomainData(timeDataArray)
-
-      // Append new samples to buffer
-      const currentBuffer = audioChunkBuffer.value
-      const newBuffer = new Float32Array(currentBuffer.length + timeDataArray.length)
-      newBuffer.set(currentBuffer, 0)
-      newBuffer.set(timeDataArray, currentBuffer.length)
-      audioChunkBuffer.value = newBuffer
-    }
-
-    animationFrame.value = requestAnimationFrame(analyze)
-  }
-  analyze()
-}
-
-// Update playback routing when playback setting changes
-async function updatePlayback() {
-  if (!audioContext.value || !gainNode.value)
-    return
-
-  if (enablePlayback.value) {
-    gainNode.value.gain.value = monitorVolume.value / 100
-    gainNode.value.connect(audioContext.value.destination)
-  }
-  else {
-    gainNode.value.gain.value = 0
-    gainNode.value.disconnect()
-  }
-}
-
-// Watchers
-watch(selectedAudioInput, async () => {
-  if (isMonitoring.value) {
-    await setupAudioMonitoring()
-  }
+// Get current source data reactively
+const currentSource = computed(() => {
+  return selectedAudioInputSourceId.value ? audioManager.getSourceData(selectedAudioInputSourceId.value) : null
 })
 
-watch(enablePlayback, updatePlayback)
-watch(monitorVolume, () => {
-  if (gainNode.value && enablePlayback.value) {
-    gainNode.value.gain.value = monitorVolume.value / 100
-  }
+// Extract reactive values from the current source
+const volumeLevel = computed(() => currentSource.value?.volume?.level.value ?? 0)
+const vadProbability = computed(() => currentSource.value?.vad?.probability.value ?? 0)
+const vadThreshold = computed({
+  get: () => currentSource.value?.vad?.config.value.threshold ?? 0.5,
+  set: (value) => {
+    if (currentSource.value?.vad?.config.value.threshold) {
+      currentSource.value.vad.config.value.threshold = value
+    }
+  },
 })
 
-watch(audioInputs, () => {
-  if (!selectedAudioInput.value && audioInputs.value.length > 0) {
-    selectedAudioInput.value = audioInputs.value[0]?.deviceId
-  }
+const speakingThreshold = computed({
+  get: () => currentSource.value?.volume?.threshold.value ?? 25,
+  set: (value) => {
+    if (currentSource.value?.volume?.threshold) {
+      currentSource.value.volume.threshold.value = value
+    }
+  },
 })
 
-watch(selectedAudioInput, async () => {
-  if (isMonitoring.value) {
-    await stopAudioMonitoring()
-    await setupAudioMonitoring()
+const isVADModelLoaded = computed(() => currentSource.value?.vad?.isModelLoaded.value ?? false)
+const isLoadingVADModel = computed(() => currentSource.value?.vad?.isLoading.value ?? false)
+const vadModelError = computed(() => currentSource.value?.vad?.error.value ?? '')
+const vadHistory = computed(() => currentSource.value?.vad?.history.value ?? [])
+
+// Speaking detection - prioritize VAD if enabled and loaded
+const isSpeaking = computed(() => {
+  if (useVADModel.value && isVADModelLoaded.value) {
+    return currentSource.value?.vad?.isSpeaking.value ?? false
   }
+  return currentSource.value?.volume?.isSpeaking.value ?? false
 })
 
-// Monitoring toggle
-async function toggleMonitoring() {
-  if (isMonitoring.value) {
-    await setupAudioMonitoring()
-  }
-  else {
-    await stopAudioMonitoring()
-  }
-}
+// Playback controls
+const playbackEnabled = computed({
+  get: () => currentSource.value?.playback?.isEnabled.value ?? false,
+  set: (value) => {
+    if (currentSource.value?.playback?.isEnabled) {
+      currentSource.value.playback.isEnabled.value = value
+    }
+  },
+})
 
-// Speaking indicator with enhanced VAD visualization
+const playbackVolume = computed({
+  get: () => currentSource.value?.playback?.volume.value ?? 50,
+  set: (value) => {
+    if (currentSource.value?.playback?.volume) {
+      currentSource.value.playback.volume.value = value
+    }
+  },
+})
+
+// Speaking indicator styling
 const speakingIndicatorClass = computed(() => {
   if (!useVADModel.value || !isVADModelLoaded.value) {
     // Volume-based: simple green/white
@@ -337,30 +94,80 @@ const speakingIndicatorClass = computed(() => {
   const threshold = vadThreshold.value
 
   if (prob > threshold) {
-    // Speaking: green (could add intensity in future)
-    return `bg-green-500 shadow-lg shadow-green-500/50`
+    return 'bg-green-500 shadow-lg shadow-green-500/50'
   }
   else if (prob > threshold * 0.5) {
-    // Close to threshold: yellow
     return 'bg-yellow-500 shadow-lg shadow-yellow-500/30'
   }
   else {
-    // Low probability: neutral
     return 'bg-white dark:bg-neutral-900 border-2 border-neutral-300 dark:border-neutral-600'
   }
 })
 
-// Lifecycle
-onMounted(() => {
-  devices.ensurePermissions().then(() => nextTick()).then(() => {
-    if (audioInputs.value.length > 0 && !selectedAudioInput.value) {
-      selectedAudioInput.value = audioInputs.value[0]?.deviceId
-    }
-  })
+// Setup primary microphone source when device changes
+watch(selectedAudioInput, async (newDeviceId) => {
+  // Remove existing source
+  if (selectedAudioInputSourceId.value) {
+    await audioManager.stopSource(selectedAudioInputSourceId.value)
+    audioManager.removeSource(selectedAudioInputSourceId.value)
+    selectedAudioInputSourceId.value = ''
+  }
+
+  // Add new source if device selected
+  if (newDeviceId) {
+    const device = audioInputs.value.find(d => d.deviceId === newDeviceId)
+    selectedAudioInputSourceId.value = audioManager.addMicrophone(
+      newDeviceId,
+      device?.label || 'Primary Microphone',
+    )
+  }
 })
 
-onUnmounted(() => {
-  stopAudioMonitoring()
+// Watch for VAD model toggle
+watch([useVADModel, currentSource], ([enabled, source]) => {
+  if (source?.vad?.isEnabled) {
+    source.vad.isEnabled.value = enabled
+  }
+})
+
+// Sync playback settings
+watch(enabledPlayback, (enabled) => {
+  playbackEnabled.value = enabled
+})
+
+watch(monitorVolume, (volume) => {
+  playbackVolume.value = volume
+})
+
+// Monitoring toggle
+async function toggleMonitoring() {
+  if (!selectedAudioInputSourceId.value)
+    return
+
+  if (enabledMonitoring.value) {
+    await audioManager.startSource(selectedAudioInputSourceId.value)
+  }
+  else {
+    await audioManager.stopSource(selectedAudioInputSourceId.value)
+  }
+}
+
+// Initialize with first available device
+onMounted(async () => {
+  await devices.ensurePermissions()
+  await nextTick()
+
+  if (audioInputs.value.length > 0 && !selectedAudioInput.value) {
+    selectedAudioInput.value = audioInputs.value[0]?.deviceId
+  }
+})
+
+// Cleanup on unmount
+onUnmounted(async () => {
+  if (selectedAudioInputSourceId.value) {
+    await audioManager.stopSource(selectedAudioInputSourceId.value)
+    audioManager.removeSource(selectedAudioInputSourceId.value)
+  }
 })
 </script>
 
@@ -369,14 +176,11 @@ onUnmounted(() => {
     <!-- Audio Input Selection -->
     <div>
       <FieldSelect
-        v-model="selectedAudioInput"
-        label="Audio Input Device"
-        description="Select the audio input device for your hearing module."
-        :options="audioInputs.map(input => ({
+        v-model="selectedAudioInput" label="Audio Input Device"
+        description="Select the audio input device for your hearing module." :options="audioInputs.map(input => ({
           label: input.label || input.deviceId,
           value: input.deviceId,
-        }))"
-        placeholder="Select an audio input device"
+        }))" placeholder="Select an audio input device"
       />
     </div>
 
@@ -390,59 +194,43 @@ onUnmounted(() => {
       <div class="space-y-4">
         <!-- Start/Stop Monitoring -->
         <FieldCheckbox
-          v-model="isMonitoring"
-          label="Enable Audio Monitoring"
+          v-model="enabledMonitoring" label="Enable Audio Monitoring"
           description="Start monitoring audio input levels and voice activity detection"
           @update:model-value="toggleMonitoring"
         />
 
         <!-- Audio Level Visualization -->
-        <div v-if="isMonitoring" class="space-y-3">
+        <div v-if="enabledMonitoring && currentSource" class="space-y-3">
           <!-- Volume Meter -->
           <LevelMeter :level="volumeLevel" label="Input Level" />
 
           <!-- VAD Probability Meter (when VAD model is active) -->
           <ThresholdMeter
-            v-if="useVADModel && isVADModelLoaded"
-            :value="vadProbability"
-            :threshold="vadThreshold"
-            label="Probability of Speech"
-            below-label="Silence"
-            above-label="Speech"
+            v-if="useVADModel && isVADModelLoaded" :value="vadProbability" :threshold="vadThreshold"
+            label="Probability of Speech" below-label="Silence" above-label="Speech"
             threshold-label="Detection threshold"
           />
 
           <!-- Threshold Controls -->
           <div v-if="useVADModel && isVADModelLoaded" class="space-y-3">
             <FieldRange
-              v-model="vadThreshold"
-              label="Sensitivity"
-              description="Adjust the threshold for speech detection"
-              :min="0.1"
-              :max="0.9"
-              :step="0.05"
+              v-model="vadThreshold" label="Sensitivity"
+              description="Adjust the threshold for speech detection" :min="0.1" :max="0.9" :step="0.0001"
               :format-value="value => `${(value * 100).toFixed(0)}%`"
             />
           </div>
 
           <div v-else class="space-y-3">
             <FieldRange
-              v-model="speakingThreshold"
-              label="Sensitivity"
-              description="Adjust the threshold for speech detection"
-              :min="1"
-              :max="80"
-              :step="1"
+              v-model="speakingThreshold" label="Sensitivity"
+              description="Adjust the threshold for speech detection" :min="1" :max="80" :step="1"
               :format-value="value => `${value}%`"
             />
           </div>
 
           <!-- Speaking Indicator -->
           <div class="flex items-center gap-3">
-            <div
-              class="h-4 w-4 rounded-full transition-all duration-200"
-              :class="speakingIndicatorClass"
-            />
+            <div class="h-4 w-4 rounded-full transition-all duration-200" :class="speakingIndicatorClass" />
             <span class="text-sm font-medium">
               {{ isSpeaking ? 'Speaking Detected' : 'Silence' }}
             </span>
@@ -454,8 +242,7 @@ onUnmounted(() => {
           <!-- VAD Method Selection -->
           <div class="border-t border-neutral-200 pt-3 dark:border-neutral-700">
             <FieldCheckbox
-              v-model="useVADModel"
-              label="Model Based"
+              v-model="useVADModel" label="Model Based"
               description="Use AI models for more accurate speech detection"
             />
 
@@ -466,7 +253,10 @@ onUnmounted(() => {
                 <span class="text-sm">Loading...</span>
               </div>
 
-              <div v-else-if="vadModelError" class="flex items-center gap-2 whitespace-break-spaces break-anywhere text-red-600 dark:text-red-400">
+              <div
+                v-else-if="vadModelError"
+                class="flex items-center gap-2 whitespace-break-spaces break-anywhere text-red-600 dark:text-red-400"
+              >
                 <span class="text-sm">Inference error: {{ vadModelError }}</span>
               </div>
 
@@ -482,43 +272,34 @@ onUnmounted(() => {
 
           <!-- Voice Activity Visualization (when VAD model is active) -->
           <TimeSeriesChart
-            v-if="useVADModel && isVADModelLoaded"
-            :history="vadHistory"
-            :current-value="vadProbability"
-            :threshold="vadThreshold"
-            :is-active="isSpeaking"
-            title="Voice Activity"
-            subtitle="Last 2 seconds"
-            active-label="Speaking"
-            active-legend-label="Voice detected"
-            inactive-legend-label="Silence"
+            v-if="useVADModel && isVADModelLoaded" :history="vadHistory" :current-value="vadProbability"
+            :threshold="vadThreshold" :is-active="isSpeaking" title="Voice Activity" subtitle="Last 2 seconds"
+            active-label="Speaking" active-legend-label="Voice detected" inactive-legend-label="Silence"
             threshold-label="Speech threshold"
           />
         </div>
 
         <!-- Audio Playback (Monitor) -->
-        <div v-if="isMonitoring" class="border-t border-neutral-200 pt-4 dark:border-neutral-700">
+        <div v-if="enabledMonitoring && currentSource" class="border-t border-neutral-200 pt-4 dark:border-neutral-700">
           <FieldCheckbox
-            v-model="enablePlayback"
-            label="Monitor Audio (Listen)"
+            v-model="enabledPlayback" label="Monitor Audio (Listen)"
             description="Enable audio playback monitoring (like OBS). Be careful of feedback!"
           />
 
-          <div v-if="enablePlayback" class="mt-3">
+          <div v-if="enabledPlayback" class="mt-3">
             <FieldRange
-              v-model="monitorVolume"
-              label="Monitor Volume"
-              description="Control the volume of audio monitoring playback"
-              :min="0"
-              :max="100"
-              :step="5"
+              v-model="monitorVolume" label="Monitor Volume"
+              description="Control the volume of audio monitoring playback" :min="0" :max="100" :step="5"
               :format-value="value => `${value}%`"
             />
           </div>
         </div>
 
         <!-- Warning for playback -->
-        <div v-if="enablePlayback" class="border border-amber-200 rounded-lg bg-amber-50 p-3 dark:border-amber-800 dark:bg-amber-900/20">
+        <div
+          v-if="enabledPlayback"
+          class="border border-amber-200 rounded-lg bg-amber-50 p-3 dark:border-amber-800 dark:bg-amber-900/20"
+        >
           <div class="flex items-center gap-2 text-amber-700 dark:text-amber-300">
             <div class="text-sm" i-solar:warning-circle-bold-duotone />
             <span class="text-sm font-medium">Audio feedback warning</span>

cspell.config.yaml

@@ -8,6 +8,7 @@ words:
   - airi
   - airi-vtuber
   - Alaya
+  - alexanderolsen
   - alibabacloud
   - aliyun
   - allseto
@@ -98,6 +99,7 @@ words:
   - Keyyable
   - kwaa
   - lemonnekogh
+  - libsamplerate
   - libsodium
   - lightningcss
   - listhen
@@ -212,6 +214,7 @@ words:
   - wgpu
   - wlipsync
   - worklet
+  - worklets
   - xast
   - xastscript
   - Xenova

packages/audio/src/vue/audio-stream.ts

@@ -158,6 +158,11 @@ export function useAudioStream(cfg: MaybeRefOrGetter<AudioStreamConfig | undefin
           console.error('Audio chunk callback error:', err)
         }
       })
+
+      // Only continue if still active
+      if (isActive.value) {
+        requestAnimationFrame(analyze)
+      }
     }
 
     analyze()
@@ -174,6 +179,10 @@ export function useAudioStream(cfg: MaybeRefOrGetter<AudioStreamConfig | undefin
     return () => chunkCallbacks.delete(callback)
   }
 
+  function stop() {
+    mediaStream.value?.getTracks().forEach(track => track.stop())
+  }
+
   // Cleanup
   onUnmounted(() => {
     stop()

packages/stage-ui/src/components/Gadgets/ThresholdMeter.vue

@@ -71,7 +71,7 @@ const thresholdBars = computed(() => {
 
     <!-- Threshold Bars -->
     <div
-      class="flex items-end gap-1 rounded bg-neutral-200 p-1 dark:bg-neutral-700"
+      class="flex items-end gap-1 rounded bg-neutral-200/45 p-1 p-1 dark:bg-neutral-700"
       :style="{ height: `${height}px` }"
     >
       <div
@@ -94,15 +94,15 @@ const thresholdBars = computed(() => {
 
     <div v-if="showLegend" class="mt-1 flex gap-3 text-xs text-neutral-500">
       <span class="flex items-center gap-1">
-        <div :class="`inline-block h-1 w-2 ${belowThresholdClass}`" />
+        <div :class="`inline-block h-0.5lh w-1lh rounded-full ${belowThresholdClass}`" />
         {{ belowLabel }}
       </span>
       <span class="flex items-center gap-1">
-        <div :class="`inline-block h-1 w-2 border border-neutral-400 ${thresholdBarClass}`" />
+        <div :class="`inline-block h-0.5lh w-1lh rounded-full border border-neutral-400 ${thresholdBarClass}`" />
         {{ thresholdLabel }}
       </span>
       <span class="flex items-center gap-1">
-        <div :class="`inline-block h-1 w-2 ${aboveThresholdClass}`" />
+        <div :class="`inline-block h-0.5lh w-1lh rounded-full ${aboveThresholdClass}`" />
         {{ aboveLabel }}
       </span>
     </div>

packages/stage-ui/src/components/Gadgets/TimeSeriesChart.vue

@@ -9,7 +9,7 @@ import { chromaticHue as hue } from '../../constants'
 import { chromaticHueDefault as hueDefault } from '../../constants/theme'
 
 interface Props {
-  history: number[] // Array of values (normalized 0-1)
+  history: Readonly<number[]> // Array of values (normalized 0-1)
   currentValue: number // Current value (0-1)
   threshold?: number | null // Threshold value (0-1)
   isActive: boolean // Whether current state is "active"