From e2ab28cdf3b87565ae9c16d5b1370affba707476 Mon Sep 17 00:00:00 2001 From: Neko Ayaka Date: Thu, 3 Jul 2025 03:46:00 +0800 Subject: [PATCH] feat(stage-tamagotchi): realtime whisper --- .../src-tauri/src/app/models/mod.rs | 13 +- .../src/plugins/audio_transcription/mod.rs | 14 +- .../src/composables/audio/analysis-vad.ts | 397 +++++++++++++ .../src/composables/audio/analysis-volume.ts | 46 ++ .../audio/extract-whisper-languages.ts | 112 ++++ .../src/composables/audio/extract-whisper.ts | 206 +++++++ .../src/composables/audio/manager.ts | 171 ++++++ .../stage-tamagotchi/src/composables/tauri.ts | 4 +- apps/stage-tamagotchi/src/pages/index.vue | 2 +- .../src/pages/settings/modules/hearing.vue | 523 +++++------------- cspell.config.yaml | 3 + packages/audio/src/vue/audio-stream.ts | 9 + .../src/components/Gadgets/ThresholdMeter.vue | 8 +- .../components/Gadgets/TimeSeriesChart.vue | 2 +- 14 files changed, 1127 insertions(+), 383 deletions(-) create mode 100644 apps/stage-tamagotchi/src/composables/audio/analysis-vad.ts create mode 100644 apps/stage-tamagotchi/src/composables/audio/analysis-volume.ts create mode 100644 apps/stage-tamagotchi/src/composables/audio/extract-whisper-languages.ts create mode 100644 apps/stage-tamagotchi/src/composables/audio/extract-whisper.ts create mode 100644 apps/stage-tamagotchi/src/composables/audio/manager.ts diff --git a/apps/stage-tamagotchi/src-tauri/src/app/models/mod.rs b/apps/stage-tamagotchi/src-tauri/src/app/models/mod.rs index 2e226ecf2..0ecac62d5 100644 --- a/apps/stage-tamagotchi/src-tauri/src/app/models/mod.rs +++ b/apps/stage-tamagotchi/src-tauri/src/app/models/mod.rs @@ -8,10 +8,19 @@ use tauri::Runtime; use crate::{app::models::whisper::WhichWhisperModel, helpers::huggingface::load_device}; pub fn new_whisper_processor( - window: tauri::WebviewWindow + window: tauri::WebviewWindow, + model_type: Option, ) -> anyhow::Result { let device = load_device().map_err(|err| anyhow::anyhow!("Failed to load device: {}", err))?; - let whisper_model = WhichWhisperModel::Tiny; + let whisper_model = model_type.unwrap_or_else(|| { + if device.is_cuda() { + WhichWhisperModel::LargeV3 + } else if device.is_metal() { + WhichWhisperModel::Base + } else { + WhichWhisperModel::Tiny + } + }); info!("Loading whisper model: {:?}", whisper_model); whisper::Processor::new(whisper_model, device.clone(), window) } diff --git a/apps/stage-tamagotchi/src-tauri/src/plugins/audio_transcription/mod.rs b/apps/stage-tamagotchi/src-tauri/src/plugins/audio_transcription/mod.rs index 172b7f26a..c6c4fbf39 100644 --- a/apps/stage-tamagotchi/src-tauri/src/plugins/audio_transcription/mod.rs +++ b/apps/stage-tamagotchi/src-tauri/src/plugins/audio_transcription/mod.rs @@ -1,5 +1,6 @@ use std::sync::Mutex; +use clap::ValueEnum; use log::info; use tauri::{ Manager, @@ -7,7 +8,7 @@ use tauri::{ plugin::{Builder as PluginBuilder, TauriPlugin}, }; -use crate::app::models::new_whisper_processor; +use crate::app::models::{new_whisper_processor, whisper::WhichWhisperModel}; #[derive(Default)] struct AppDataWhisperProcessor { @@ -18,6 +19,7 @@ struct AppDataWhisperProcessor { pub async fn load_model_whisper( app: tauri::AppHandle, window: tauri::WebviewWindow, + model_type: Option, ) -> Result<(), String> { info!("Loading models..."); @@ -31,7 +33,15 @@ pub async fn load_model_whisper( } // Load the traditional whisper models first - match new_whisper_processor(window) { + match new_whisper_processor( + window, + Some(WhichWhisperModel::from_str( + model_type + .unwrap_or_else(|| "medium".to_string()) + .as_str(), + true, + )?), + ) { Ok(p) => { let data = app.state::>(); let mut data = data.lock().unwrap(); diff --git a/apps/stage-tamagotchi/src/composables/audio/analysis-vad.ts b/apps/stage-tamagotchi/src/composables/audio/analysis-vad.ts new file mode 100644 index 000000000..eaffaf850 --- /dev/null +++ b/apps/stage-tamagotchi/src/composables/audio/analysis-vad.ts @@ -0,0 +1,397 @@ +import type { AudioChunkCallback } from '@proj-airi/audio/vue' + +import { onUnmounted, readonly, ref, watch } from 'vue' + +import { useTauriCore } from '../tauri' + +export interface VADSegment { + id: string + audioData: Float32Array + startTime: number + endTime: number + probability: number + isComplete: boolean +} + +export interface VADConfig { + threshold: number + silenceGapMs: number // How long silence before ending segment + minSpeechDurationMs: number // Minimum speech duration to be valid + maxSpeechDurationMs: number // Maximum speech duration before forced split + overlapMs: number // Overlap between segments for better transcription + bufferSizeMs: number // How much audio to keep in memory +} + +export function useVADAnalysis(config: Partial = {}) { + const { invoke } = useTauriCore() + + const isModelLoaded = ref(false) + const isLoading = ref(false) + const error = ref('') + const isEnabled = ref(true) + + const probability = ref(0) + const history = ref([]) + const maxHistory = 50 + + // Configuration with defaults + const vadConfig = ref({ + threshold: 0.5, + silenceGapMs: 500, // 500ms of silence to end segment + minSpeechDurationMs: 300, // Minimum 300ms speech + maxSpeechDurationMs: 30000, // Max 30s per segment + overlapMs: 200, // 200ms overlap + bufferSizeMs: 60000, // Keep 60s of audio in memory + ...config, + }) + + // Audio buffering + const audioBuffer = ref(new Float32Array(0)) + const chunkSize = 512 + const targetSampleRate = 16000 + let processingInterval: number | null = null + + // Segment management + const currentSegment = ref(null) + const completedSegments = ref([]) + const maxCompletedSegments = 10 // Keep last 10 completed segments + + // State tracking + const isSpeaking = ref(false) + const lastSpeechTime = ref(0) + const lastSilenceTime = ref(0) + const segmentStartTime = ref(0) + const segmentAudioBuffer = ref(new Float32Array(0)) + + // Callbacks for segment events + const segmentCallbacks = new Set<(segment: VADSegment) => void>() + + async function loadModel() { + if (isModelLoaded.value || isLoading.value) + return + + isLoading.value = true + error.value = '' + + try { + await invoke('plugin:proj-airi-tauri-plugin-audio-vad|load_model_silero_vad') + isModelLoaded.value = true + } + catch (err) { + error.value = err instanceof Error ? err.message : String(err) + console.error('Failed to load VAD model:', err) + } + finally { + isLoading.value = false + } + } + + function resampleIfNeeded(chunk: Float32Array, sourceSampleRate: number): Float32Array { + if (sourceSampleRate === targetSampleRate) { + return chunk + } + + const ratio = targetSampleRate / sourceSampleRate + const outputLength = Math.floor(chunk.length * ratio) + const resampled = new Float32Array(outputLength) + + for (let i = 0; i < outputLength; i++) { + const sourceIndex = i / ratio + const index = Math.floor(sourceIndex) + const fraction = sourceIndex - index + + if (index + 1 < chunk.length) { + resampled[i] = chunk[index] * (1 - fraction) + chunk[index + 1] * fraction + } + else { + resampled[i] = chunk[index] || 0 + } + } + + return resampled + } + + async function processChunk(chunk: Float32Array) { + if (!isModelLoaded.value || chunk.length !== chunkSize) + return + + try { + const chunkArray = Array.from(chunk) + const prob = await invoke('plugin:proj-airi-tauri-plugin-audio-vad|audio_vad', { + chunk: chunkArray, + }) + + if (typeof prob === 'number') { + const now = performance.now() + probability.value = prob + + // Update history + history.value.push(prob) + if (history.value.length > maxHistory) { + history.value.shift() + } + + // Process speech detection with hysteresis + const wasSpeaking = isSpeaking.value + const currentlySpeaking = prob > vadConfig.value.threshold + + if (currentlySpeaking) { + lastSpeechTime.value = now + } + else { + lastSilenceTime.value = now + } + + // Handle speech state transitions + if (!wasSpeaking && currentlySpeaking) { + // Start of speech + startSpeechSegment(now, chunk) + isSpeaking.value = true + } + else if (wasSpeaking && !currentlySpeaking) { + // Potential end of speech - wait for silence gap + const silenceDuration = now - lastSpeechTime.value + if (silenceDuration >= vadConfig.value.silenceGapMs) { + await endSpeechSegment(now) + isSpeaking.value = false + } + } + else if (wasSpeaking && currentlySpeaking) { + // Continuing speech - add to current segment + addToCurrentSegment(chunk) + + // Check for max duration + const segmentDuration = now - segmentStartTime.value + if (segmentDuration >= vadConfig.value.maxSpeechDurationMs) { + await endSpeechSegment(now, true) // Force end + startSpeechSegment(now, chunk) // Start new segment + } + } + + // Always add audio to segment buffer when speaking or recently speaking + const timeSinceLastSpeech = now - lastSpeechTime.value + if (timeSinceLastSpeech <= vadConfig.value.silenceGapMs + vadConfig.value.overlapMs) { + addToCurrentSegment(chunk) + } + } + } + catch (err) { + error.value = err instanceof Error ? err.message : String(err) + console.error('VAD processing error:', err) + } + } + + function startSpeechSegment(timestamp: number, initialChunk: Float32Array) { + segmentStartTime.value = timestamp + + // Include some pre-speech audio for context (overlap) + const overlapSamples = Math.floor((vadConfig.value.overlapMs / 1000) * targetSampleRate) + const totalBufferLength = audioBuffer.value.length + const startIndex = Math.max(0, totalBufferLength - overlapSamples) + + const preAudio = audioBuffer.value.slice(startIndex) + segmentAudioBuffer.value = new Float32Array(preAudio.length + initialChunk.length) + segmentAudioBuffer.value.set(preAudio, 0) + segmentAudioBuffer.value.set(initialChunk, preAudio.length) + + currentSegment.value = { + id: `segment_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`, + audioData: new Float32Array(0), // Will be set when segment ends + startTime: timestamp, + endTime: 0, + probability: probability.value, + isComplete: false, + } + } + + function addToCurrentSegment(chunk: Float32Array) { + if (!currentSegment.value) + return + + // Append to segment buffer + const oldBuffer = segmentAudioBuffer.value + const newBuffer = new Float32Array(oldBuffer.length + chunk.length) + newBuffer.set(oldBuffer, 0) + newBuffer.set(chunk, oldBuffer.length) + segmentAudioBuffer.value = newBuffer + } + + async function endSpeechSegment(timestamp: number, forced = false) { + if (!currentSegment.value) + return + + const segmentDuration = timestamp - segmentStartTime.value + + // Check minimum duration requirement + if (!forced && segmentDuration < vadConfig.value.minSpeechDurationMs) { + // Too short, discard + currentSegment.value = null + segmentAudioBuffer.value = new Float32Array(0) + return + } + + // Add some post-speech audio for context + const overlapSamples = Math.floor((vadConfig.value.overlapMs / 1000) * targetSampleRate) + const postAudioLength = Math.min(overlapSamples, audioBuffer.value.length) + const postAudio = audioBuffer.value.slice(-postAudioLength) + + // Final segment audio + const finalAudioBuffer = new Float32Array(segmentAudioBuffer.value.length + postAudio.length) + finalAudioBuffer.set(segmentAudioBuffer.value, 0) + finalAudioBuffer.set(postAudio, segmentAudioBuffer.value.length) + + // Complete the segment + const completedSegment: VADSegment = { + ...currentSegment.value, + audioData: finalAudioBuffer, + endTime: timestamp, + isComplete: true, + } + + // Add to completed segments + completedSegments.value.push(completedSegment) + if (completedSegments.value.length > maxCompletedSegments) { + completedSegments.value.shift() + } + + // Notify callbacks + segmentCallbacks.forEach((callback) => { + try { + callback(completedSegment) + } + catch (err) { + console.error('Segment callback error:', err) + } + }) + + // Reset for next segment + currentSegment.value = null + segmentAudioBuffer.value = new Float32Array(0) + } + + function startProcessing() { + if (processingInterval) + return + + const intervalMs = (chunkSize / targetSampleRate) * 1000 + + processingInterval = window.setInterval(async () => { + if (audioBuffer.value.length >= chunkSize) { + const chunk = audioBuffer.value.slice(0, chunkSize) + await processChunk(chunk) + + const remaining = audioBuffer.value.slice(chunkSize) + audioBuffer.value = remaining.length > 0 ? remaining : new Float32Array(0) + } + }, Math.max(1, intervalMs / 2)) + } + + function stopProcessing() { + if (processingInterval) { + clearInterval(processingInterval) + processingInterval = null + } + + // End current segment if active + if (currentSegment.value) { + endSpeechSegment(performance.now(), true) + } + + audioBuffer.value = new Float32Array(0) + probability.value = 0 + history.value = [] + } + + // Audio chunk callback + const audioChunkCallback: AudioChunkCallback = (chunk: Float32Array, sampleRate: number) => { + if (!isEnabled.value || !isModelLoaded.value) + return + + const processedChunk = resampleIfNeeded(chunk, sampleRate) + + // Add to main buffer + const currentBuffer = audioBuffer.value + const newBuffer = new Float32Array(currentBuffer.length + processedChunk.length) + newBuffer.set(currentBuffer, 0) + newBuffer.set(processedChunk, currentBuffer.length) + + // Trim buffer to max size + const maxBufferSamples = (vadConfig.value.bufferSizeMs / 1000) * targetSampleRate + if (newBuffer.length > maxBufferSamples) { + const trimAmount = newBuffer.length - maxBufferSamples + audioBuffer.value = newBuffer.slice(trimAmount) + } + else { + audioBuffer.value = newBuffer + } + } + + // Segment callback management + function onSegmentComplete(callback: (segment: VADSegment) => void) { + segmentCallbacks.add(callback) + return () => segmentCallbacks.delete(callback) + } + + // Manual segment control + async function forceEndCurrentSegment() { + if (currentSegment.value) { + await endSpeechSegment(performance.now(), true) + isSpeaking.value = false + } + } + + function clearCompletedSegments() { + completedSegments.value = [] + } + + // Get segment by ID + function getSegment(id: string): VADSegment | undefined { + return completedSegments.value.find(s => s.id === id) + } + + // Auto-start processing when enabled and model is loaded + watch([isEnabled, isModelLoaded], ([enabled, loaded]) => { + if (enabled && loaded) { + startProcessing() + } + else { + stopProcessing() + } + }) + + onUnmounted(() => { + stopProcessing() + }) + + return { + // State + isModelLoaded: readonly(isModelLoaded), + isLoading: readonly(isLoading), + error: readonly(error), + isEnabled, + + // VAD data + probability: readonly(probability), + history: readonly(history), + isSpeaking: readonly(isSpeaking), + + // Configuration + config: vadConfig, + + // Segments + currentSegment: readonly(currentSegment), + completedSegments: readonly(completedSegments), + + // Controls + loadModel, + forceEndCurrentSegment, + clearCompletedSegments, + getSegment, + + // Events + onSegmentComplete, + + // For audio stream integration + audioChunkCallback, + } +} diff --git a/apps/stage-tamagotchi/src/composables/audio/analysis-volume.ts b/apps/stage-tamagotchi/src/composables/audio/analysis-volume.ts new file mode 100644 index 000000000..3ac76cb9c --- /dev/null +++ b/apps/stage-tamagotchi/src/composables/audio/analysis-volume.ts @@ -0,0 +1,46 @@ +import type { AudioAnalysisCallback } from '@proj-airi/audio/vue' + +import { computed, readonly, ref } from 'vue' + +export function useVolumeAnalysis() { + const level = ref(0) + const threshold = ref(25) + const history = ref([]) + const maxHistory = 100 + const isEnabled = ref(true) + + const isSpeaking = computed(() => level.value > threshold.value) + + const audioAnalysisCallback: AudioAnalysisCallback = (data) => { + if (!isEnabled.value) + return + + level.value = data.volumeLevel + + // Update history + history.value.push(data.volumeLevel) + if (history.value.length > maxHistory) { + history.value.shift() + } + } + + function reset() { + level.value = 0 + history.value = [] + } + + return { + // State + level: readonly(level), + threshold, + history: readonly(history), + isSpeaking, + isEnabled, + + // Controls + reset, + + // For audio stream integration + audioAnalysisCallback, + } +} diff --git a/apps/stage-tamagotchi/src/composables/audio/extract-whisper-languages.ts b/apps/stage-tamagotchi/src/composables/audio/extract-whisper-languages.ts new file mode 100644 index 000000000..b36c72730 --- /dev/null +++ b/apps/stage-tamagotchi/src/composables/audio/extract-whisper-languages.ts @@ -0,0 +1,112 @@ +// Language constants +const LANGUAGES: Record[] = [ + { en: 'english' }, + { zh: 'chinese' }, + { 'zh-Hans': 'chinese' }, + { 'zh-Hant': 'chinese' }, + { 'zh-CN': 'chinese' }, + { 'zh-TW': 'chinese' }, + { 'zh-HK': 'chinese' }, + { de: 'german' }, + { es: 'spanish' }, + { ru: 'russian' }, + { ko: 'korean' }, + { fr: 'french' }, + { ja: 'japanese' }, + { pt: 'portuguese' }, + { tr: 'turkish' }, + { pl: 'polish' }, + { ca: 'catalan' }, + { nl: 'dutch' }, + { ar: 'arabic' }, + { sv: 'swedish' }, + { it: 'italian' }, + { id: 'indonesian' }, + { hi: 'hindi' }, + { fi: 'finnish' }, + { vi: 'vietnamese' }, + { he: 'hebrew' }, + { uk: 'ukrainian' }, + { el: 'greek' }, + { ms: 'malay' }, + { cs: 'czech' }, + { ro: 'romanian' }, + { da: 'danish' }, + { hu: 'hungarian' }, + { ta: 'tamil' }, + { no: 'norwegian' }, + { th: 'thai' }, + { ur: 'urdu' }, + { hr: 'croatian' }, + { bg: 'bulgarian' }, + { lt: 'lithuanian' }, + { la: 'latin' }, + { mi: 'maori' }, + { ml: 'malayalam' }, + { cy: 'welsh' }, + { sk: 'slovak' }, + { te: 'telugu' }, + { fa: 'persian' }, + { lv: 'latvian' }, + { bn: 'bengali' }, + { sr: 'serbian' }, + { az: 'azerbaijani' }, + { sl: 'slovenian' }, + { kn: 'kannada' }, + { et: 'estonian' }, + { mk: 'macedonian' }, + { br: 'breton' }, + { eu: 'basque' }, + { is: 'icelandic' }, + { hy: 'armenian' }, + { ne: 'nepali' }, + { mn: 'mongolian' }, + { bs: 'bosnian' }, + { kk: 'kazakh' }, + { sq: 'albanian' }, + { sw: 'swahili' }, + { gl: 'galician' }, + { mr: 'marathi' }, + { pa: 'punjabi' }, + { si: 'sinhala' }, + { km: 'khmer' }, + { sn: 'shona' }, + { yo: 'yoruba' }, + { so: 'somali' }, + { af: 'afrikaans' }, + { oc: 'occitan' }, + { ka: 'georgian' }, + { be: 'belarusian' }, + { tg: 'tajik' }, + { sd: 'sindhi' }, + { gu: 'gujarati' }, + { am: 'amharic' }, + { yi: 'yiddish' }, + { lo: 'lao' }, + { uz: 'uzbek' }, + { fo: 'faroese' }, + { ht: 'haitian creole' }, + { ps: 'pashto' }, + { tk: 'turkmen' }, + { nn: 'nynorsk' }, + { mt: 'maltese' }, + { sa: 'sanskrit' }, + { lb: 'luxembourgish' }, + { my: 'myanmar' }, + { bo: 'tibetan' }, + { tl: 'tagalog' }, + { mg: 'malagasy' }, + { as: 'assamese' }, + { tt: 'tatar' }, + { haw: 'hawaiian' }, + { ln: 'lingala' }, + { ha: 'hausa' }, + { ba: 'bashkir' }, + { jw: 'javanese' }, + { su: 'sundanese' }, +] + +export function mapLanguageCodeToName(code: string): string { + const lang = LANGUAGES.find(lang => lang[code]) + return lang ? lang[code] : 'Unknown Language' +} diff --git a/apps/stage-tamagotchi/src/composables/audio/extract-whisper.ts b/apps/stage-tamagotchi/src/composables/audio/extract-whisper.ts new file mode 100644 index 000000000..12e068b1a --- /dev/null +++ b/apps/stage-tamagotchi/src/composables/audio/extract-whisper.ts @@ -0,0 +1,206 @@ +import type { VADSegment } from './analysis-vad' + +import { computed, readonly, ref } from 'vue' + +import { useTauriCore } from '../tauri' +import { mapLanguageCodeToName } from './extract-whisper-languages' + +export interface WhisperConfig { + modelSize: 'tiny' | 'base' | 'small' | 'medium' | 'large' + language?: string + temperature: number + beamSize: number + bestOf: number +} + +export interface TranscriptionResult { + id: string + segmentId: string + text: string + language?: string + processingTimeMs: number + timestamp: number +} + +export function useWhisperTranscription(config: Partial = {}) { + const { invoke } = useTauriCore() + + const isModelLoaded = ref(false) + const isLoading = ref(false) + const error = ref('') + const isProcessing = ref(false) + + const whisperConfig = ref({ + modelSize: 'base', + language: undefined, // Auto-detect + temperature: 0.0, + beamSize: 5, + bestOf: 5, + ...config, + }) + + const transcriptionQueue = ref([]) + const transcriptionResults = ref([]) + const maxResults = 50 + + const onTranscriptionResultHooks = ref<((result: TranscriptionResult) => void)[]>([]) + + const currentTranscription = computed(() => + transcriptionResults.value[transcriptionResults.value.length - 1], + ) + + async function loadModel(modelType: 'base' | 'largev3' | 'tiny' | 'medium' = 'base') { + if (isModelLoaded.value || isLoading.value) + return + + isLoading.value = true + error.value = '' + + try { + await invoke('plugin:proj-airi-tauri-plugin-audio-transcription|load_model_whisper', { modelType }) + isModelLoaded.value = true + } + catch (err) { + error.value = err instanceof Error ? err.message : String(err) + console.error('Failed to load Whisper model:', err) + } + finally { + isLoading.value = false + } + } + + async function transcribeSegment(segment: VADSegment, locale: string): Promise { + if (!isModelLoaded.value || !segment.isComplete) + return null + + const startTime = performance.now() + + try { + isProcessing.value = true + + const audioArray = Array.from(segment.audioData) + const [result, language] = await invoke('plugin:proj-airi-tauri-plugin-audio-transcription|audio_transcription', { + chunk: audioArray, + language: mapLanguageCodeToName(locale), + }) || ['', ''] + + const transcription: TranscriptionResult = { + id: `transcription_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`, + segmentId: segment.id, + text: result?.trim() || '', + language: language?.trim() || '', + processingTimeMs: performance.now() - startTime, + timestamp: Date.now(), + } + + // Add to results + transcriptionResults.value.push(transcription) + if (transcriptionResults.value.length > maxResults) { + transcriptionResults.value.shift() + } + + // Notify hooks + onTranscriptionResultHooks.value.forEach(cb => cb(transcription)) + + return transcription + } + catch (err) { + error.value = err instanceof Error ? err.message : String(err) + console.error('Transcription error:', err) + return null + } + finally { + isProcessing.value = false + } + } + + async function transcribeAudioData(audioData: Float32Array, locale: string): Promise { + if (!isModelLoaded.value) + return null + + const fakeSegment: VADSegment = { + id: `manual_${Date.now()}`, + audioData, + startTime: 0, + endTime: (audioData.length / 16000) * 1000, + probability: 1.0, + isComplete: true, + } + + return await transcribeSegment(fakeSegment, locale) + } + + function queueSegment(segment: VADSegment, locale: string) { + if (segment.isComplete) { + transcriptionQueue.value.push(segment) + processQueue(locale) + } + } + + async function processQueue(locale: string) { + if (isProcessing.value || transcriptionQueue.value.length === 0) + return + + const segment = transcriptionQueue.value.shift() + if (segment) { + await transcribeSegment(segment, locale) + // Process next item in queue + if (transcriptionQueue.value.length > 0) { + setTimeout(() => processQueue(locale), 100) + } + } + } + + async function onTranscriptionResult(cb: (result: TranscriptionResult) => void) { + onTranscriptionResultHooks.value.push(cb) + + // Immediately call with current results + transcriptionResults.value.forEach(result => cb(result)) + + // Return a cleanup function + return () => { + const index = onTranscriptionResultHooks.value.indexOf(cb) + if (index !== -1) { + onTranscriptionResultHooks.value.splice(index, 1) + } + } + } + + function clearResults() { + transcriptionResults.value = [] + } + + function clearQueue() { + transcriptionQueue.value = [] + } + + function getResultBySegmentId(segmentId: string): TranscriptionResult | undefined { + return transcriptionResults.value.find(r => r.segmentId === segmentId) + } + + return { + // State + isModelLoaded: readonly(isModelLoaded), + isLoading: readonly(isLoading), + isProcessing: readonly(isProcessing), + error: readonly(error), + + // Configuration + config: whisperConfig, + + // Results + transcriptionResults: readonly(transcriptionResults), + currentTranscription, + queueLength: computed(() => transcriptionQueue.value.length), + + // Controls + loadModel, + transcribeSegment, + transcribeAudioData, + queueSegment, + clearResults, + clearQueue, + getResultBySegmentId, + onTranscriptionResult, + } +} diff --git a/apps/stage-tamagotchi/src/composables/audio/manager.ts b/apps/stage-tamagotchi/src/composables/audio/manager.ts new file mode 100644 index 000000000..71208ef58 --- /dev/null +++ b/apps/stage-tamagotchi/src/composables/audio/manager.ts @@ -0,0 +1,171 @@ +import type { AudioStreamConfig } from '@proj-airi/audio/vue' +import type { MaybeRefOrGetter } from 'vue' + +import { cleanupAudioContext, getAudioContext, getAudioContextState, isAudioContextReady } from '@proj-airi/audio/audio-context' +import { useAudioPlayback, useAudioStream } from '@proj-airi/audio/vue' +import { computed, readonly, shallowRef, toRef } from 'vue' + +import { useVADAnalysis } from './analysis-vad' +import { useVolumeAnalysis } from './analysis-volume' +import { useWhisperTranscription } from './extract-whisper' + +export interface AudioSource { + id: string + name: string + deviceId: string + type: 'microphone' | 'screen' | 'system' + config: AudioStreamConfig +} + +export function useAudioManager(locale: MaybeRefOrGetter = 'en') { + const audioContext = getAudioContext() + + const sources = shallowRef(new Map()) + const activeStreams = shallowRef(new Map>()) + const vadAnalyzers = shallowRef(new Map>()) + const volumeAnalyzers = shallowRef(new Map>()) + const playbackControllers = shallowRef(new Map>()) + const whisperTranscribers = shallowRef(new Map>()) + + const audioContextState = toRef(() => getAudioContextState()) + const localeRef = toRef(locale) + + function addSource(source: AudioSource) { + sources.value.set(source.id, source) + + // Create composables + const configRef = computed(() => sources.value.get(source.id)?.config) + const stream = useAudioStream(configRef) + const vadAnalyzer = useVADAnalysis() + const volumeAnalyzer = useVolumeAnalysis() + const whisperTranscriber = useWhisperTranscription({ modelSize: 'medium', temperature: 0.0 }) + + whisperTranscriber.loadModel() + + // Store the composables FIRST + activeStreams.value.set(source.id, stream) + vadAnalyzers.value.set(source.id, vadAnalyzer) + volumeAnalyzers.value.set(source.id, volumeAnalyzer) + whisperTranscribers.value.set(source.id, whisperTranscriber) + + // NOW create the playback with the stream reference + const streamRef = computed(() => activeStreams.value.get(source.id)?.mediaStream.value) + const playback = useAudioPlayback(streamRef) + playbackControllers.value.set(source.id, playback) + + // Connect analyzers + stream.addChunkCallback(vadAnalyzer.audioChunkCallback) + stream.addAnalysisCallback(volumeAnalyzer.audioAnalysisCallback) + + vadAnalyzer.onSegmentComplete((segment) => { + whisperTranscriber.queueSegment(segment, localeRef.value) + }) + + return source.id + } + + function removeSource(sourceId: string) { + const stream = activeStreams.value.get(sourceId) + if (stream) { + stream.stop() + activeStreams.value.delete(sourceId) + } + + sources.value.delete(sourceId) + vadAnalyzers.value.delete(sourceId) + volumeAnalyzers.value.delete(sourceId) + playbackControllers.value.delete(sourceId) + } + + function getSourceData(sourceId: string) { + return { + source: sources.value.get(sourceId), + stream: activeStreams.value.get(sourceId), + vad: vadAnalyzers.value.get(sourceId), + volume: volumeAnalyzers.value.get(sourceId), + playback: playbackControllers.value.get(sourceId), + whisper: whisperTranscribers.value.get(sourceId), + } + } + + async function startSource(sourceId: string) { + const stream = activeStreams.value.get(sourceId) + const vadAnalyzer = vadAnalyzers.value.get(sourceId) + + if (stream) { + await stream.start() + } + + if (vadAnalyzer) { + vadAnalyzer.loadModel() + } + } + + async function stopSource(sourceId: string) { + const stream = activeStreams.value.get(sourceId) + if (stream) { + await stream.stop() + } + } + + // Global audio context controls + async function suspendGlobalAudio() { + await audioContext?.suspend() + } + + async function resumeGlobalAudio() { + await audioContext?.resume() + } + + async function cleanupGlobalAudio() { + // Stop all sources first + for (const sourceId of Object.keys(sources.value)) { + await stopSource(sourceId) + } + + // Cleanup global context + await cleanupAudioContext() + } + + // Convenience methods + function addMicrophone(deviceId: string, name: string = 'Primary Microphone') { + return addSource({ + id: `mic-${deviceId}`, + name, + deviceId, + type: 'microphone', + config: { + deviceId, + sampleRate: 16000, + echoCancellation: true, + noiseSuppression: false, + autoGainControl: false, + }, + }) + } + + return { + // State + sources: readonly(sources.value), + audioContextState: { + isReady: isAudioContextReady, + sampleRate: audioContext?.sampleRate ?? 0, + error: audioContextState.value.error, + }, + + // Management + addSource, + removeSource, + startSource, + stopSource, + getSourceData, + + // Global controls + suspendGlobalAudio, + resumeGlobalAudio, + cleanupGlobalAudio, + + // Convenience + addMicrophone, + } +} diff --git a/apps/stage-tamagotchi/src/composables/tauri.ts b/apps/stage-tamagotchi/src/composables/tauri.ts index afb63f529..5d91e37fa 100644 --- a/apps/stage-tamagotchi/src/composables/tauri.ts +++ b/apps/stage-tamagotchi/src/composables/tauri.ts @@ -134,8 +134,8 @@ export interface InvokeMethods { 'open_chat_window': { args: undefined, options: undefined, returns: void } // Plugin - Audio Transcription - 'plugin:proj-airi-tauri-plugin-audio-transcription|load_model_whisper': { args: undefined, options: undefined, returns: void } - 'plugin:proj-airi-tauri-plugin-audio-transcription|audio_transcription': { args: { chunk: number[] }, options: undefined, returns: string } + 'plugin:proj-airi-tauri-plugin-audio-transcription|load_model_whisper': { args: { modelType: 'base' | 'largev3' | 'tiny' | 'medium' }, options: undefined, returns: void } + 'plugin:proj-airi-tauri-plugin-audio-transcription|audio_transcription': { args: { chunk: number[], language: string }, options: undefined, returns: [string, string] } // Plugin - Audio VAD 'plugin:proj-airi-tauri-plugin-audio-vad|load_model_silero_vad': { args: undefined, options: undefined, returns: void } diff --git a/apps/stage-tamagotchi/src/pages/index.vue b/apps/stage-tamagotchi/src/pages/index.vue index 78bded2d8..72257f563 100644 --- a/apps/stage-tamagotchi/src/pages/index.vue +++ b/apps/stage-tamagotchi/src/pages/index.vue @@ -83,7 +83,7 @@ onMounted(async () => { })) // Load models - invoke('plugin:proj-airi-tauri-plugin-audio-transcription|load_model_whisper') + invoke('plugin:proj-airi-tauri-plugin-audio-transcription|load_model_whisper', { modelType: 'medium' }) invoke('plugin:proj-airi-tauri-plugin-audio-vad|load_model_silero_vad') if (connected.value) diff --git a/apps/stage-tamagotchi/src/pages/settings/modules/hearing.vue b/apps/stage-tamagotchi/src/pages/settings/modules/hearing.vue index ecf3f5254..66627e241 100644 --- a/apps/stage-tamagotchi/src/pages/settings/modules/hearing.vue +++ b/apps/stage-tamagotchi/src/pages/settings/modules/hearing.vue @@ -3,327 +3,84 @@ import { LevelMeter, ThresholdMeter, TimeSeriesChart } from '@proj-airi/stage-ui import { FieldCheckbox, FieldRange, FieldSelect } from '@proj-airi/ui' import { useDevicesList } from '@vueuse/core' import { computed, nextTick, onMounted, onUnmounted, ref, watch } from 'vue' +import { useI18n } from 'vue-i18n' -import { useTauriCore } from '../../../composables/tauri' +import { useAudioManager } from '../../../composables/audio/manager' const devices = useDevicesList({ constraints: { audio: true } }) const audioInputs = computed(() => devices.audioInputs.value) -const { invoke } = useTauriCore() -const selectedAudioInput = ref(devices.audioInputs.value[0]?.deviceId || '') +const i18n = useI18n() +// Initialize audio manager +const audioManager = useAudioManager(i18n.locale) -const isMonitoring = ref(false) -const enablePlayback = ref(false) +const selectedAudioInput = ref('') +const selectedAudioInputSourceId = ref('') -// Audio processing state -const audioContext = ref() -const mediaStream = ref() -const analyser = ref() -const gainNode = ref() -const dataArray = ref() -const animationFrame = ref() +const enabledMonitoring = ref(false) +const enabledPlayback = ref(false) +const monitorVolume = ref(50) +const useVADModel = ref(true) -// Audio levels and indicators -const volumeLevel = ref(0) // 0-100 -const isSpeaking = ref(false) -const speakingThreshold = ref(25) // 0-100 (for volume-based fallback) -const monitorVolume = ref(50) // 0-100 - -// Tauri VAD integration -const isVADModelLoaded = ref(false) -const isLoadingVADModel = ref(false) -const vadModelError = ref('') -const useVADModel = ref(true) // Toggle between Tauri VAD and volume-based detection -const vadProbability = ref(0) // Raw VAD probability from Tauri -const vadThreshold = ref(0.5) // VAD probability threshold for speech detection - -// Audio chunk buffering for Tauri VAD -const audioChunkBuffer = ref(new Float32Array(0)) -const chunkSize = 512 // Exactly 512 samples for 16kHz as expected by VAD model -const vadProcessingInterval = ref(null) -const sampleRate = 16000 // Fixed sample rate for VAD - -// VAD visualization -const vadHistory = ref([]) // History for chart visualization -const maxVadHistory = 50 // Keep 50 samples (~1.6 seconds at 32ms intervals) - -// Tauri VAD functions -async function loadVADModel() { - if (isVADModelLoaded.value || isLoadingVADModel.value) - return - - isLoadingVADModel.value = true - vadModelError.value = '' - - try { - await invoke('plugin:proj-airi-tauri-plugin-audio-vad|load_model_silero_vad') - isVADModelLoaded.value = true - } - catch (error) { - vadModelError.value = error as string - console.error('Failed to load VAD model:', error) - } - finally { - isLoadingVADModel.value = false - } -} - -async function processAudioChunkWithVAD(audioData: Float32Array) { - if (!isVADModelLoaded.value) - return - - try { - // Ensure we have exactly 512 samples as expected by the VAD model - if (audioData.length !== chunkSize) { - console.warn(`VAD received ${audioData.length} samples, expected ${chunkSize}`) - return - } - - // Convert Float32Array to regular array for Tauri - const chunk = Array.from(audioData) - const probability = await invoke('plugin:proj-airi-tauri-plugin-audio-vad|audio_vad', { chunk }) - - if (probability != null && typeof probability === 'number') { - vadProbability.value = probability - - // Update VAD history for visualization - vadHistory.value.push(probability) - if (vadHistory.value.length > maxVadHistory) { - vadHistory.value.shift() - } - - // Update speaking detection based on VAD - if (useVADModel.value) { - isSpeaking.value = vadProbability.value > vadThreshold.value - } - } - } - catch (error) { - console.error('VAD processing error:', error) - vadModelError.value = error as string - // Fall back to volume-based detection on error - if (useVADModel.value) { - isSpeaking.value = volumeLevel.value > speakingThreshold.value - } - } -} - -function startVADProcessing() { - if (vadProcessingInterval.value) - return - - // Process chunks immediately when buffer has enough samples - vadProcessingInterval.value = window.setInterval(async () => { - if (audioChunkBuffer.value.length >= chunkSize) { - // Process the chunk with Tauri VAD (exactly 512 samples) - const chunk = audioChunkBuffer.value.slice(0, chunkSize) - await processAudioChunkWithVAD(chunk) - - // Remove processed samples from buffer - const remaining = audioChunkBuffer.value.slice(chunkSize) - audioChunkBuffer.value = remaining.length > 0 ? remaining : new Float32Array(0) - } - }, 10) // Check every 10ms for available chunks -} - -function stopVADProcessing() { - if (vadProcessingInterval.value) { - clearInterval(vadProcessingInterval.value) - vadProcessingInterval.value = null - } - audioChunkBuffer.value = new Float32Array(0) - vadProbability.value = 0 - vadHistory.value = [] -} - -// Audio monitoring -async function setupAudioMonitoring() { - try { - if (!selectedAudioInput.value) { - console.warn('No audio input device selected') - return - } - - // Clean up existing connections - await stopAudioMonitoring() - - // Get user media with selected device - mediaStream.value = await navigator.mediaDevices.getUserMedia({ - audio: { - deviceId: selectedAudioInput.value, - echoCancellation: true, - noiseSuppression: true, - autoGainControl: true, - sampleRate, // Explicitly request 16kHz - }, - }) - - // Create audio context with fixed sample rate for VAD - audioContext.value = new AudioContext({ sampleRate }) - const source = audioContext.value.createMediaStreamSource(mediaStream.value) - - // Create analyser for volume detection - analyser.value = audioContext.value.createAnalyser() - analyser.value.fftSize = 512 // Match our chunk size for better alignment - analyser.value.smoothingTimeConstant = 0.1 // Less smoothing for better real-time response - - // Create gain node for playback volume control - gainNode.value = audioContext.value.createGain() - gainNode.value.gain.value = enablePlayback.value ? (monitorVolume.value / 100) : 0 - - // Connect audio graph - source.connect(analyser.value) - - if (enablePlayback.value) { - source.connect(gainNode.value) - gainNode.value.connect(audioContext.value.destination) - } - - // Set up data array for analysis - const bufferLength = analyser.value.frequencyBinCount - dataArray.value = new Uint8Array(bufferLength) - - // Start audio analysis loop - startAudioAnalysis() - - // Load VAD model and start VAD processing if enabled - if (useVADModel.value) { - await loadVADModel() - if (isVADModelLoaded.value) { - audioChunkBuffer.value = new Float32Array(0) - startVADProcessing() - } - } - } - catch (error) { - console.error('Error setting up audio monitoring:', error) - } -} - -async function stopAudioMonitoring() { - // Stop animation frame - if (animationFrame.value) { - cancelAnimationFrame(animationFrame.value) - animationFrame.value = undefined - } - - // Stop media stream - if (mediaStream.value) { - mediaStream.value.getTracks().forEach(track => track.stop()) - mediaStream.value = undefined - } - - // Close audio context - if (audioContext.value) { - await audioContext.value.close() - audioContext.value = undefined - } - - analyser.value = undefined - gainNode.value = undefined - dataArray.value = undefined - volumeLevel.value = 0 - isSpeaking.value = false - - // Stop VAD processing - stopVADProcessing() -} - -function startAudioAnalysis() { - const analyze = () => { - if (!analyser.value || !dataArray.value) - return - - // Get frequency data for volume visualization - analyser.value.getByteFrequencyData(dataArray.value) - - // Calculate RMS volume level - let sum = 0 - for (let i = 0; i < dataArray.value.length; i++) { - sum += dataArray.value[i] * dataArray.value[i] - } - const rms = Math.sqrt(sum / dataArray.value.length) - volumeLevel.value = Math.min(100, (rms / 255) * 100 * 3) // Amplify for better visualization - - // Fallback speaking detection (when VAD model is not used) - if (!useVADModel.value || !isVADModelLoaded.value) { - isSpeaking.value = volumeLevel.value > speakingThreshold.value - } - - // Collect audio samples for VAD processing - if (useVADModel.value && isVADModelLoaded.value) { - // Get time domain data for VAD (raw audio samples) - // Use smaller buffer size for more frequent updates - const bufferSize = 128 // Smaller chunks for better real-time processing - const timeDataArray = new Float32Array(bufferSize) - analyser.value.getFloatTimeDomainData(timeDataArray) - - // Append new samples to buffer - const currentBuffer = audioChunkBuffer.value - const newBuffer = new Float32Array(currentBuffer.length + timeDataArray.length) - newBuffer.set(currentBuffer, 0) - newBuffer.set(timeDataArray, currentBuffer.length) - audioChunkBuffer.value = newBuffer - } - - animationFrame.value = requestAnimationFrame(analyze) - } - analyze() -} - -// Update playback routing when playback setting changes -async function updatePlayback() { - if (!audioContext.value || !gainNode.value) - return - - if (enablePlayback.value) { - gainNode.value.gain.value = monitorVolume.value / 100 - gainNode.value.connect(audioContext.value.destination) - } - else { - gainNode.value.gain.value = 0 - gainNode.value.disconnect() - } -} - -// Watchers -watch(selectedAudioInput, async () => { - if (isMonitoring.value) { - await setupAudioMonitoring() - } +// Get current source data reactively +const currentSource = computed(() => { + return selectedAudioInputSourceId.value ? audioManager.getSourceData(selectedAudioInputSourceId.value) : null }) -watch(enablePlayback, updatePlayback) -watch(monitorVolume, () => { - if (gainNode.value && enablePlayback.value) { - gainNode.value.gain.value = monitorVolume.value / 100 - } +// Extract reactive values from the current source +const volumeLevel = computed(() => currentSource.value?.volume?.level.value ?? 0) +const vadProbability = computed(() => currentSource.value?.vad?.probability.value ?? 0) +const vadThreshold = computed({ + get: () => currentSource.value?.vad?.config.value.threshold ?? 0.5, + set: (value) => { + if (currentSource.value?.vad?.config.value.threshold) { + currentSource.value.vad.config.value.threshold = value + } + }, }) -watch(audioInputs, () => { - if (!selectedAudioInput.value && audioInputs.value.length > 0) { - selectedAudioInput.value = audioInputs.value[0]?.deviceId - } +const speakingThreshold = computed({ + get: () => currentSource.value?.volume?.threshold.value ?? 25, + set: (value) => { + if (currentSource.value?.volume?.threshold) { + currentSource.value.volume.threshold.value = value + } + }, }) -watch(selectedAudioInput, async () => { - if (isMonitoring.value) { - await stopAudioMonitoring() - await setupAudioMonitoring() +const isVADModelLoaded = computed(() => currentSource.value?.vad?.isModelLoaded.value ?? false) +const isLoadingVADModel = computed(() => currentSource.value?.vad?.isLoading.value ?? false) +const vadModelError = computed(() => currentSource.value?.vad?.error.value ?? '') +const vadHistory = computed(() => currentSource.value?.vad?.history.value ?? []) + +// Speaking detection - prioritize VAD if enabled and loaded +const isSpeaking = computed(() => { + if (useVADModel.value && isVADModelLoaded.value) { + return currentSource.value?.vad?.isSpeaking.value ?? false } + return currentSource.value?.volume?.isSpeaking.value ?? false }) -// Monitoring toggle -async function toggleMonitoring() { - if (isMonitoring.value) { - await setupAudioMonitoring() - } - else { - await stopAudioMonitoring() - } -} +// Playback controls +const playbackEnabled = computed({ + get: () => currentSource.value?.playback?.isEnabled.value ?? false, + set: (value) => { + if (currentSource.value?.playback?.isEnabled) { + currentSource.value.playback.isEnabled.value = value + } + }, +}) -// Speaking indicator with enhanced VAD visualization +const playbackVolume = computed({ + get: () => currentSource.value?.playback?.volume.value ?? 50, + set: (value) => { + if (currentSource.value?.playback?.volume) { + currentSource.value.playback.volume.value = value + } + }, +}) + +// Speaking indicator styling const speakingIndicatorClass = computed(() => { if (!useVADModel.value || !isVADModelLoaded.value) { // Volume-based: simple green/white @@ -337,30 +94,80 @@ const speakingIndicatorClass = computed(() => { const threshold = vadThreshold.value if (prob > threshold) { - // Speaking: green (could add intensity in future) - return `bg-green-500 shadow-lg shadow-green-500/50` + return 'bg-green-500 shadow-lg shadow-green-500/50' } else if (prob > threshold * 0.5) { - // Close to threshold: yellow return 'bg-yellow-500 shadow-lg shadow-yellow-500/30' } else { - // Low probability: neutral return 'bg-white dark:bg-neutral-900 border-2 border-neutral-300 dark:border-neutral-600' } }) -// Lifecycle -onMounted(() => { - devices.ensurePermissions().then(() => nextTick()).then(() => { - if (audioInputs.value.length > 0 && !selectedAudioInput.value) { - selectedAudioInput.value = audioInputs.value[0]?.deviceId - } - }) +// Setup primary microphone source when device changes +watch(selectedAudioInput, async (newDeviceId) => { + // Remove existing source + if (selectedAudioInputSourceId.value) { + await audioManager.stopSource(selectedAudioInputSourceId.value) + audioManager.removeSource(selectedAudioInputSourceId.value) + selectedAudioInputSourceId.value = '' + } + + // Add new source if device selected + if (newDeviceId) { + const device = audioInputs.value.find(d => d.deviceId === newDeviceId) + selectedAudioInputSourceId.value = audioManager.addMicrophone( + newDeviceId, + device?.label || 'Primary Microphone', + ) + } }) -onUnmounted(() => { - stopAudioMonitoring() +// Watch for VAD model toggle +watch([useVADModel, currentSource], ([enabled, source]) => { + if (source?.vad?.isEnabled) { + source.vad.isEnabled.value = enabled + } +}) + +// Sync playback settings +watch(enabledPlayback, (enabled) => { + playbackEnabled.value = enabled +}) + +watch(monitorVolume, (volume) => { + playbackVolume.value = volume +}) + +// Monitoring toggle +async function toggleMonitoring() { + if (!selectedAudioInputSourceId.value) + return + + if (enabledMonitoring.value) { + await audioManager.startSource(selectedAudioInputSourceId.value) + } + else { + await audioManager.stopSource(selectedAudioInputSourceId.value) + } +} + +// Initialize with first available device +onMounted(async () => { + await devices.ensurePermissions() + await nextTick() + + if (audioInputs.value.length > 0 && !selectedAudioInput.value) { + selectedAudioInput.value = audioInputs.value[0]?.deviceId + } +}) + +// Cleanup on unmount +onUnmounted(async () => { + if (selectedAudioInputSourceId.value) { + await audioManager.stopSource(selectedAudioInputSourceId.value) + audioManager.removeSource(selectedAudioInputSourceId.value) + } }) @@ -369,14 +176,11 @@ onUnmounted(() => {
@@ -390,59 +194,43 @@ onUnmounted(() => {
-
+
-
+
{{ isSpeaking ? 'Speaking Detected' : 'Silence' }} @@ -454,8 +242,7 @@ onUnmounted(() => {
@@ -466,7 +253,10 @@ onUnmounted(() => { Loading...
-
+
Inference error: {{ vadModelError }}
@@ -482,43 +272,34 @@ onUnmounted(() => {
-
+
-
+
-
+
Audio feedback warning diff --git a/cspell.config.yaml b/cspell.config.yaml index e5fc71a5a..6a65b5c25 100644 --- a/cspell.config.yaml +++ b/cspell.config.yaml @@ -8,6 +8,7 @@ words: - airi - airi-vtuber - Alaya + - alexanderolsen - alibabacloud - aliyun - allseto @@ -98,6 +99,7 @@ words: - Keyyable - kwaa - lemonnekogh + - libsamplerate - libsodium - lightningcss - listhen @@ -212,6 +214,7 @@ words: - wgpu - wlipsync - worklet + - worklets - xast - xastscript - Xenova diff --git a/packages/audio/src/vue/audio-stream.ts b/packages/audio/src/vue/audio-stream.ts index f06f89543..6de05526d 100644 --- a/packages/audio/src/vue/audio-stream.ts +++ b/packages/audio/src/vue/audio-stream.ts @@ -158,6 +158,11 @@ export function useAudioStream(cfg: MaybeRefOrGetter chunkCallbacks.delete(callback) } + function stop() { + mediaStream.value?.getTracks().forEach(track => track.stop()) + } + // Cleanup onUnmounted(() => { stop() diff --git a/packages/stage-ui/src/components/Gadgets/ThresholdMeter.vue b/packages/stage-ui/src/components/Gadgets/ThresholdMeter.vue index da58ae898..d9352944d 100644 --- a/packages/stage-ui/src/components/Gadgets/ThresholdMeter.vue +++ b/packages/stage-ui/src/components/Gadgets/ThresholdMeter.vue @@ -71,7 +71,7 @@ const thresholdBars = computed(() => {
{
-
+
{{ belowLabel }} -
+
{{ thresholdLabel }} -
+
{{ aboveLabel }}
diff --git a/packages/stage-ui/src/components/Gadgets/TimeSeriesChart.vue b/packages/stage-ui/src/components/Gadgets/TimeSeriesChart.vue index a0aab716d..cca9f94c6 100644 --- a/packages/stage-ui/src/components/Gadgets/TimeSeriesChart.vue +++ b/packages/stage-ui/src/components/Gadgets/TimeSeriesChart.vue @@ -9,7 +9,7 @@ import { chromaticHue as hue } from '../../constants' import { chromaticHueDefault as hueDefault } from '../../constants/theme' interface Props { - history: number[] // Array of values (normalized 0-1) + history: Readonly // Array of values (normalized 0-1) currentValue: number // Current value (0-1) threshold?: number | null // Threshold value (0-1) isActive: boolean // Whether current state is "active"