refactor(stage-tamagotchi): sync vad impl to web
This commit is contained in:
Neko Ayaka
@@ -35,6 +35,7 @@
|
||||
"@tauri-apps/plugin-window-state": "^2.3.0",
|
||||
"@tresjs/cientos": "^4.3.1",
|
||||
"@tresjs/core": "^4.3.6",
|
||||
"@types/audioworklet": "^0.0.77",
|
||||
"@vueuse/core": "^13.4.0",
|
||||
"@vueuse/shared": "^13.4.0",
|
||||
"@xsai-ext/providers-cloud": "catalog:",
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
pub mod silero_vad;
|
||||
pub mod whisper;
|
||||
|
||||
use candle_core::Device;
|
||||
use log::info;
|
||||
use tauri::Runtime;
|
||||
|
||||
@@ -29,5 +28,5 @@ pub fn new_silero_vad_processor<R: Runtime>(
|
||||
window: tauri::WebviewWindow<R>
|
||||
) -> anyhow::Result<silero_vad::Processor> {
|
||||
info!("Loading VAD model");
|
||||
silero_vad::Processor::new(Device::Cpu, window)
|
||||
silero_vad::Processor::new(window)
|
||||
}
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
use std::path::PathBuf;
|
||||
use std::{path::PathBuf, sync::Arc};
|
||||
|
||||
use anyhow::Result;
|
||||
use hf_hub::Repo;
|
||||
use log::info;
|
||||
use ndarray::{Array2, Array3};
|
||||
use ort::{
|
||||
execution_providers::{
|
||||
CPUExecutionProvider,
|
||||
@@ -12,28 +11,33 @@ use ort::{
|
||||
DirectMLExecutionProvider,
|
||||
},
|
||||
session::{Session, builder::GraphOptimizationLevel},
|
||||
util::Mutex,
|
||||
value::Tensor,
|
||||
};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use tauri::Runtime;
|
||||
|
||||
use crate::helpers::huggingface::create_progress_emitter;
|
||||
|
||||
/// Main Silero VAD model wrapper with hardware acceleration support
|
||||
#[derive(Serialize, Deserialize, Clone)]
|
||||
pub struct VADInferenceResult {
|
||||
pub output: Vec<f32>, // Speech probability output
|
||||
pub state: Vec<f32>, // Updated state for next inference
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Clone)]
|
||||
pub struct VADInferenceInput {
|
||||
pub input: Vec<f32>, // Audio input buffer
|
||||
pub sr: i64, // Sample rate
|
||||
pub state: Vec<f32>, // Current state
|
||||
}
|
||||
|
||||
pub struct Processor {
|
||||
session: Session,
|
||||
context: Array2<f32>,
|
||||
state: ndarray::Array3<f32>,
|
||||
last_batch_size: usize,
|
||||
frame_size: usize,
|
||||
context_size: usize,
|
||||
sample_rate: i64,
|
||||
session: Arc<Mutex<Session>>,
|
||||
}
|
||||
|
||||
impl Processor {
|
||||
pub fn new<R: Runtime>(
|
||||
_device: candle_core::Device,
|
||||
window: tauri::WebviewWindow<R>,
|
||||
) -> Result<Self> {
|
||||
pub fn new<R: Runtime>(window: tauri::WebviewWindow<R>) -> Result<Self> {
|
||||
let model_id = "onnx-community/silero-vad";
|
||||
let revision = "main";
|
||||
|
||||
@@ -61,16 +65,9 @@ impl Processor {
|
||||
};
|
||||
|
||||
let session = Self::create_optimized_session(model_path.clone())?;
|
||||
let (frame_size, context_size) = (512, 64);
|
||||
|
||||
Ok(Self {
|
||||
session,
|
||||
context: Array2::zeros((1, context_size)),
|
||||
state: Array3::zeros((2, 1, 128)),
|
||||
last_batch_size: 0,
|
||||
frame_size,
|
||||
context_size,
|
||||
sample_rate: 16000,
|
||||
session: Arc::new(Mutex::new(session)),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -97,87 +94,52 @@ impl Processor {
|
||||
Ok(session)
|
||||
}
|
||||
|
||||
/// Reset the model's internal state
|
||||
fn reset_states(
|
||||
&mut self,
|
||||
batch_size: usize,
|
||||
) {
|
||||
self.context = Array2::zeros((batch_size, self.context_size));
|
||||
self.state = Array3::zeros((2, batch_size, 128));
|
||||
}
|
||||
|
||||
/// Validate input audio chunk
|
||||
fn validate_input(
|
||||
/// Stateless inference that matches JavaScript interface
|
||||
/// Returns both output and updated state like the JS version
|
||||
pub fn inference(
|
||||
&self,
|
||||
x: &[f32],
|
||||
) -> Result<()> {
|
||||
if x.len() != self.frame_size {
|
||||
input_data: VADInferenceInput,
|
||||
) -> Result<VADInferenceResult> {
|
||||
// Validate input dimensions
|
||||
if input_data.state.len() != 2 * 1 * 128 {
|
||||
return Err(anyhow::anyhow!(
|
||||
"Input chunk must be {} samples, got {}",
|
||||
self.frame_size,
|
||||
x.len()
|
||||
"State must have 256 elements (2*1*128), got {}",
|
||||
input_data.state.len()
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Process a single audio chunk and return speech probability
|
||||
/// This is the main API used by hearing.vue via the audio_vad plugin function
|
||||
pub fn process_chunk(
|
||||
&mut self,
|
||||
chunk: &[f32],
|
||||
) -> Result<f32> {
|
||||
self.validate_input(chunk)?;
|
||||
|
||||
let batch_size = 1;
|
||||
if self.last_batch_size != batch_size {
|
||||
self.reset_states(batch_size);
|
||||
}
|
||||
|
||||
// Prepare input tensor by concatenating context and new audio data
|
||||
let mut input_data = Vec::with_capacity(self.context_size + chunk.len());
|
||||
input_data.extend_from_slice(self.context.row(0).as_slice().unwrap());
|
||||
input_data.extend_from_slice(chunk);
|
||||
|
||||
let input_shape = vec![batch_size, input_data.len()];
|
||||
|
||||
// Create input tensors for the ONNX model
|
||||
// Silero VAD requires audio input, sample rate, and state
|
||||
let inputs = vec![
|
||||
(
|
||||
"input",
|
||||
Tensor::from_array((input_shape.clone(), input_data.clone()))?.into_dyn(),
|
||||
Tensor::from_array((vec![1, input_data.input.len()], input_data.input.clone()))?.into_dyn(),
|
||||
),
|
||||
(
|
||||
"sr",
|
||||
Tensor::from_array(([1], vec![self.sample_rate]))?.into_dyn(),
|
||||
Tensor::from_array(([1], vec![input_data.sr]))?.into_dyn(),
|
||||
),
|
||||
(
|
||||
"state",
|
||||
Tensor::from_array((vec![2, 1, 128], self.state.as_slice().unwrap().to_vec()))?.into_dyn(),
|
||||
Tensor::from_array((vec![2, 1, 128], input_data.state.clone()))?.into_dyn(),
|
||||
),
|
||||
];
|
||||
|
||||
// Run inference
|
||||
let outputs = self.session.run(inputs)?;
|
||||
// Run inference and extract data while session is still locked
|
||||
let (state_data, speech_data) = {
|
||||
let mut session = self.session.lock();
|
||||
let outputs = session.run(inputs)?;
|
||||
|
||||
// Update context from the last portion of the input
|
||||
let context_start = input_data.len() - self.context_size;
|
||||
let new_context_data = input_data[context_start..].to_vec();
|
||||
self.context = Array2::from_shape_vec((batch_size, self.context_size), new_context_data)
|
||||
.map_err(|e| anyhow::anyhow!("Failed to update context: {}", e))?;
|
||||
// Extract and clone the data immediately while session is locked
|
||||
let (_state_shape, state_slice) = outputs[1].try_extract_tensor::<f32>()?;
|
||||
let (_speech_shape, speech_slice) = outputs[0].try_extract_tensor::<f32>()?;
|
||||
|
||||
// Update state from model output
|
||||
let (_state_shape, state_data) = outputs[1].try_extract_tensor::<f32>()?;
|
||||
self.state = Array3::from_shape_vec((2, 1, 128), state_data.to_vec())
|
||||
.map_err(|e| anyhow::anyhow!("Failed to update state: {}", e))?;
|
||||
// Clone the data to owned vectors before the session lock is released
|
||||
(state_slice.to_vec(), speech_slice.to_vec())
|
||||
};
|
||||
|
||||
self.last_batch_size = batch_size;
|
||||
|
||||
// Extract speech probability
|
||||
let (_shape, data) = outputs[0].try_extract_tensor::<f32>()?;
|
||||
let speech_prob = data[0];
|
||||
|
||||
Ok(speech_prob)
|
||||
Ok(VADInferenceResult {
|
||||
output: speech_data,
|
||||
state: state_data,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,7 +7,10 @@ use tauri::{
|
||||
plugin::{Builder as PluginBuilder, TauriPlugin},
|
||||
};
|
||||
|
||||
use crate::app::models::new_silero_vad_processor;
|
||||
use crate::app::models::{
|
||||
new_silero_vad_processor,
|
||||
silero_vad::{VADInferenceInput, VADInferenceResult},
|
||||
};
|
||||
|
||||
#[derive(Default)]
|
||||
struct AppDataSileroVadProcessor {
|
||||
@@ -51,27 +54,18 @@ pub async fn load_model_silero_vad<R: Runtime>(
|
||||
#[tauri::command]
|
||||
pub async fn audio_vad<R: Runtime>(
|
||||
app: tauri::AppHandle<R>,
|
||||
chunk: Vec<f32>,
|
||||
) -> Result<f32, String> {
|
||||
input_data: VADInferenceInput,
|
||||
) -> Result<VADInferenceResult, String> {
|
||||
let data = app.state::<Mutex<AppDataSileroVadProcessor>>();
|
||||
let data = data.lock().unwrap();
|
||||
|
||||
// Check if processor exists first
|
||||
{
|
||||
let data = data.lock().unwrap();
|
||||
if data.silero_vad_processor.is_none() {
|
||||
return Err("Silero VAD model is not loaded".to_string());
|
||||
}
|
||||
if let Some(processor) = &data.silero_vad_processor {
|
||||
processor
|
||||
.inference(input_data)
|
||||
.map_err(|e| e.to_string())
|
||||
} else {
|
||||
Err("Silero VAD model is not loaded".to_string())
|
||||
}
|
||||
|
||||
// Then mutable borrow
|
||||
let mut data = data.lock().unwrap();
|
||||
let processor = data.silero_vad_processor.as_mut().unwrap();
|
||||
|
||||
let speech_prob = processor
|
||||
.process_chunk(chunk.as_slice())
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
Ok(speech_prob)
|
||||
}
|
||||
|
||||
pub fn init<R: Runtime>() -> TauriPlugin<R> {
|
||||
|
||||
@@ -1,397 +0,0 @@
|
||||
import type { AudioChunkCallback } from '@proj-airi/audio/vue'
|
||||
|
||||
import { onUnmounted, readonly, ref, watch } from 'vue'
|
||||
|
||||
import { useTauriCore } from '../tauri'
|
||||
|
||||
export interface VADSegment {
|
||||
id: string
|
||||
audioData: Float32Array
|
||||
startTime: number
|
||||
endTime: number
|
||||
probability: number
|
||||
isComplete: boolean
|
||||
}
|
||||
|
||||
export interface VADConfig {
|
||||
threshold: number
|
||||
silenceGapMs: number // How long silence before ending segment
|
||||
minSpeechDurationMs: number // Minimum speech duration to be valid
|
||||
maxSpeechDurationMs: number // Maximum speech duration before forced split
|
||||
overlapMs: number // Overlap between segments for better transcription
|
||||
bufferSizeMs: number // How much audio to keep in memory
|
||||
}
|
||||
|
||||
export function useVADAnalysis(config: Partial<VADConfig> = {}) {
|
||||
const { invoke } = useTauriCore()
|
||||
|
||||
const isModelLoaded = ref(false)
|
||||
const isLoading = ref(false)
|
||||
const error = ref('')
|
||||
const isEnabled = ref(true)
|
||||
|
||||
const probability = ref(0)
|
||||
const history = ref<number[]>([])
|
||||
const maxHistory = 50
|
||||
|
||||
// Configuration with defaults
|
||||
const vadConfig = ref<VADConfig>({
|
||||
threshold: 0.5,
|
||||
silenceGapMs: 500, // 500ms of silence to end segment
|
||||
minSpeechDurationMs: 300, // Minimum 300ms speech
|
||||
maxSpeechDurationMs: 30000, // Max 30s per segment
|
||||
overlapMs: 200, // 200ms overlap
|
||||
bufferSizeMs: 60000, // Keep 60s of audio in memory
|
||||
...config,
|
||||
})
|
||||
|
||||
// Audio buffering
|
||||
const audioBuffer = ref<Float32Array>(new Float32Array(0))
|
||||
const chunkSize = 512
|
||||
const targetSampleRate = 16000
|
||||
let processingInterval: number | null = null
|
||||
|
||||
// Segment management
|
||||
const currentSegment = ref<VADSegment | null>(null)
|
||||
const completedSegments = ref<VADSegment[]>([])
|
||||
const maxCompletedSegments = 10 // Keep last 10 completed segments
|
||||
|
||||
// State tracking
|
||||
const isSpeaking = ref(false)
|
||||
const lastSpeechTime = ref(0)
|
||||
const lastSilenceTime = ref(0)
|
||||
const segmentStartTime = ref(0)
|
||||
const segmentAudioBuffer = ref<Float32Array>(new Float32Array(0))
|
||||
|
||||
// Callbacks for segment events
|
||||
const segmentCallbacks = new Set<(segment: VADSegment) => void>()
|
||||
|
||||
async function loadModel() {
|
||||
if (isModelLoaded.value || isLoading.value)
|
||||
return
|
||||
|
||||
isLoading.value = true
|
||||
error.value = ''
|
||||
|
||||
try {
|
||||
await invoke('plugin:proj-airi-tauri-plugin-audio-vad|load_model_silero_vad')
|
||||
isModelLoaded.value = true
|
||||
}
|
||||
catch (err) {
|
||||
error.value = err instanceof Error ? err.message : String(err)
|
||||
console.error('Failed to load VAD model:', err)
|
||||
}
|
||||
finally {
|
||||
isLoading.value = false
|
||||
}
|
||||
}
|
||||
|
||||
function resampleIfNeeded(chunk: Float32Array, sourceSampleRate: number): Float32Array {
|
||||
if (sourceSampleRate === targetSampleRate) {
|
||||
return chunk
|
||||
}
|
||||
|
||||
const ratio = targetSampleRate / sourceSampleRate
|
||||
const outputLength = Math.floor(chunk.length * ratio)
|
||||
const resampled = new Float32Array(outputLength)
|
||||
|
||||
for (let i = 0; i < outputLength; i++) {
|
||||
const sourceIndex = i / ratio
|
||||
const index = Math.floor(sourceIndex)
|
||||
const fraction = sourceIndex - index
|
||||
|
||||
if (index + 1 < chunk.length) {
|
||||
resampled[i] = chunk[index] * (1 - fraction) + chunk[index + 1] * fraction
|
||||
}
|
||||
else {
|
||||
resampled[i] = chunk[index] || 0
|
||||
}
|
||||
}
|
||||
|
||||
return resampled
|
||||
}
|
||||
|
||||
async function processChunk(chunk: Float32Array) {
|
||||
if (!isModelLoaded.value || chunk.length !== chunkSize)
|
||||
return
|
||||
|
||||
try {
|
||||
const chunkArray = Array.from(chunk)
|
||||
const prob = await invoke('plugin:proj-airi-tauri-plugin-audio-vad|audio_vad', {
|
||||
chunk: chunkArray,
|
||||
})
|
||||
|
||||
if (typeof prob === 'number') {
|
||||
const now = performance.now()
|
||||
probability.value = prob
|
||||
|
||||
// Update history
|
||||
history.value.push(prob)
|
||||
if (history.value.length > maxHistory) {
|
||||
history.value.shift()
|
||||
}
|
||||
|
||||
// Process speech detection with hysteresis
|
||||
const wasSpeaking = isSpeaking.value
|
||||
const currentlySpeaking = prob > vadConfig.value.threshold
|
||||
|
||||
if (currentlySpeaking) {
|
||||
lastSpeechTime.value = now
|
||||
}
|
||||
else {
|
||||
lastSilenceTime.value = now
|
||||
}
|
||||
|
||||
// Handle speech state transitions
|
||||
if (!wasSpeaking && currentlySpeaking) {
|
||||
// Start of speech
|
||||
startSpeechSegment(now, chunk)
|
||||
isSpeaking.value = true
|
||||
}
|
||||
else if (wasSpeaking && !currentlySpeaking) {
|
||||
// Potential end of speech - wait for silence gap
|
||||
const silenceDuration = now - lastSpeechTime.value
|
||||
if (silenceDuration >= vadConfig.value.silenceGapMs) {
|
||||
await endSpeechSegment(now)
|
||||
isSpeaking.value = false
|
||||
}
|
||||
}
|
||||
else if (wasSpeaking && currentlySpeaking) {
|
||||
// Continuing speech - add to current segment
|
||||
addToCurrentSegment(chunk)
|
||||
|
||||
// Check for max duration
|
||||
const segmentDuration = now - segmentStartTime.value
|
||||
if (segmentDuration >= vadConfig.value.maxSpeechDurationMs) {
|
||||
await endSpeechSegment(now, true) // Force end
|
||||
startSpeechSegment(now, chunk) // Start new segment
|
||||
}
|
||||
}
|
||||
|
||||
// Always add audio to segment buffer when speaking or recently speaking
|
||||
const timeSinceLastSpeech = now - lastSpeechTime.value
|
||||
if (timeSinceLastSpeech <= vadConfig.value.silenceGapMs + vadConfig.value.overlapMs) {
|
||||
addToCurrentSegment(chunk)
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (err) {
|
||||
error.value = err instanceof Error ? err.message : String(err)
|
||||
console.error('VAD processing error:', err)
|
||||
}
|
||||
}
|
||||
|
||||
function startSpeechSegment(timestamp: number, initialChunk: Float32Array) {
|
||||
segmentStartTime.value = timestamp
|
||||
|
||||
// Include some pre-speech audio for context (overlap)
|
||||
const overlapSamples = Math.floor((vadConfig.value.overlapMs / 1000) * targetSampleRate)
|
||||
const totalBufferLength = audioBuffer.value.length
|
||||
const startIndex = Math.max(0, totalBufferLength - overlapSamples)
|
||||
|
||||
const preAudio = audioBuffer.value.slice(startIndex)
|
||||
segmentAudioBuffer.value = new Float32Array(preAudio.length + initialChunk.length)
|
||||
segmentAudioBuffer.value.set(preAudio, 0)
|
||||
segmentAudioBuffer.value.set(initialChunk, preAudio.length)
|
||||
|
||||
currentSegment.value = {
|
||||
id: `segment_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`,
|
||||
audioData: new Float32Array(0), // Will be set when segment ends
|
||||
startTime: timestamp,
|
||||
endTime: 0,
|
||||
probability: probability.value,
|
||||
isComplete: false,
|
||||
}
|
||||
}
|
||||
|
||||
function addToCurrentSegment(chunk: Float32Array) {
|
||||
if (!currentSegment.value)
|
||||
return
|
||||
|
||||
// Append to segment buffer
|
||||
const oldBuffer = segmentAudioBuffer.value
|
||||
const newBuffer = new Float32Array(oldBuffer.length + chunk.length)
|
||||
newBuffer.set(oldBuffer, 0)
|
||||
newBuffer.set(chunk, oldBuffer.length)
|
||||
segmentAudioBuffer.value = newBuffer
|
||||
}
|
||||
|
||||
async function endSpeechSegment(timestamp: number, forced = false) {
|
||||
if (!currentSegment.value)
|
||||
return
|
||||
|
||||
const segmentDuration = timestamp - segmentStartTime.value
|
||||
|
||||
// Check minimum duration requirement
|
||||
if (!forced && segmentDuration < vadConfig.value.minSpeechDurationMs) {
|
||||
// Too short, discard
|
||||
currentSegment.value = null
|
||||
segmentAudioBuffer.value = new Float32Array(0)
|
||||
return
|
||||
}
|
||||
|
||||
// Add some post-speech audio for context
|
||||
const overlapSamples = Math.floor((vadConfig.value.overlapMs / 1000) * targetSampleRate)
|
||||
const postAudioLength = Math.min(overlapSamples, audioBuffer.value.length)
|
||||
const postAudio = audioBuffer.value.slice(-postAudioLength)
|
||||
|
||||
// Final segment audio
|
||||
const finalAudioBuffer = new Float32Array(segmentAudioBuffer.value.length + postAudio.length)
|
||||
finalAudioBuffer.set(segmentAudioBuffer.value, 0)
|
||||
finalAudioBuffer.set(postAudio, segmentAudioBuffer.value.length)
|
||||
|
||||
// Complete the segment
|
||||
const completedSegment: VADSegment = {
|
||||
...currentSegment.value,
|
||||
audioData: finalAudioBuffer,
|
||||
endTime: timestamp,
|
||||
isComplete: true,
|
||||
}
|
||||
|
||||
// Add to completed segments
|
||||
completedSegments.value.push(completedSegment)
|
||||
if (completedSegments.value.length > maxCompletedSegments) {
|
||||
completedSegments.value.shift()
|
||||
}
|
||||
|
||||
// Notify callbacks
|
||||
segmentCallbacks.forEach((callback) => {
|
||||
try {
|
||||
callback(completedSegment)
|
||||
}
|
||||
catch (err) {
|
||||
console.error('Segment callback error:', err)
|
||||
}
|
||||
})
|
||||
|
||||
// Reset for next segment
|
||||
currentSegment.value = null
|
||||
segmentAudioBuffer.value = new Float32Array(0)
|
||||
}
|
||||
|
||||
function startProcessing() {
|
||||
if (processingInterval)
|
||||
return
|
||||
|
||||
const intervalMs = (chunkSize / targetSampleRate) * 1000
|
||||
|
||||
processingInterval = window.setInterval(async () => {
|
||||
if (audioBuffer.value.length >= chunkSize) {
|
||||
const chunk = audioBuffer.value.slice(0, chunkSize)
|
||||
await processChunk(chunk)
|
||||
|
||||
const remaining = audioBuffer.value.slice(chunkSize)
|
||||
audioBuffer.value = remaining.length > 0 ? remaining : new Float32Array(0)
|
||||
}
|
||||
}, Math.max(1, intervalMs / 2))
|
||||
}
|
||||
|
||||
function stopProcessing() {
|
||||
if (processingInterval) {
|
||||
clearInterval(processingInterval)
|
||||
processingInterval = null
|
||||
}
|
||||
|
||||
// End current segment if active
|
||||
if (currentSegment.value) {
|
||||
endSpeechSegment(performance.now(), true)
|
||||
}
|
||||
|
||||
audioBuffer.value = new Float32Array(0)
|
||||
probability.value = 0
|
||||
history.value = []
|
||||
}
|
||||
|
||||
// Audio chunk callback
|
||||
const audioChunkCallback: AudioChunkCallback = (chunk: Float32Array, sampleRate: number) => {
|
||||
if (!isEnabled.value || !isModelLoaded.value)
|
||||
return
|
||||
|
||||
const processedChunk = resampleIfNeeded(chunk, sampleRate)
|
||||
|
||||
// Add to main buffer
|
||||
const currentBuffer = audioBuffer.value
|
||||
const newBuffer = new Float32Array(currentBuffer.length + processedChunk.length)
|
||||
newBuffer.set(currentBuffer, 0)
|
||||
newBuffer.set(processedChunk, currentBuffer.length)
|
||||
|
||||
// Trim buffer to max size
|
||||
const maxBufferSamples = (vadConfig.value.bufferSizeMs / 1000) * targetSampleRate
|
||||
if (newBuffer.length > maxBufferSamples) {
|
||||
const trimAmount = newBuffer.length - maxBufferSamples
|
||||
audioBuffer.value = newBuffer.slice(trimAmount)
|
||||
}
|
||||
else {
|
||||
audioBuffer.value = newBuffer
|
||||
}
|
||||
}
|
||||
|
||||
// Segment callback management
|
||||
function onSegmentComplete(callback: (segment: VADSegment) => void) {
|
||||
segmentCallbacks.add(callback)
|
||||
return () => segmentCallbacks.delete(callback)
|
||||
}
|
||||
|
||||
// Manual segment control
|
||||
async function forceEndCurrentSegment() {
|
||||
if (currentSegment.value) {
|
||||
await endSpeechSegment(performance.now(), true)
|
||||
isSpeaking.value = false
|
||||
}
|
||||
}
|
||||
|
||||
function clearCompletedSegments() {
|
||||
completedSegments.value = []
|
||||
}
|
||||
|
||||
// Get segment by ID
|
||||
function getSegment(id: string): VADSegment | undefined {
|
||||
return completedSegments.value.find(s => s.id === id)
|
||||
}
|
||||
|
||||
// Auto-start processing when enabled and model is loaded
|
||||
watch([isEnabled, isModelLoaded], ([enabled, loaded]) => {
|
||||
if (enabled && loaded) {
|
||||
startProcessing()
|
||||
}
|
||||
else {
|
||||
stopProcessing()
|
||||
}
|
||||
})
|
||||
|
||||
onUnmounted(() => {
|
||||
stopProcessing()
|
||||
})
|
||||
|
||||
return {
|
||||
// State
|
||||
isModelLoaded: readonly(isModelLoaded),
|
||||
isLoading: readonly(isLoading),
|
||||
error: readonly(error),
|
||||
isEnabled,
|
||||
|
||||
// VAD data
|
||||
probability: readonly(probability),
|
||||
history: readonly(history),
|
||||
isSpeaking: readonly(isSpeaking),
|
||||
|
||||
// Configuration
|
||||
config: vadConfig,
|
||||
|
||||
// Segments
|
||||
currentSegment: readonly(currentSegment),
|
||||
completedSegments: readonly(completedSegments),
|
||||
|
||||
// Controls
|
||||
loadModel,
|
||||
forceEndCurrentSegment,
|
||||
clearCompletedSegments,
|
||||
getSegment,
|
||||
|
||||
// Events
|
||||
onSegmentComplete,
|
||||
|
||||
// For audio stream integration
|
||||
audioChunkCallback,
|
||||
}
|
||||
}
|
||||
@@ -1,46 +0,0 @@
|
||||
import type { AudioAnalysisCallback } from '@proj-airi/audio/vue'
|
||||
|
||||
import { computed, readonly, ref } from 'vue'
|
||||
|
||||
export function useVolumeAnalysis() {
|
||||
const level = ref(0)
|
||||
const threshold = ref(25)
|
||||
const history = ref<number[]>([])
|
||||
const maxHistory = 100
|
||||
const isEnabled = ref(true)
|
||||
|
||||
const isSpeaking = computed(() => level.value > threshold.value)
|
||||
|
||||
const audioAnalysisCallback: AudioAnalysisCallback = (data) => {
|
||||
if (!isEnabled.value)
|
||||
return
|
||||
|
||||
level.value = data.volumeLevel
|
||||
|
||||
// Update history
|
||||
history.value.push(data.volumeLevel)
|
||||
if (history.value.length > maxHistory) {
|
||||
history.value.shift()
|
||||
}
|
||||
}
|
||||
|
||||
function reset() {
|
||||
level.value = 0
|
||||
history.value = []
|
||||
}
|
||||
|
||||
return {
|
||||
// State
|
||||
level: readonly(level),
|
||||
threshold,
|
||||
history: readonly(history),
|
||||
isSpeaking,
|
||||
isEnabled,
|
||||
|
||||
// Controls
|
||||
reset,
|
||||
|
||||
// For audio stream integration
|
||||
audioAnalysisCallback,
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,3 @@
|
||||
import type { VADSegment } from './analysis-vad'
|
||||
|
||||
import { computed, readonly, ref } from 'vue'
|
||||
|
||||
import { useTauriCore } from '../tauri'
|
||||
@@ -13,6 +11,15 @@ export interface WhisperConfig {
|
||||
bestOf: number
|
||||
}
|
||||
|
||||
export interface VADSegment {
|
||||
id: string
|
||||
audioData: Float32Array
|
||||
startTime: number
|
||||
endTime: number
|
||||
probability: number
|
||||
isComplete: boolean
|
||||
}
|
||||
|
||||
export interface TranscriptionResult {
|
||||
id: string
|
||||
segmentId: string
|
||||
|
||||
@@ -1,171 +0,0 @@
|
||||
import type { AudioStreamConfig } from '@proj-airi/audio/vue'
|
||||
import type { MaybeRefOrGetter } from 'vue'
|
||||
|
||||
import { cleanupAudioContext, getAudioContext, getAudioContextState, isAudioContextReady } from '@proj-airi/audio/audio-context'
|
||||
import { useAudioPlayback, useAudioStream } from '@proj-airi/audio/vue'
|
||||
import { computed, readonly, shallowRef, toRef } from 'vue'
|
||||
|
||||
import { useVADAnalysis } from './analysis-vad'
|
||||
import { useVolumeAnalysis } from './analysis-volume'
|
||||
import { useWhisperTranscription } from './extract-whisper'
|
||||
|
||||
export interface AudioSource {
|
||||
id: string
|
||||
name: string
|
||||
deviceId: string
|
||||
type: 'microphone' | 'screen' | 'system'
|
||||
config: AudioStreamConfig
|
||||
}
|
||||
|
||||
export function useAudioManager(locale: MaybeRefOrGetter<string> = 'en') {
|
||||
const audioContext = getAudioContext()
|
||||
|
||||
const sources = shallowRef(new Map<string, AudioSource>())
|
||||
const activeStreams = shallowRef(new Map<string, ReturnType<typeof useAudioStream>>())
|
||||
const vadAnalyzers = shallowRef(new Map<string, ReturnType<typeof useVADAnalysis>>())
|
||||
const volumeAnalyzers = shallowRef(new Map<string, ReturnType<typeof useVolumeAnalysis>>())
|
||||
const playbackControllers = shallowRef(new Map<string, ReturnType<typeof useAudioPlayback>>())
|
||||
const whisperTranscribers = shallowRef(new Map<string, ReturnType<typeof useWhisperTranscription>>())
|
||||
|
||||
const audioContextState = toRef(() => getAudioContextState())
|
||||
const localeRef = toRef(locale)
|
||||
|
||||
function addSource(source: AudioSource) {
|
||||
sources.value.set(source.id, source)
|
||||
|
||||
// Create composables
|
||||
const configRef = computed(() => sources.value.get(source.id)?.config)
|
||||
const stream = useAudioStream(configRef)
|
||||
const vadAnalyzer = useVADAnalysis()
|
||||
const volumeAnalyzer = useVolumeAnalysis()
|
||||
const whisperTranscriber = useWhisperTranscription({ modelSize: 'medium', temperature: 0.0 })
|
||||
|
||||
whisperTranscriber.loadModel()
|
||||
|
||||
// Store the composables FIRST
|
||||
activeStreams.value.set(source.id, stream)
|
||||
vadAnalyzers.value.set(source.id, vadAnalyzer)
|
||||
volumeAnalyzers.value.set(source.id, volumeAnalyzer)
|
||||
whisperTranscribers.value.set(source.id, whisperTranscriber)
|
||||
|
||||
// NOW create the playback with the stream reference
|
||||
const streamRef = computed(() => activeStreams.value.get(source.id)?.mediaStream.value)
|
||||
const playback = useAudioPlayback(streamRef)
|
||||
playbackControllers.value.set(source.id, playback)
|
||||
|
||||
// Connect analyzers
|
||||
stream.addChunkCallback(vadAnalyzer.audioChunkCallback)
|
||||
stream.addAnalysisCallback(volumeAnalyzer.audioAnalysisCallback)
|
||||
|
||||
vadAnalyzer.onSegmentComplete((segment) => {
|
||||
whisperTranscriber.queueSegment(segment, localeRef.value)
|
||||
})
|
||||
|
||||
return source.id
|
||||
}
|
||||
|
||||
function removeSource(sourceId: string) {
|
||||
const stream = activeStreams.value.get(sourceId)
|
||||
if (stream) {
|
||||
stream.stop()
|
||||
activeStreams.value.delete(sourceId)
|
||||
}
|
||||
|
||||
sources.value.delete(sourceId)
|
||||
vadAnalyzers.value.delete(sourceId)
|
||||
volumeAnalyzers.value.delete(sourceId)
|
||||
playbackControllers.value.delete(sourceId)
|
||||
}
|
||||
|
||||
function getSourceData(sourceId: string) {
|
||||
return {
|
||||
source: sources.value.get(sourceId),
|
||||
stream: activeStreams.value.get(sourceId),
|
||||
vad: vadAnalyzers.value.get(sourceId),
|
||||
volume: volumeAnalyzers.value.get(sourceId),
|
||||
playback: playbackControllers.value.get(sourceId),
|
||||
whisper: whisperTranscribers.value.get(sourceId),
|
||||
}
|
||||
}
|
||||
|
||||
async function startSource(sourceId: string) {
|
||||
const stream = activeStreams.value.get(sourceId)
|
||||
const vadAnalyzer = vadAnalyzers.value.get(sourceId)
|
||||
|
||||
if (stream) {
|
||||
await stream.start()
|
||||
}
|
||||
|
||||
if (vadAnalyzer) {
|
||||
vadAnalyzer.loadModel()
|
||||
}
|
||||
}
|
||||
|
||||
async function stopSource(sourceId: string) {
|
||||
const stream = activeStreams.value.get(sourceId)
|
||||
if (stream) {
|
||||
await stream.stop()
|
||||
}
|
||||
}
|
||||
|
||||
// Global audio context controls
|
||||
async function suspendGlobalAudio() {
|
||||
await audioContext?.suspend()
|
||||
}
|
||||
|
||||
async function resumeGlobalAudio() {
|
||||
await audioContext?.resume()
|
||||
}
|
||||
|
||||
async function cleanupGlobalAudio() {
|
||||
// Stop all sources first
|
||||
for (const sourceId of Object.keys(sources.value)) {
|
||||
await stopSource(sourceId)
|
||||
}
|
||||
|
||||
// Cleanup global context
|
||||
await cleanupAudioContext()
|
||||
}
|
||||
|
||||
// Convenience methods
|
||||
function addMicrophone(deviceId: string, name: string = 'Primary Microphone') {
|
||||
return addSource({
|
||||
id: `mic-${deviceId}`,
|
||||
name,
|
||||
deviceId,
|
||||
type: 'microphone',
|
||||
config: {
|
||||
deviceId,
|
||||
sampleRate: 16000,
|
||||
echoCancellation: true,
|
||||
noiseSuppression: false,
|
||||
autoGainControl: false,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
return {
|
||||
// State
|
||||
sources: readonly(sources.value),
|
||||
audioContextState: {
|
||||
isReady: isAudioContextReady,
|
||||
sampleRate: audioContext?.sampleRate ?? 0,
|
||||
error: audioContextState.value.error,
|
||||
},
|
||||
|
||||
// Management
|
||||
addSource,
|
||||
removeSource,
|
||||
startSource,
|
||||
stopSource,
|
||||
getSourceData,
|
||||
|
||||
// Global controls
|
||||
suspendGlobalAudio,
|
||||
resumeGlobalAudio,
|
||||
cleanupGlobalAudio,
|
||||
|
||||
// Convenience
|
||||
addMicrophone,
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,8 @@ import type { InvokeArgs, InvokeOptions } from '@tauri-apps/api/core'
|
||||
import type { EventCallback, EventName, UnlistenFn } from '@tauri-apps/api/event'
|
||||
import type { Monitor } from '@tauri-apps/api/window'
|
||||
|
||||
import type { InvokeMethods, InvokeMethodShape } from '../tauri/invoke'
|
||||
|
||||
import { withRetry } from '@moeru/std'
|
||||
import { computedAsync, until } from '@vueuse/core'
|
||||
|
||||
@@ -128,39 +130,6 @@ export function useTauriEvent<ES = Events>() {
|
||||
}
|
||||
}
|
||||
|
||||
export interface InvokeMethods {
|
||||
// app windows
|
||||
'open_settings_window': { args: undefined, options: undefined, returns: void }
|
||||
'open_chat_window': { args: undefined, options: undefined, returns: void }
|
||||
|
||||
// Plugin - Audio Transcription
|
||||
'plugin:proj-airi-tauri-plugin-audio-transcription|load_model_whisper': { args: { modelType: 'base' | 'largev3' | 'tiny' | 'medium' }, options: undefined, returns: void }
|
||||
'plugin:proj-airi-tauri-plugin-audio-transcription|audio_transcription': { args: { chunk: number[], language: string }, options: undefined, returns: [string, string] }
|
||||
|
||||
// Plugin - Audio VAD
|
||||
'plugin:proj-airi-tauri-plugin-audio-vad|load_model_silero_vad': { args: undefined, options: undefined, returns: void }
|
||||
'plugin:proj-airi-tauri-plugin-audio-vad|audio_vad': { args: { chunk: number[] }, options: undefined, returns: number }
|
||||
|
||||
// Plugin - Window Pass through on hover
|
||||
'plugin:proj-airi-tauri-plugin-window-pass-through-on-hover|start_monitor': { args: undefined, options: undefined, returns: void }
|
||||
'plugin:proj-airi-tauri-plugin-window-pass-through-on-hover|stop_monitor': { args: undefined, options: undefined, returns: void }
|
||||
'plugin:proj-airi-tauri-plugin-window-pass-through-on-hover|start_pass_through': { args: undefined, options: undefined, returns: void }
|
||||
'plugin:proj-airi-tauri-plugin-window-pass-through-on-hover|stop_pass_through': { args: undefined, options: undefined, returns: void }
|
||||
|
||||
// Plugin - WindowRouterLink
|
||||
'plugin:proj-airi-tauri-plugin-window-router-link|go': {
|
||||
args: { route: string, windowLabel?: string } | undefined
|
||||
options: undefined
|
||||
returns: void
|
||||
}
|
||||
}
|
||||
|
||||
interface InvokeMethodShape {
|
||||
args: InvokeArgs | undefined
|
||||
options: InvokeOptions | undefined
|
||||
returns: any
|
||||
}
|
||||
|
||||
export function useTauriCore<IM extends Record<keyof IM, InvokeMethodShape> = InvokeMethods>() {
|
||||
const { platform, isInitialized } = useAppRuntime()
|
||||
|
||||
@@ -172,6 +141,14 @@ export function useTauriCore<IM extends Record<keyof IM, InvokeMethodShape> = In
|
||||
}
|
||||
})
|
||||
|
||||
const tauriCoreApiInvoke = computedAsync(async () => {
|
||||
await until(isInitialized).toBeTruthy()
|
||||
|
||||
if (platform.value !== 'web') {
|
||||
return untilImported(() => import('../tauri/invoke'), console.warn)
|
||||
}
|
||||
})
|
||||
|
||||
async function invoke<C extends keyof IM>(
|
||||
command: C,
|
||||
args?: IM[C]['args'],
|
||||
@@ -184,13 +161,13 @@ export function useTauriCore<IM extends Record<keyof IM, InvokeMethodShape> = In
|
||||
return
|
||||
}
|
||||
|
||||
await until(tauriCoreApi).toBeTruthy()
|
||||
const imported = await tauriCoreApi.value
|
||||
await until(tauriCoreApiInvoke).toBeTruthy()
|
||||
const imported = await tauriCoreApiInvoke.value
|
||||
if (!imported) {
|
||||
throw new Error('Tauri core API not available')
|
||||
}
|
||||
|
||||
return await imported.invoke(command as string, args as InvokeArgs | undefined, options as InvokeOptions | undefined)
|
||||
return await imported.invoke<C, IM>(command, args as InvokeArgs | undefined, options as InvokeOptions | undefined)
|
||||
}
|
||||
|
||||
return {
|
||||
|
||||
@@ -3,84 +3,288 @@ import { LevelMeter, ThresholdMeter, TimeSeriesChart } from '@proj-airi/stage-ui
|
||||
import { FieldCheckbox, FieldRange, FieldSelect } from '@proj-airi/ui'
|
||||
import { useDevicesList } from '@vueuse/core'
|
||||
import { computed, nextTick, onMounted, onUnmounted, ref, watch } from 'vue'
|
||||
import { useI18n } from 'vue-i18n'
|
||||
|
||||
import { useAudioManager } from '../../../composables/audio/manager'
|
||||
import workletUrl from '../../../tauri/vad/process.worklet?worker&url'
|
||||
|
||||
import { createVAD, VADAudioManager } from '../../../tauri/vad'
|
||||
|
||||
const devices = useDevicesList({ constraints: { audio: true } })
|
||||
const audioInputs = computed(() => devices.audioInputs.value)
|
||||
|
||||
const i18n = useI18n()
|
||||
// Initialize audio manager
|
||||
const audioManager = useAudioManager(i18n.locale)
|
||||
const selectedAudioInput = ref<string>(devices.audioInputs.value[0]?.deviceId || '')
|
||||
|
||||
const selectedAudioInput = ref<string>('')
|
||||
const selectedAudioInputSourceId = ref<string>('')
|
||||
const isMonitoring = ref(false)
|
||||
const enablePlayback = ref(false)
|
||||
|
||||
const enabledMonitoring = ref(false)
|
||||
const enabledPlayback = ref(false)
|
||||
const monitorVolume = ref(50)
|
||||
const useVADModel = ref(true)
|
||||
// Audio processing state
|
||||
const audioContext = ref<AudioContext>()
|
||||
const mediaStream = ref<MediaStream>()
|
||||
const analyser = ref<AnalyserNode>()
|
||||
const gainNode = ref<GainNode>()
|
||||
const dataArray = ref<Uint8Array>()
|
||||
const animationFrame = ref<number>()
|
||||
|
||||
// Get current source data reactively
|
||||
const currentSource = computed(() => {
|
||||
return selectedAudioInputSourceId.value ? audioManager.getSourceData(selectedAudioInputSourceId.value) : null
|
||||
})
|
||||
// Audio levels and indicators
|
||||
const volumeLevel = ref(0) // 0-100
|
||||
const isSpeaking = ref(false)
|
||||
const speakingThreshold = ref(25) // 0-100 (for volume-based fallback)
|
||||
const monitorVolume = ref(50) // 0-100
|
||||
|
||||
// Extract reactive values from the current source
|
||||
const volumeLevel = computed(() => currentSource.value?.volume?.level.value ?? 0)
|
||||
const vadProbability = computed(() => currentSource.value?.vad?.probability.value ?? 0)
|
||||
const vadThreshold = computed({
|
||||
get: () => currentSource.value?.vad?.config.value.threshold ?? 0.5,
|
||||
set: (value) => {
|
||||
if (currentSource.value?.vad?.config.value.threshold) {
|
||||
currentSource.value.vad.config.value.threshold = value
|
||||
}
|
||||
},
|
||||
})
|
||||
// VAD integration
|
||||
const vadManager = ref<VADAudioManager>()
|
||||
const isVADModelLoaded = ref(false)
|
||||
const isLoadingVADModel = ref(false)
|
||||
const vadModelError = ref('')
|
||||
const useVADModel = ref(true) // Toggle between VAD and volume-based detection
|
||||
const vadProbability = ref(0) // Raw VAD probability
|
||||
const vadThreshold = ref(0.5) // VAD probability threshold for speech detection
|
||||
|
||||
const speakingThreshold = computed({
|
||||
get: () => currentSource.value?.volume?.threshold.value ?? 25,
|
||||
set: (value) => {
|
||||
if (currentSource.value?.volume?.threshold) {
|
||||
currentSource.value.volume.threshold.value = value
|
||||
}
|
||||
},
|
||||
})
|
||||
// VAD visualization
|
||||
const vadHistory = ref<number[]>([]) // History for chart visualization
|
||||
const maxVadHistory = 50 // Keep 50 samples (~1.6 seconds at 32ms intervals)
|
||||
|
||||
const isVADModelLoaded = computed(() => currentSource.value?.vad?.isModelLoaded.value ?? false)
|
||||
const isLoadingVADModel = computed(() => currentSource.value?.vad?.isLoading.value ?? false)
|
||||
const vadModelError = computed(() => currentSource.value?.vad?.error.value ?? '')
|
||||
const vadHistory = computed(() => currentSource.value?.vad?.history.value ?? [])
|
||||
// VAD functions
|
||||
async function loadVADModel() {
|
||||
if (isVADModelLoaded.value || isLoadingVADModel.value)
|
||||
return
|
||||
|
||||
// Speaking detection - prioritize VAD if enabled and loaded
|
||||
const isSpeaking = computed(() => {
|
||||
if (useVADModel.value && isVADModelLoaded.value) {
|
||||
return currentSource.value?.vad?.isSpeaking.value ?? false
|
||||
isLoadingVADModel.value = true
|
||||
vadModelError.value = ''
|
||||
|
||||
try {
|
||||
// Create and initialize the VAD
|
||||
const vad = await createVAD({
|
||||
sampleRate: 16000,
|
||||
speechThreshold: vadThreshold.value,
|
||||
exitThreshold: vadThreshold.value * 0.3,
|
||||
minSilenceDurationMs: 400,
|
||||
})
|
||||
|
||||
// Set up event handlers
|
||||
vad.on('speech-start', () => {
|
||||
isSpeaking.value = true
|
||||
})
|
||||
|
||||
vad.on('speech-end', () => {
|
||||
isSpeaking.value = false
|
||||
})
|
||||
|
||||
vad.on('debug', ({ data }) => {
|
||||
if (data?.probability !== undefined) {
|
||||
vadProbability.value = data.probability
|
||||
|
||||
// Update VAD history for visualization
|
||||
vadHistory.value.push(data.probability)
|
||||
if (vadHistory.value.length > maxVadHistory) {
|
||||
vadHistory.value.shift()
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
vad.on('status', ({ type, message }) => {
|
||||
if (type === 'error') {
|
||||
vadModelError.value = message
|
||||
}
|
||||
})
|
||||
|
||||
// Create and initialize audio manager
|
||||
const manager = new VADAudioManager(vad, {
|
||||
minChunkSize: 512,
|
||||
audioContextOptions: {
|
||||
sampleRate: 16000,
|
||||
latencyHint: 'interactive',
|
||||
},
|
||||
})
|
||||
|
||||
await manager.initialize(workletUrl)
|
||||
vadManager.value = manager
|
||||
isVADModelLoaded.value = true
|
||||
}
|
||||
return currentSource.value?.volume?.isSpeaking.value ?? false
|
||||
})
|
||||
catch (error) {
|
||||
vadModelError.value = error instanceof Error ? error.message : String(error)
|
||||
console.error('Failed to load VAD model:', error)
|
||||
}
|
||||
finally {
|
||||
isLoadingVADModel.value = false
|
||||
}
|
||||
}
|
||||
|
||||
// Playback controls
|
||||
const playbackEnabled = computed({
|
||||
get: () => currentSource.value?.playback?.isEnabled.value ?? false,
|
||||
set: (value) => {
|
||||
if (currentSource.value?.playback?.isEnabled) {
|
||||
currentSource.value.playback.isEnabled.value = value
|
||||
// Audio monitoring
|
||||
async function setupAudioMonitoring() {
|
||||
try {
|
||||
if (!selectedAudioInput.value) {
|
||||
console.warn('No audio input device selected')
|
||||
return
|
||||
}
|
||||
},
|
||||
})
|
||||
|
||||
const playbackVolume = computed({
|
||||
get: () => currentSource.value?.playback?.volume.value ?? 50,
|
||||
set: (value) => {
|
||||
if (currentSource.value?.playback?.volume) {
|
||||
currentSource.value.playback.volume.value = value
|
||||
// Clean up existing connections
|
||||
await stopAudioMonitoring()
|
||||
|
||||
// Get user media with selected device
|
||||
mediaStream.value = await navigator.mediaDevices.getUserMedia({
|
||||
audio: {
|
||||
deviceId: selectedAudioInput.value,
|
||||
echoCancellation: true,
|
||||
noiseSuppression: true,
|
||||
autoGainControl: true,
|
||||
},
|
||||
})
|
||||
|
||||
// Create audio context
|
||||
audioContext.value = new AudioContext()
|
||||
const source = audioContext.value.createMediaStreamSource(mediaStream.value)
|
||||
|
||||
// Create analyser for volume detection
|
||||
analyser.value = audioContext.value.createAnalyser()
|
||||
analyser.value.fftSize = 256
|
||||
analyser.value.smoothingTimeConstant = 0.3
|
||||
|
||||
// Create gain node for playback volume control
|
||||
gainNode.value = audioContext.value.createGain()
|
||||
gainNode.value.gain.value = enablePlayback.value ? (monitorVolume.value / 100) : 0
|
||||
|
||||
// Connect audio graph
|
||||
source.connect(analyser.value)
|
||||
|
||||
if (enablePlayback.value) {
|
||||
source.connect(gainNode.value)
|
||||
gainNode.value.connect(audioContext.value.destination)
|
||||
}
|
||||
},
|
||||
|
||||
// Set up data array for analysis
|
||||
const bufferLength = analyser.value.frequencyBinCount
|
||||
dataArray.value = new Uint8Array(bufferLength)
|
||||
|
||||
// Start audio analysis loop
|
||||
startAudioAnalysis()
|
||||
|
||||
// Load VAD model and start VAD processing if enabled
|
||||
if (useVADModel.value) {
|
||||
await loadVADModel()
|
||||
if (vadManager.value) {
|
||||
await vadManager.value.startMicrophone()
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (error) {
|
||||
console.error('Error setting up audio monitoring:', error)
|
||||
vadModelError.value = error instanceof Error ? error.message : String(error)
|
||||
}
|
||||
}
|
||||
|
||||
async function stopAudioMonitoring() {
|
||||
// Stop animation frame
|
||||
if (animationFrame.value) {
|
||||
cancelAnimationFrame(animationFrame.value)
|
||||
animationFrame.value = undefined
|
||||
}
|
||||
|
||||
// Stop VAD manager
|
||||
if (vadManager.value) {
|
||||
await vadManager.value.stopMicrophone()
|
||||
}
|
||||
|
||||
// Stop media stream
|
||||
if (mediaStream.value) {
|
||||
mediaStream.value.getTracks().forEach(track => track.stop())
|
||||
mediaStream.value = undefined
|
||||
}
|
||||
|
||||
// Close audio context
|
||||
if (audioContext.value) {
|
||||
await audioContext.value.close()
|
||||
audioContext.value = undefined
|
||||
}
|
||||
|
||||
analyser.value = undefined
|
||||
gainNode.value = undefined
|
||||
dataArray.value = undefined
|
||||
volumeLevel.value = 0
|
||||
isSpeaking.value = false
|
||||
vadProbability.value = 0
|
||||
vadHistory.value = []
|
||||
}
|
||||
|
||||
function startAudioAnalysis() {
|
||||
const analyze = () => {
|
||||
if (!analyser.value || !dataArray.value)
|
||||
return
|
||||
|
||||
// Get frequency data for volume visualization
|
||||
analyser.value.getByteFrequencyData(dataArray.value)
|
||||
|
||||
// Calculate RMS volume level
|
||||
let sum = 0
|
||||
for (let i = 0; i < dataArray.value.length; i++) {
|
||||
sum += dataArray.value[i] * dataArray.value[i]
|
||||
}
|
||||
const rms = Math.sqrt(sum / dataArray.value.length)
|
||||
volumeLevel.value = Math.min(100, (rms / 255) * 100 * 3) // Amplify for better visualization
|
||||
|
||||
// Fallback speaking detection (when VAD model is not used)
|
||||
if (!useVADModel.value || !isVADModelLoaded.value) {
|
||||
isSpeaking.value = volumeLevel.value > speakingThreshold.value
|
||||
}
|
||||
|
||||
animationFrame.value = requestAnimationFrame(analyze)
|
||||
}
|
||||
analyze()
|
||||
}
|
||||
|
||||
// Update playback routing when playback setting changes
|
||||
async function updatePlayback() {
|
||||
if (!audioContext.value || !gainNode.value)
|
||||
return
|
||||
|
||||
if (enablePlayback.value) {
|
||||
gainNode.value.gain.value = monitorVolume.value / 100
|
||||
gainNode.value.connect(audioContext.value.destination)
|
||||
}
|
||||
else {
|
||||
gainNode.value.gain.value = 0
|
||||
gainNode.value.disconnect()
|
||||
}
|
||||
}
|
||||
|
||||
// Watchers
|
||||
watch(selectedAudioInput, async () => {
|
||||
if (isMonitoring.value) {
|
||||
await setupAudioMonitoring()
|
||||
}
|
||||
})
|
||||
|
||||
// Speaking indicator styling
|
||||
watch(enablePlayback, updatePlayback)
|
||||
watch(monitorVolume, () => {
|
||||
if (gainNode.value && enablePlayback.value) {
|
||||
gainNode.value.gain.value = monitorVolume.value / 100
|
||||
}
|
||||
})
|
||||
|
||||
watch(audioInputs, () => {
|
||||
if (!selectedAudioInput.value && audioInputs.value.length > 0) {
|
||||
selectedAudioInput.value = audioInputs.value[0]?.deviceId
|
||||
}
|
||||
})
|
||||
|
||||
watch(vadThreshold, () => {
|
||||
// Update VAD threshold if model is loaded
|
||||
if (vadManager.value && isVADModelLoaded.value) {
|
||||
// Note: We would need to add an updateConfig method to VADAudioManager
|
||||
// For now, this is a placeholder
|
||||
}
|
||||
})
|
||||
|
||||
// Monitoring toggle
|
||||
async function toggleMonitoring() {
|
||||
if (isMonitoring.value) {
|
||||
await setupAudioMonitoring()
|
||||
}
|
||||
else {
|
||||
await stopAudioMonitoring()
|
||||
}
|
||||
}
|
||||
|
||||
// Speaking indicator with enhanced VAD visualization
|
||||
const speakingIndicatorClass = computed(() => {
|
||||
if (!useVADModel.value || !isVADModelLoaded.value) {
|
||||
// Volume-based: simple green/white
|
||||
@@ -94,79 +298,32 @@ const speakingIndicatorClass = computed(() => {
|
||||
const threshold = vadThreshold.value
|
||||
|
||||
if (prob > threshold) {
|
||||
return 'bg-green-500 shadow-lg shadow-green-500/50'
|
||||
// Speaking: green (could add intensity in future)
|
||||
return `bg-green-500 shadow-lg shadow-green-500/50`
|
||||
}
|
||||
else if (prob > threshold * 0.5) {
|
||||
// Close to threshold: yellow
|
||||
return 'bg-yellow-500 shadow-lg shadow-yellow-500/30'
|
||||
}
|
||||
else {
|
||||
// Low probability: neutral
|
||||
return 'bg-white dark:bg-neutral-900 border-2 border-neutral-300 dark:border-neutral-600'
|
||||
}
|
||||
})
|
||||
|
||||
// Setup primary microphone source when device changes
|
||||
watch(selectedAudioInput, async (newDeviceId) => {
|
||||
// Remove existing source
|
||||
if (selectedAudioInputSourceId.value) {
|
||||
await audioManager.stopSource(selectedAudioInputSourceId.value)
|
||||
audioManager.removeSource(selectedAudioInputSourceId.value)
|
||||
selectedAudioInputSourceId.value = ''
|
||||
}
|
||||
|
||||
// Add new source if device selected
|
||||
if (newDeviceId) {
|
||||
const device = audioInputs.value.find(d => d.deviceId === newDeviceId)
|
||||
selectedAudioInputSourceId.value = audioManager.addMicrophone(
|
||||
newDeviceId,
|
||||
device?.label || 'Primary Microphone',
|
||||
)
|
||||
}
|
||||
// Lifecycle
|
||||
onMounted(() => {
|
||||
devices.ensurePermissions().then(() => nextTick()).then(() => {
|
||||
if (audioInputs.value.length > 0 && !selectedAudioInput.value) {
|
||||
selectedAudioInput.value = audioInputs.value[0]?.deviceId
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
// Watch for VAD model toggle
|
||||
watch([useVADModel, currentSource], ([enabled, source]) => {
|
||||
if (source?.vad?.isEnabled) {
|
||||
source.vad.isEnabled.value = enabled
|
||||
}
|
||||
})
|
||||
|
||||
// Sync playback settings
|
||||
watch(enabledPlayback, (enabled) => {
|
||||
playbackEnabled.value = enabled
|
||||
})
|
||||
|
||||
watch(monitorVolume, (volume) => {
|
||||
playbackVolume.value = volume
|
||||
})
|
||||
|
||||
// Monitoring toggle
|
||||
async function toggleMonitoring() {
|
||||
if (!selectedAudioInputSourceId.value)
|
||||
return
|
||||
|
||||
if (enabledMonitoring.value) {
|
||||
await audioManager.startSource(selectedAudioInputSourceId.value)
|
||||
}
|
||||
else {
|
||||
await audioManager.stopSource(selectedAudioInputSourceId.value)
|
||||
}
|
||||
}
|
||||
|
||||
// Initialize with first available device
|
||||
onMounted(async () => {
|
||||
await devices.ensurePermissions()
|
||||
await nextTick()
|
||||
|
||||
if (audioInputs.value.length > 0 && !selectedAudioInput.value) {
|
||||
selectedAudioInput.value = audioInputs.value[0]?.deviceId
|
||||
}
|
||||
})
|
||||
|
||||
// Cleanup on unmount
|
||||
onUnmounted(async () => {
|
||||
if (selectedAudioInputSourceId.value) {
|
||||
await audioManager.stopSource(selectedAudioInputSourceId.value)
|
||||
audioManager.removeSource(selectedAudioInputSourceId.value)
|
||||
onUnmounted(() => {
|
||||
stopAudioMonitoring()
|
||||
if (vadManager.value) {
|
||||
vadManager.value.dispose()
|
||||
}
|
||||
})
|
||||
</script>
|
||||
@@ -176,11 +333,16 @@ onUnmounted(async () => {
|
||||
<!-- Audio Input Selection -->
|
||||
<div>
|
||||
<FieldSelect
|
||||
v-model="selectedAudioInput" label="Audio Input Device"
|
||||
description="Select the audio input device for your hearing module." :options="audioInputs.map(input => ({
|
||||
v-model="selectedAudioInput"
|
||||
label="Audio Input Device"
|
||||
description="Select the audio input device for your hearing module."
|
||||
:options="audioInputs.map(input => ({
|
||||
label: input.label || input.deviceId,
|
||||
value: input.deviceId,
|
||||
}))" placeholder="Select an audio input device"
|
||||
}))"
|
||||
placeholder="Select an audio input device"
|
||||
layout="vertical"
|
||||
h-fit w-full
|
||||
/>
|
||||
</div>
|
||||
|
||||
@@ -194,43 +356,59 @@ onUnmounted(async () => {
|
||||
<div class="space-y-4">
|
||||
<!-- Start/Stop Monitoring -->
|
||||
<FieldCheckbox
|
||||
v-model="enabledMonitoring" label="Enable Audio Monitoring"
|
||||
v-model="isMonitoring"
|
||||
label="Enable Audio Monitoring"
|
||||
description="Start monitoring audio input levels and voice activity detection"
|
||||
@update:model-value="toggleMonitoring"
|
||||
/>
|
||||
|
||||
<!-- Audio Level Visualization -->
|
||||
<div v-if="enabledMonitoring && currentSource" class="space-y-3">
|
||||
<div v-if="isMonitoring" class="space-y-3">
|
||||
<!-- Volume Meter -->
|
||||
<LevelMeter :level="volumeLevel" label="Input Level" />
|
||||
|
||||
<!-- VAD Probability Meter (when VAD model is active) -->
|
||||
<ThresholdMeter
|
||||
v-if="useVADModel && isVADModelLoaded" :value="vadProbability" :threshold="vadThreshold"
|
||||
label="Probability of Speech" below-label="Silence" above-label="Speech"
|
||||
v-if="useVADModel && isVADModelLoaded"
|
||||
:value="vadProbability"
|
||||
:threshold="vadThreshold"
|
||||
label="Probability of Speech"
|
||||
below-label="Silence"
|
||||
above-label="Speech"
|
||||
threshold-label="Detection threshold"
|
||||
/>
|
||||
|
||||
<!-- Threshold Controls -->
|
||||
<div v-if="useVADModel && isVADModelLoaded" class="space-y-3">
|
||||
<FieldRange
|
||||
v-model="vadThreshold" label="Sensitivity"
|
||||
description="Adjust the threshold for speech detection" :min="0.1" :max="0.9" :step="0.0001"
|
||||
v-model="vadThreshold"
|
||||
label="Sensitivity"
|
||||
description="Adjust the threshold for speech detection"
|
||||
:min="0.1"
|
||||
:max="0.9"
|
||||
:step="0.05"
|
||||
:format-value="value => `${(value * 100).toFixed(0)}%`"
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div v-else class="space-y-3">
|
||||
<FieldRange
|
||||
v-model="speakingThreshold" label="Sensitivity"
|
||||
description="Adjust the threshold for speech detection" :min="1" :max="80" :step="1"
|
||||
v-model="speakingThreshold"
|
||||
label="Sensitivity"
|
||||
description="Adjust the threshold for speech detection"
|
||||
:min="1"
|
||||
:max="80"
|
||||
:step="1"
|
||||
:format-value="value => `${value}%`"
|
||||
/>
|
||||
</div>
|
||||
|
||||
<!-- Speaking Indicator -->
|
||||
<div class="flex items-center gap-3">
|
||||
<div class="h-4 w-4 rounded-full transition-all duration-200" :class="speakingIndicatorClass" />
|
||||
<div
|
||||
class="h-4 w-4 rounded-full transition-all duration-200"
|
||||
:class="speakingIndicatorClass"
|
||||
/>
|
||||
<span class="text-sm font-medium">
|
||||
{{ isSpeaking ? 'Speaking Detected' : 'Silence' }}
|
||||
</span>
|
||||
@@ -242,7 +420,8 @@ onUnmounted(async () => {
|
||||
<!-- VAD Method Selection -->
|
||||
<div class="border-t border-neutral-200 pt-3 dark:border-neutral-700">
|
||||
<FieldCheckbox
|
||||
v-model="useVADModel" label="Model Based"
|
||||
v-model="useVADModel"
|
||||
label="Model Based"
|
||||
description="Use AI models for more accurate speech detection"
|
||||
/>
|
||||
|
||||
@@ -253,10 +432,8 @@ onUnmounted(async () => {
|
||||
<span class="text-sm">Loading...</span>
|
||||
</div>
|
||||
|
||||
<div
|
||||
v-else-if="vadModelError"
|
||||
class="flex items-center gap-2 whitespace-break-spaces break-anywhere text-red-600 dark:text-red-400"
|
||||
>
|
||||
<div v-else-if="vadModelError" class="flex items-center gap-2 text-red-600 dark:text-red-400">
|
||||
<div class="text-sm" i-solar:close-circle-bold-duotone />
|
||||
<span class="text-sm">Inference error: {{ vadModelError }}</span>
|
||||
</div>
|
||||
|
||||
@@ -272,34 +449,43 @@ onUnmounted(async () => {
|
||||
|
||||
<!-- Voice Activity Visualization (when VAD model is active) -->
|
||||
<TimeSeriesChart
|
||||
v-if="useVADModel && isVADModelLoaded" :history="vadHistory" :current-value="vadProbability"
|
||||
:threshold="vadThreshold" :is-active="isSpeaking" title="Voice Activity" subtitle="Last 2 seconds"
|
||||
active-label="Speaking" active-legend-label="Voice detected" inactive-legend-label="Silence"
|
||||
v-if="useVADModel && isVADModelLoaded"
|
||||
:history="vadHistory"
|
||||
:current-value="vadProbability"
|
||||
:threshold="vadThreshold"
|
||||
:is-active="isSpeaking"
|
||||
title="Voice Activity"
|
||||
subtitle="Last 2 seconds"
|
||||
active-label="Speaking"
|
||||
active-legend-label="Voice detected"
|
||||
inactive-legend-label="Silence"
|
||||
threshold-label="Speech threshold"
|
||||
/>
|
||||
</div>
|
||||
|
||||
<!-- Audio Playback (Monitor) -->
|
||||
<div v-if="enabledMonitoring && currentSource" class="border-t border-neutral-200 pt-4 dark:border-neutral-700">
|
||||
<div v-if="isMonitoring" class="border-t border-neutral-200 pt-4 dark:border-neutral-700">
|
||||
<FieldCheckbox
|
||||
v-model="enabledPlayback" label="Monitor Audio (Listen)"
|
||||
v-model="enablePlayback"
|
||||
label="Monitor Audio (Listen)"
|
||||
description="Enable audio playback monitoring (like OBS). Be careful of feedback!"
|
||||
/>
|
||||
|
||||
<div v-if="enabledPlayback" class="mt-3">
|
||||
<div v-if="enablePlayback" class="mt-3">
|
||||
<FieldRange
|
||||
v-model="monitorVolume" label="Monitor Volume"
|
||||
description="Control the volume of audio monitoring playback" :min="0" :max="100" :step="5"
|
||||
v-model="monitorVolume"
|
||||
label="Monitor Volume"
|
||||
description="Control the volume of audio monitoring playback"
|
||||
:min="0"
|
||||
:max="100"
|
||||
:step="5"
|
||||
:format-value="value => `${value}%`"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Warning for playback -->
|
||||
<div
|
||||
v-if="enabledPlayback"
|
||||
class="border border-amber-200 rounded-lg bg-amber-50 p-3 dark:border-amber-800 dark:bg-amber-900/20"
|
||||
>
|
||||
<div v-if="enablePlayback" class="border border-amber-200 rounded-lg bg-amber-50 p-3 dark:border-amber-800 dark:bg-amber-900/20">
|
||||
<div class="flex items-center gap-2 text-amber-700 dark:text-amber-300">
|
||||
<div class="text-sm" i-solar:warning-circle-bold-duotone />
|
||||
<span class="text-sm font-medium">Audio feedback warning</span>
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
import type { InvokeArgs, InvokeOptions } from '@tauri-apps/api/core'
|
||||
|
||||
import { invoke as tauriInvoke } from '@tauri-apps/api/core'
|
||||
|
||||
export interface InvokeMethods {
|
||||
// app windows
|
||||
'open_settings_window': { args: undefined, options: undefined, returns: void }
|
||||
'open_chat_window': { args: undefined, options: undefined, returns: void }
|
||||
|
||||
// Plugin - Audio Transcription
|
||||
'plugin:proj-airi-tauri-plugin-audio-transcription|load_model_whisper': { args: { modelType: 'base' | 'largev3' | 'tiny' | 'medium' }, options: undefined, returns: void }
|
||||
'plugin:proj-airi-tauri-plugin-audio-transcription|audio_transcription': { args: { chunk: number[], language: string }, options: undefined, returns: [string, string] }
|
||||
|
||||
// Plugin - Audio VAD
|
||||
'plugin:proj-airi-tauri-plugin-audio-vad|load_model_silero_vad': { args: undefined, options: undefined, returns: void }
|
||||
'plugin:proj-airi-tauri-plugin-audio-vad|audio_vad': { args: { inputData: { input: number[], sr: number, state: number[] } }, options: undefined, returns: number }
|
||||
|
||||
// Plugin - Window Pass through on hover
|
||||
'plugin:proj-airi-tauri-plugin-window-pass-through-on-hover|start_monitor': { args: undefined, options: undefined, returns: void }
|
||||
'plugin:proj-airi-tauri-plugin-window-pass-through-on-hover|stop_monitor': { args: undefined, options: undefined, returns: void }
|
||||
'plugin:proj-airi-tauri-plugin-window-pass-through-on-hover|start_pass_through': { args: undefined, options: undefined, returns: void }
|
||||
'plugin:proj-airi-tauri-plugin-window-pass-through-on-hover|stop_pass_through': { args: undefined, options: undefined, returns: void }
|
||||
|
||||
// Plugin - WindowRouterLink
|
||||
'plugin:proj-airi-tauri-plugin-window-router-link|go': {
|
||||
args: { route: string, windowLabel?: string } | undefined
|
||||
options: undefined
|
||||
returns: void
|
||||
}
|
||||
}
|
||||
|
||||
export interface InvokeMethodShape {
|
||||
args: InvokeArgs | undefined
|
||||
options: InvokeOptions | undefined
|
||||
returns: any
|
||||
}
|
||||
|
||||
export async function invoke<C extends keyof IM, IM extends Record<keyof IM, InvokeMethodShape> = InvokeMethods>(
|
||||
command: C,
|
||||
args?: IM[C]['args'],
|
||||
options?: IM[C]['options'],
|
||||
): Promise<IM[C]['returns'] | undefined> {
|
||||
return await tauriInvoke(command as string, args as InvokeArgs | undefined, options as InvokeOptions | undefined)
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
export { VADAudioManager } from './manager'
|
||||
export type { VADAudioOptions } from './manager'
|
||||
export { createVAD, VAD } from './vad'
|
||||
export type { VADConfig, VADEventCallback, VADEvents } from './vad'
|
||||
@@ -0,0 +1,171 @@
|
||||
import type { VAD, VADConfig } from './vad'
|
||||
|
||||
export interface VADAudioOptions {
|
||||
/**
|
||||
* Audio context options
|
||||
*/
|
||||
audioContextOptions?: AudioContextOptions
|
||||
|
||||
/**
|
||||
* The minimum size of audio chunks to process
|
||||
*/
|
||||
minChunkSize?: number
|
||||
|
||||
/**
|
||||
* VAD configuration options
|
||||
*/
|
||||
vadConfig?: Partial<VADConfig>
|
||||
}
|
||||
|
||||
/**
|
||||
* Manages audio input and worklet processing for the VAD module
|
||||
*/
|
||||
export class VADAudioManager {
|
||||
private audioContext: AudioContext | null = null
|
||||
private audioWorkletNode: AudioWorkletNode | null = null
|
||||
private mediaStream: MediaStream | null = null
|
||||
private sourceNode: MediaStreamAudioSourceNode | null = null
|
||||
private vad: VAD // Updated type
|
||||
private workletInitialized: boolean = false
|
||||
|
||||
/**
|
||||
* Create a new VAD audio manager
|
||||
*/
|
||||
constructor(vad: VAD, options: VADAudioOptions = {}) { // Updated parameter type
|
||||
this.vad = vad
|
||||
|
||||
// Create audio context with user options or defaults
|
||||
this.audioContext = new AudioContext(options.audioContextOptions || {
|
||||
sampleRate: 16000, // Match the VAD sample rate
|
||||
latencyHint: 'interactive',
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize the audio worklet and connect to microphone
|
||||
*/
|
||||
public async initialize(workletUrl: string): Promise<void> {
|
||||
if (!this.audioContext) {
|
||||
throw new Error('Audio context not created')
|
||||
}
|
||||
|
||||
try {
|
||||
if (!this.workletInitialized) {
|
||||
await this.audioContext.audioWorklet.addModule(workletUrl)
|
||||
URL.revokeObjectURL(workletUrl)
|
||||
this.workletInitialized = true
|
||||
}
|
||||
|
||||
// Create the worklet node
|
||||
this.audioWorkletNode = new AudioWorkletNode(this.audioContext, 'vad-processor')
|
||||
|
||||
// Set up message handling from the worklet
|
||||
this.audioWorkletNode.port.onmessage = async (event) => {
|
||||
const { buffer } = event.data
|
||||
if (buffer && buffer.length > 0) {
|
||||
await this.vad.processAudio(new Float32Array(buffer))
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (error) {
|
||||
console.error('Failed to initialize audio worklet:', error)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Start capturing audio from the microphone
|
||||
*/
|
||||
public async startMicrophone(): Promise<void> {
|
||||
if (!this.audioContext || !this.audioWorkletNode) {
|
||||
throw new Error('Audio system not initialized. Call initialize() first.')
|
||||
}
|
||||
|
||||
try {
|
||||
if (this.audioContext.state === 'suspended') {
|
||||
await this.audioContext.resume()
|
||||
}
|
||||
|
||||
// Request microphone access
|
||||
this.mediaStream = await navigator.mediaDevices.getUserMedia({
|
||||
audio: {
|
||||
echoCancellation: true,
|
||||
noiseSuppression: true,
|
||||
autoGainControl: true,
|
||||
sampleRate: this.audioContext.sampleRate,
|
||||
},
|
||||
})
|
||||
|
||||
// Create source node and connect to worklet
|
||||
this.sourceNode = this.audioContext.createMediaStreamSource(this.mediaStream)
|
||||
this.sourceNode.connect(this.audioWorkletNode)
|
||||
|
||||
// Connect worklet to a silent destination (to keep the audio graph active)
|
||||
// Using a GainNode with gain=0 to ensure no sound is output
|
||||
const silentGain = this.audioContext.createGain()
|
||||
silentGain.gain.value = 0
|
||||
this.audioWorkletNode.connect(silentGain)
|
||||
silentGain.connect(this.audioContext.destination)
|
||||
}
|
||||
catch (error) {
|
||||
console.error('Failed to start microphone:', error)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
public async stopMicrophone(): Promise<void> {
|
||||
if (this.mediaStream) {
|
||||
this.mediaStream.getTracks().forEach(track => track.stop())
|
||||
this.mediaStream = null
|
||||
}
|
||||
|
||||
if (this.sourceNode) {
|
||||
this.sourceNode.disconnect()
|
||||
this.sourceNode = null
|
||||
}
|
||||
|
||||
this.audioContext?.suspend()
|
||||
this.audioWorkletNode?.disconnect()
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop capturing audio
|
||||
*/
|
||||
public stop(): void {
|
||||
// Disconnect nodes
|
||||
if (this.sourceNode && this.audioWorkletNode) {
|
||||
this.sourceNode.disconnect()
|
||||
this.audioWorkletNode.disconnect()
|
||||
}
|
||||
|
||||
// Stop all tracks in the media stream
|
||||
if (this.mediaStream) {
|
||||
this.mediaStream.getTracks().forEach(track => track.stop())
|
||||
this.mediaStream = null
|
||||
}
|
||||
|
||||
// Suspend the audio context rather than closing it
|
||||
// This allows us to reuse it later
|
||||
if (this.audioContext && this.audioContext.state !== 'closed') {
|
||||
this.audioContext.suspend()
|
||||
}
|
||||
|
||||
this.sourceNode = null
|
||||
this.audioWorkletNode = null
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean up all resources
|
||||
*/
|
||||
public dispose(): void {
|
||||
this.stop()
|
||||
|
||||
// Now fully close the audio context
|
||||
if (this.audioContext && this.audioContext.state !== 'closed') {
|
||||
this.audioContext.close()
|
||||
this.audioContext = null
|
||||
}
|
||||
|
||||
this.workletInitialized = false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
// vad-worklet-processor.ts
|
||||
// This file needs to be registered as an AudioWorklet
|
||||
|
||||
/**
|
||||
* Minimum chunk size for processing audio
|
||||
*/
|
||||
const MIN_CHUNK_SIZE = 512
|
||||
|
||||
/**
|
||||
* Global state for audio buffer accumulation
|
||||
*/
|
||||
let globalPointer = 0
|
||||
const globalBuffer = new Float32Array(MIN_CHUNK_SIZE)
|
||||
|
||||
/**
|
||||
* VAD AudioWorklet Processor - processes audio chunks and sends them to the main thread
|
||||
*/
|
||||
class VADProcessor extends AudioWorkletProcessor {
|
||||
process(inputs: Float32Array[][], _outputs: Float32Array[][], _parameters: Record<string, Float32Array>) {
|
||||
const buffer = inputs[0][0]
|
||||
if (!buffer)
|
||||
return true // buffer is null when the stream ends
|
||||
|
||||
if (buffer.length > MIN_CHUNK_SIZE) {
|
||||
// If the buffer is larger than the minimum chunk size, send the entire buffer
|
||||
this.port.postMessage({ buffer })
|
||||
}
|
||||
else {
|
||||
const remaining = MIN_CHUNK_SIZE - globalPointer
|
||||
if (buffer.length >= remaining) {
|
||||
// If the buffer is larger than (or equal to) the remaining space in the global buffer, copy the remaining space
|
||||
globalBuffer.set(buffer.subarray(0, remaining), globalPointer)
|
||||
|
||||
// Send the global buffer
|
||||
this.port.postMessage({ buffer: globalBuffer })
|
||||
|
||||
// Reset the global buffer and set the remaining buffer
|
||||
globalBuffer.fill(0)
|
||||
globalBuffer.set(buffer.subarray(remaining), 0)
|
||||
globalPointer = buffer.length - remaining
|
||||
}
|
||||
else {
|
||||
// If the buffer is smaller than the remaining space in the global buffer, copy the buffer to the global buffer
|
||||
globalBuffer.set(buffer, globalPointer)
|
||||
globalPointer += buffer.length
|
||||
}
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
registerProcessor('vad-processor', VADProcessor)
|
||||
@@ -0,0 +1,236 @@
|
||||
import { invoke } from '../invoke'
|
||||
|
||||
export interface VADConfig {
|
||||
sampleRate: number
|
||||
speechThreshold: number
|
||||
exitThreshold: number
|
||||
minSilenceDurationMs: number
|
||||
speechPadMs: number
|
||||
minSpeechDurationMs: number
|
||||
maxBufferDuration: number
|
||||
newBufferSize: number
|
||||
}
|
||||
|
||||
export interface VADEvents {
|
||||
'speech-start': void
|
||||
'speech-end': void
|
||||
'speech-ready': { buffer: Float32Array, duration: number }
|
||||
'status': { type: string, message: string }
|
||||
'debug': { message: string, data?: any }
|
||||
}
|
||||
|
||||
export type VADEventCallback<K extends keyof VADEvents> = (event: VADEvents[K]) => void
|
||||
|
||||
export class VAD {
|
||||
private config: VADConfig
|
||||
private state: Float32Array = new Float32Array(2 * 1 * 128) // 2, 1, 128
|
||||
private buffer: Float32Array
|
||||
private bufferPointer: number = 0
|
||||
private isRecording: boolean = false
|
||||
private postSpeechSamples: number = 0
|
||||
private prevBuffers: Float32Array[] = []
|
||||
private inferenceChain: Promise<any> = Promise.resolve()
|
||||
private eventListeners: Partial<Record<keyof VADEvents, VADEventCallback<any>[]>> = {}
|
||||
private isReady: boolean = false
|
||||
|
||||
constructor(userConfig: Partial<VADConfig> = {}) {
|
||||
const defaultConfig: VADConfig = {
|
||||
sampleRate: 16000,
|
||||
speechThreshold: 0.3,
|
||||
exitThreshold: 0.1,
|
||||
minSilenceDurationMs: 400,
|
||||
speechPadMs: 80,
|
||||
minSpeechDurationMs: 250,
|
||||
maxBufferDuration: 30,
|
||||
newBufferSize: 512,
|
||||
}
|
||||
|
||||
this.config = { ...defaultConfig, ...userConfig }
|
||||
this.buffer = new Float32Array(this.config.maxBufferDuration * this.config.sampleRate)
|
||||
}
|
||||
|
||||
public async initialize(): Promise<void> {
|
||||
try {
|
||||
this.emit('status', { type: 'info', message: 'Loading VAD model...' })
|
||||
|
||||
await invoke('plugin:proj-airi-tauri-plugin-audio-vad|load_model_silero_vad')
|
||||
this.isReady = true
|
||||
|
||||
this.emit('status', { type: 'info', message: 'VAD model loaded successfully' })
|
||||
}
|
||||
catch (error) {
|
||||
this.emit('status', { type: 'error', message: `Failed to load VAD model: ${error}` })
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
public on<K extends keyof VADEvents>(event: K, callback: VADEventCallback<K>): void {
|
||||
if (!this.eventListeners[event]) {
|
||||
this.eventListeners[event] = []
|
||||
}
|
||||
this.eventListeners[event]!.push(callback as any)
|
||||
}
|
||||
|
||||
public off<K extends keyof VADEvents>(event: K, callback: VADEventCallback<K>): void {
|
||||
if (!this.eventListeners[event])
|
||||
return
|
||||
this.eventListeners[event] = this.eventListeners[event]!.filter(cb => cb !== callback)
|
||||
}
|
||||
|
||||
private emit<K extends keyof VADEvents>(event: K, data: VADEvents[K]): void {
|
||||
if (!this.eventListeners[event])
|
||||
return
|
||||
for (const callback of this.eventListeners[event]!) {
|
||||
callback(data)
|
||||
}
|
||||
}
|
||||
|
||||
public async processAudio(inputBuffer: Float32Array): Promise<void> {
|
||||
if (!this.isReady) {
|
||||
throw new Error('VAD model is not initialized. Call initialize() first.')
|
||||
}
|
||||
|
||||
const wasRecording = this.isRecording
|
||||
|
||||
// Perform VAD using Rust backend
|
||||
const isSpeech = await this.detectSpeech(inputBuffer)
|
||||
|
||||
// The rest of the logic remains the same as your original implementation
|
||||
const sampleRateMs = this.config.sampleRate / 1000
|
||||
const minSilenceDurationSamples = this.config.minSilenceDurationMs * sampleRateMs
|
||||
const speechPadSamples = this.config.speechPadMs * sampleRateMs
|
||||
const minSpeechDurationSamples = this.config.minSpeechDurationMs * sampleRateMs
|
||||
const maxPrevBuffers = Math.ceil(speechPadSamples / this.config.newBufferSize)
|
||||
|
||||
if (!wasRecording && !isSpeech) {
|
||||
if (this.prevBuffers.length >= maxPrevBuffers) {
|
||||
this.prevBuffers.shift()
|
||||
}
|
||||
this.prevBuffers.push(inputBuffer.slice(0))
|
||||
return
|
||||
}
|
||||
|
||||
const remaining = this.buffer.length - this.bufferPointer
|
||||
if (inputBuffer.length >= remaining) {
|
||||
this.buffer.set(inputBuffer.subarray(0, remaining), this.bufferPointer)
|
||||
this.bufferPointer += remaining
|
||||
|
||||
const overflow = inputBuffer.subarray(remaining)
|
||||
this.processSpeechSegment(overflow)
|
||||
return
|
||||
}
|
||||
else {
|
||||
this.buffer.set(inputBuffer, this.bufferPointer)
|
||||
this.bufferPointer += inputBuffer.length
|
||||
}
|
||||
|
||||
if (isSpeech) {
|
||||
if (!this.isRecording) {
|
||||
this.emit('speech-start', undefined)
|
||||
this.emit('status', { type: 'info', message: 'Speech detected' })
|
||||
}
|
||||
|
||||
this.isRecording = true
|
||||
this.postSpeechSamples = 0
|
||||
return
|
||||
}
|
||||
|
||||
this.postSpeechSamples += inputBuffer.length
|
||||
|
||||
if (this.postSpeechSamples >= minSilenceDurationSamples) {
|
||||
if (this.bufferPointer < minSpeechDurationSamples) {
|
||||
this.reset()
|
||||
return
|
||||
}
|
||||
|
||||
this.processSpeechSegment()
|
||||
}
|
||||
}
|
||||
|
||||
private async detectSpeech(buffer: Float32Array): Promise<boolean> {
|
||||
// Use Rust backend for inference
|
||||
const result = await (this.inferenceChain = this.inferenceChain.then(() =>
|
||||
invoke('plugin:proj-airi-tauri-plugin-audio-vad|audio_vad', {
|
||||
inputData: {
|
||||
input: Array.from(buffer),
|
||||
sr: this.config.sampleRate,
|
||||
state: Array.from(this.state),
|
||||
},
|
||||
}),
|
||||
)) as { output: number[], state: number[] }
|
||||
|
||||
// Update the state
|
||||
this.state = new Float32Array(result.state)
|
||||
|
||||
// Get the speech probability
|
||||
const speechProb = result.output[0]
|
||||
|
||||
this.emit('debug', {
|
||||
message: 'VAD score',
|
||||
data: { probability: speechProb },
|
||||
})
|
||||
|
||||
// Apply thresholds
|
||||
return (
|
||||
speechProb > this.config.speechThreshold
|
||||
|| (this.isRecording && speechProb >= this.config.exitThreshold)
|
||||
)
|
||||
}
|
||||
|
||||
private processSpeechSegment(overflow?: Float32Array): void {
|
||||
const sampleRateMs = this.config.sampleRate / 1000
|
||||
const speechPadSamples = this.config.speechPadMs * sampleRateMs
|
||||
|
||||
const duration = (this.bufferPointer / this.config.sampleRate) * 1000
|
||||
const overflowLength = overflow?.length ?? 0
|
||||
|
||||
const prevLength = this.prevBuffers.reduce((acc, b) => acc + b.length, 0)
|
||||
const finalBuffer = new Float32Array(prevLength + this.bufferPointer + speechPadSamples)
|
||||
|
||||
let offset = 0
|
||||
for (const prev of this.prevBuffers) {
|
||||
finalBuffer.set(prev, offset)
|
||||
offset += prev.length
|
||||
}
|
||||
|
||||
finalBuffer.set(this.buffer.slice(0, this.bufferPointer + speechPadSamples), offset)
|
||||
|
||||
this.emit('speech-end', undefined)
|
||||
this.emit('speech-ready', {
|
||||
buffer: finalBuffer,
|
||||
duration,
|
||||
})
|
||||
|
||||
if (overflow) {
|
||||
this.buffer.set(overflow, 0)
|
||||
}
|
||||
this.reset(overflowLength)
|
||||
}
|
||||
|
||||
private reset(offset: number = 0): void {
|
||||
this.buffer.fill(0, offset)
|
||||
this.bufferPointer = offset
|
||||
this.isRecording = false
|
||||
this.postSpeechSamples = 0
|
||||
this.prevBuffers = []
|
||||
}
|
||||
|
||||
public updateConfig(newConfig: Partial<VADConfig>): void {
|
||||
this.config = { ...this.config, ...newConfig }
|
||||
|
||||
if (newConfig.maxBufferDuration || newConfig.sampleRate) {
|
||||
this.buffer = new Float32Array(this.config.maxBufferDuration * this.config.sampleRate)
|
||||
this.bufferPointer = 0
|
||||
}
|
||||
}
|
||||
|
||||
public isCurrentlyRecording(): boolean {
|
||||
return this.isRecording
|
||||
}
|
||||
}
|
||||
|
||||
export async function createVAD(config?: Partial<VADConfig>): Promise<VAD> {
|
||||
const vad = new VAD(config)
|
||||
await vad.initialize()
|
||||
return vad
|
||||
}
|
||||
@@ -17,7 +17,8 @@
|
||||
"vite/client",
|
||||
"vite-plugin-vue-layouts/client",
|
||||
"unplugin-vue-macros/macros-global",
|
||||
"unplugin-vue-router/client"
|
||||
"unplugin-vue-router/client",
|
||||
"@types/audioworklet"
|
||||
],
|
||||
"allowJs": true,
|
||||
"strict": true,
|
||||
|
||||
Generated
+3
@@ -321,6 +321,9 @@ importers:
|
||||
'@tresjs/core':
|
||||
specifier: ^4.3.6
|
||||
version: 4.3.6(three@0.177.0)(typescript@5.8.3)(vue@3.5.17(typescript@5.8.3))
|
||||
'@types/audioworklet':
|
||||
specifier: ^0.0.77
|
||||
version: 0.0.77
|
||||
'@vueuse/core':
|
||||
specifier: ^13.4.0
|
||||
version: 13.4.0(vue@3.5.17(typescript@5.8.3))
|
||||
|
||||
Reference in New Issue
Block a user