From 206e5e580c95443d7d35fbfb4ac67fc7f1cd9ac8 Mon Sep 17 00:00:00 2001 From: Neko Ayaka Date: Tue, 1 Jul 2025 15:59:07 +0800 Subject: [PATCH] feat(stage-tamagotchi): tauri-plugin-audio-transcription & tauri-plugin-audio-vad --- apps/stage-tamagotchi/src-tauri/build.rs | 12 ++ .../src-tauri/capabilities/default.json | 4 +- .../stage-tamagotchi/src-tauri/src/app/mod.rs | 1 + .../src-tauri/src/app/models/mod.rs | 29 +++++ .../vad.rs => app/models/silero_vad/mod.rs} | 21 +++- .../{ => app/models}/whisper/melfilters.bytes | Bin .../models}/whisper/melfilters128.bytes | Bin .../whisper.rs => app/models/whisper/mod.rs} | 21 ++-- .../src-tauri/src/helpers/huggingface.rs | 104 ++++++++++++++++++ .../src-tauri/src/helpers/mod.rs | 1 + apps/stage-tamagotchi/src-tauri/src/lib.rs | 41 ++----- .../src/plugins/audio_transcription/mod.rs | 38 +++++++ .../src-tauri/src/plugins/audio_vad/mod.rs | 39 +++++++ .../src-tauri/src/plugins/mod.rs | 2 + .../src-tauri/src/whisper/mod.rs | 4 - .../src-tauri/src/whisper/model_manager.rs | 43 -------- .../src-tauri/src/whisper/progress.rs | 100 ----------------- .../stage-tamagotchi/src/composables/tauri.ts | 9 +- apps/stage-tamagotchi/src/pages/index.vue | 3 +- 19 files changed, 272 insertions(+), 200 deletions(-) create mode 100644 apps/stage-tamagotchi/src-tauri/src/app/models/mod.rs rename apps/stage-tamagotchi/src-tauri/src/{whisper/vad.rs => app/models/silero_vad/mod.rs} (79%) rename apps/stage-tamagotchi/src-tauri/src/{ => app/models}/whisper/melfilters.bytes (100%) rename apps/stage-tamagotchi/src-tauri/src/{ => app/models}/whisper/melfilters128.bytes (100%) rename apps/stage-tamagotchi/src-tauri/src/{whisper/whisper.rs => app/models/whisper/mod.rs} (92%) create mode 100644 apps/stage-tamagotchi/src-tauri/src/helpers/huggingface.rs create mode 100644 apps/stage-tamagotchi/src-tauri/src/helpers/mod.rs create mode 100644 apps/stage-tamagotchi/src-tauri/src/plugins/audio_transcription/mod.rs create mode 100644 apps/stage-tamagotchi/src-tauri/src/plugins/audio_vad/mod.rs delete mode 100644 apps/stage-tamagotchi/src-tauri/src/whisper/mod.rs delete mode 100644 apps/stage-tamagotchi/src-tauri/src/whisper/model_manager.rs delete mode 100644 apps/stage-tamagotchi/src-tauri/src/whisper/progress.rs diff --git a/apps/stage-tamagotchi/src-tauri/build.rs b/apps/stage-tamagotchi/src-tauri/build.rs index bc46cc478..cae80c36a 100644 --- a/apps/stage-tamagotchi/src-tauri/build.rs +++ b/apps/stage-tamagotchi/src-tauri/build.rs @@ -1,6 +1,18 @@ fn main() { tauri_build::try_build( tauri_build::Attributes::new() + .plugin( + "proj-airi-tauri-plugin-audio-transcription", + tauri_build::InlinedPlugin::new() + .commands(&["load_model_whisper"]) + .default_permission(tauri_build::DefaultPermissionRule::AllowAllCommands), + ) + .plugin( + "proj-airi-tauri-plugin-audio-vad", + tauri_build::InlinedPlugin::new() + .commands(&["load_model_silero_vad"]) + .default_permission(tauri_build::DefaultPermissionRule::AllowAllCommands), + ) .plugin( "proj-airi-tauri-plugin-window", tauri_build::InlinedPlugin::new() diff --git a/apps/stage-tamagotchi/src-tauri/capabilities/default.json b/apps/stage-tamagotchi/src-tauri/capabilities/default.json index e2f3db9cd..db9869f92 100644 --- a/apps/stage-tamagotchi/src-tauri/capabilities/default.json +++ b/apps/stage-tamagotchi/src-tauri/capabilities/default.json @@ -23,6 +23,8 @@ "proj-airi-tauri-plugin-window-pass-through-on-hover:default", "proj-airi-tauri-plugin-window-persistence:default", "proj-airi-tauri-plugin-window:default", - "proj-airi-tauri-plugin-window-router-link:default" + "proj-airi-tauri-plugin-window-router-link:default", + "proj-airi-tauri-plugin-audio-transcription:default", + "proj-airi-tauri-plugin-audio-vad:default" ] } diff --git a/apps/stage-tamagotchi/src-tauri/src/app/mod.rs b/apps/stage-tamagotchi/src-tauri/src/app/mod.rs index 3c8a18e26..6c2df8cce 100644 --- a/apps/stage-tamagotchi/src-tauri/src/app/mod.rs +++ b/apps/stage-tamagotchi/src-tauri/src/app/mod.rs @@ -1,2 +1,3 @@ pub mod commands; +pub mod models; pub mod windows; diff --git a/apps/stage-tamagotchi/src-tauri/src/app/models/mod.rs b/apps/stage-tamagotchi/src-tauri/src/app/models/mod.rs new file mode 100644 index 000000000..65a6a286a --- /dev/null +++ b/apps/stage-tamagotchi/src-tauri/src/app/models/mod.rs @@ -0,0 +1,29 @@ +pub mod silero_vad; +pub mod whisper; + +use log::info; +use tauri::Runtime; + +use crate::app::models::{ + silero_vad::VADProcessor, + whisper::{WhichWhisperModel, WhisperProcessor}, +}; + +pub fn load_whisper_model( + device: candle_core::Device, + window: tauri::WebviewWindow, +) -> anyhow::Result<()> { + let whisper_model = WhichWhisperModel::Tiny; + info!("Loading whisper model: {:?}", whisper_model); + let _ = WhisperProcessor::new(whisper_model, device.clone(), window)?; + Ok(()) +} + +pub fn load_vad_model( + device: candle_core::Device, + window: tauri::WebviewWindow, +) -> anyhow::Result<()> { + info!("Loading VAD model"); + let _ = VADProcessor::new(device.clone(), 0.3, window)?; + Ok(()) +} diff --git a/apps/stage-tamagotchi/src-tauri/src/whisper/vad.rs b/apps/stage-tamagotchi/src-tauri/src/app/models/silero_vad/mod.rs similarity index 79% rename from apps/stage-tamagotchi/src-tauri/src/whisper/vad.rs rename to apps/stage-tamagotchi/src-tauri/src/app/models/silero_vad/mod.rs index 9a05df5c8..5f91a85cf 100644 --- a/apps/stage-tamagotchi/src-tauri/src/whisper/vad.rs +++ b/apps/stage-tamagotchi/src-tauri/src/app/models/silero_vad/mod.rs @@ -4,8 +4,9 @@ use anyhow::Result; use candle_core::{DType, Device, Tensor}; use candle_onnx::simple_eval; use hf_hub::{Repo, RepoType}; +use tauri::Runtime; -use crate::whisper::progress; +use crate::helpers::huggingface::create_progress_emitter; pub struct VADProcessor { model: candle_onnx::onnx::ModelProto, @@ -19,10 +20,10 @@ pub struct VADProcessor { } impl VADProcessor { - pub fn new( + pub fn new( device: Device, threshold: f32, - manager: progress::ModelLoadProgressEmitterManager, + window: tauri::WebviewWindow, ) -> Result { let api = hf_hub::api::sync::Api::new()?; let repo = api.repo(Repo::with_revision( @@ -33,7 +34,7 @@ impl VADProcessor { let model_path = repo.download_with_progress( "onnx/model.onnx", - manager.clone().new_for("onnx/model.onnx"), + create_progress_emitter(window.clone(), "onnx/model.onnx".to_string()), )?; let model = candle_onnx::read_file(model_path)?; @@ -61,11 +62,19 @@ impl VADProcessor { return Ok(0.0); } - let next_context = Tensor::from_slice(&chunk[self.frame_size - self.context_size..], (1, self.context_size), &self.device)?; + let next_context = Tensor::from_slice( + &chunk[self.frame_size - self.context_size..], + (1, self.context_size), + &self.device, + )?; let chunk_tensor = Tensor::from_vec(chunk.to_vec(), (1, self.frame_size), &self.device)?; let input = Tensor::cat(&[&self.context, &chunk_tensor], 1)?; - let inputs: HashMap = HashMap::from_iter([("input".to_string(), input), ("sr".to_string(), self.sample_rate.clone()), ("state".to_string(), self.state.clone())]); + let inputs: HashMap = HashMap::from_iter([ + ("input".to_string(), input), + ("sr".to_string(), self.sample_rate.clone()), + ("state".to_string(), self.state.clone()), + ]); let outputs = simple_eval(&self.model, inputs)?; let graph = self.model.graph.as_ref().unwrap(); diff --git a/apps/stage-tamagotchi/src-tauri/src/whisper/melfilters.bytes b/apps/stage-tamagotchi/src-tauri/src/app/models/whisper/melfilters.bytes similarity index 100% rename from apps/stage-tamagotchi/src-tauri/src/whisper/melfilters.bytes rename to apps/stage-tamagotchi/src-tauri/src/app/models/whisper/melfilters.bytes diff --git a/apps/stage-tamagotchi/src-tauri/src/whisper/melfilters128.bytes b/apps/stage-tamagotchi/src-tauri/src/app/models/whisper/melfilters128.bytes similarity index 100% rename from apps/stage-tamagotchi/src-tauri/src/whisper/melfilters128.bytes rename to apps/stage-tamagotchi/src-tauri/src/app/models/whisper/melfilters128.bytes diff --git a/apps/stage-tamagotchi/src-tauri/src/whisper/whisper.rs b/apps/stage-tamagotchi/src-tauri/src/app/models/whisper/mod.rs similarity index 92% rename from apps/stage-tamagotchi/src-tauri/src/whisper/whisper.rs rename to apps/stage-tamagotchi/src-tauri/src/app/models/whisper/mod.rs index 0eba9da9a..da8580875 100644 --- a/apps/stage-tamagotchi/src-tauri/src/whisper/whisper.rs +++ b/apps/stage-tamagotchi/src-tauri/src/app/models/whisper/mod.rs @@ -7,9 +7,10 @@ use candle_transformers::models::whisper::{self as whisper_model, Config, audio} use clap::ValueEnum; use hf_hub::{Repo, RepoType, api::sync::ApiBuilder}; use log::info; +use tauri::Runtime; use tokenizers::Tokenizer; -use crate::whisper::progress; +use crate::helpers::huggingface::create_progress_emitter; pub enum WhisperModel { Normal(whisper_model::model::Whisper), @@ -101,10 +102,10 @@ pub struct WhisperProcessor { } impl WhisperProcessor { - pub fn new( + pub fn new( model: WhichWhisperModel, device: Device, - manager: progress::ModelLoadProgressEmitterManager, + window: tauri::WebviewWindow, ) -> Result { // Load the Whisper model based on the provided model type let api = ApiBuilder::new().with_progress(false).build()?; @@ -115,15 +116,19 @@ impl WhisperProcessor { revision.to_string(), )); - let config_filename = - repo.download_with_progress("config.json", manager.clone().new_for("config.json"))?; + let config_filename = repo.download_with_progress( + "config.json", + create_progress_emitter(window.clone(), "config.json".to_string()), + )?; info!("config_filename: {:?}", config_filename.display()); - let tokenizer_filename = - repo.download_with_progress("tokenizer.json", manager.clone().new_for("tokenizer.json"))?; + let tokenizer_filename = repo.download_with_progress( + "tokenizer.json", + create_progress_emitter(window.clone(), "tokenizer.json".to_string()), + )?; info!("tokenizer_filename: {:?}", tokenizer_filename.display()); let model_filename = repo.download_with_progress( "model.safetensors", - manager.clone().new_for("model.safetensors"), + create_progress_emitter(window.clone(), "model.safetensors".to_string()), )?; info!("model_filename: {:?}", model_filename.display()); diff --git a/apps/stage-tamagotchi/src-tauri/src/helpers/huggingface.rs b/apps/stage-tamagotchi/src-tauri/src/helpers/huggingface.rs new file mode 100644 index 000000000..025203b40 --- /dev/null +++ b/apps/stage-tamagotchi/src-tauri/src/helpers/huggingface.rs @@ -0,0 +1,104 @@ +use anyhow::Ok; +use log::{error, info}; +use tauri::{Emitter, Runtime}; + +pub fn load_device() -> anyhow::Result { + // Determine device to use + let device = if candle_core::utils::cuda_is_available() { + candle_core::Device::new_cuda(0)? + } else if candle_core::utils::metal_is_available() { + candle_core::Device::new_metal(0)? + } else { + candle_core::Device::Cpu + }; + + info!("Using device: {device:?}"); + Ok(device) +} + +pub trait ProgressEmitter: Send + Sync { + fn emit_progress( + &self, + filename: String, + progress: f32, + ); +} + +pub struct ModelLoadProgressEmitter { + filename: String, + size: usize, + total_size: usize, + progress: f32, + emitter: Box, +} + +impl ModelLoadProgressEmitter { + pub fn new( + emitter: Box, + filename: String, + ) -> Self { + Self { + filename, + size: 0, + total_size: 0, + progress: 0.0, + emitter, + } + } +} + +impl ProgressEmitter for tauri::WebviewWindow { + fn emit_progress( + &self, + filename: String, + progress: f32, + ) { + if let Err(err) = self.emit("tauri-app:model-load-progress", (filename, progress)) { + error!("Failed to emit model-load-progress: {:?}", err); + } + } +} + +pub fn create_progress_emitter( + window: impl ProgressEmitter + 'static, + filename: String, +) -> ModelLoadProgressEmitter { + ModelLoadProgressEmitter::new(Box::new(window), filename) +} + +// Remove the generic since ModelLoadProgressEmitter no longer has generics +impl hf_hub::api::Progress for ModelLoadProgressEmitter { + fn init( + &mut self, + size: usize, + _: &str, + ) { + self.total_size = size; + self.progress = 0.0; + self + .emitter + .emit_progress(self.filename.clone(), self.progress); + } + + fn update( + &mut self, + size: usize, + ) { + self.size += size; + self.progress = if self.total_size > 0 { + (self.size as f32 / self.total_size as f32 * 100.0).min(100.0) + } else { + 100.0 + }; + self + .emitter + .emit_progress(self.filename.clone(), self.progress); + } + + fn finish(&mut self) { + self.progress = 100.0; + self + .emitter + .emit_progress(self.filename.clone(), self.progress); + } +} diff --git a/apps/stage-tamagotchi/src-tauri/src/helpers/mod.rs b/apps/stage-tamagotchi/src-tauri/src/helpers/mod.rs new file mode 100644 index 000000000..bb5c9337c --- /dev/null +++ b/apps/stage-tamagotchi/src-tauri/src/helpers/mod.rs @@ -0,0 +1 @@ +pub mod huggingface; diff --git a/apps/stage-tamagotchi/src-tauri/src/lib.rs b/apps/stage-tamagotchi/src-tauri/src/lib.rs index ddd37a699..c90bef8eb 100644 --- a/apps/stage-tamagotchi/src-tauri/src/lib.rs +++ b/apps/stage-tamagotchi/src-tauri/src/lib.rs @@ -1,4 +1,3 @@ -use log::info; use tauri::{ Emitter, Manager, @@ -10,54 +9,30 @@ use tauri::{ use tauri_plugin_prevent_default::Flags; mod app; +mod helpers; mod plugins; -mod whisper; - -fn load_whisper_model(window: tauri::Window) -> anyhow::Result<()> { - let device = whisper::model_manager::load_device()?; - - whisper::model_manager::load_whisper_model(window.clone(), device.clone())?; - whisper::model_manager::load_vad_model(window.clone(), candle_core::Device::Cpu)?; - - Ok(()) -} - -#[tauri::command] -async fn load_models(window: tauri::Window) -> Result<(), String> { - info!("Loading models..."); - - load_whisper_model(window).map_or_else( - |e| { - let error_message = format!("Failed to load models: {}", e); - info!("{}", error_message); - Err(error_message) - }, - |_| { - info!("Models loaded successfully"); - Ok(()) - }, - ) -} #[cfg_attr(mobile, tauri::mobile_entry_point)] -#[allow(clippy::missing_panics_doc)] pub fn run() { let prevent_default_plugin = tauri_plugin_prevent_default::Builder::new() .with_flags(Flags::RELOAD) .build(); - #[allow(clippy::missing_panics_doc)] tauri::Builder::default() + // External plugins .plugin(prevent_default_plugin) .plugin(tauri_plugin_mcp::Builder.build()) .plugin(tauri_plugin_os::init()) .plugin(tauri_plugin_global_shortcut::Builder::new().build()) .plugin(tauri_plugin_window_state::Builder::default().build()) .plugin(tauri_plugin_positioner::init()) + // Internal plugins .plugin(plugins::window::init()) .plugin(plugins::window_persistence::init()) .plugin(plugins::window_pass_through_on_hover::init()) .plugin(plugins::window_router_link::init()) + .plugin(plugins::audio_transcription::init()) + .plugin(plugins::audio_vad::init()) .setup(|app| { let mut builder = WebviewWindowBuilder::new(app, "main", WebviewUrl::default()) .title("AIRI") @@ -152,9 +127,7 @@ pub fn run() { app::commands::open_settings_window, app::commands::open_chat_window, app::commands::debug_println, - load_models, ]) - .build(tauri::generate_context!()) - .expect("error while building tauri application") - .run(|_, _| {}); + .run(tauri::generate_context!()) + .expect("error while running tauri application"); } diff --git a/apps/stage-tamagotchi/src-tauri/src/plugins/audio_transcription/mod.rs b/apps/stage-tamagotchi/src-tauri/src/plugins/audio_transcription/mod.rs new file mode 100644 index 000000000..d0b10e4f8 --- /dev/null +++ b/apps/stage-tamagotchi/src-tauri/src/plugins/audio_transcription/mod.rs @@ -0,0 +1,38 @@ +use log::info; +use tauri::{ + Runtime, + plugin::{Builder as PluginBuilder, TauriPlugin}, +}; + +use crate::{app::models::load_whisper_model, helpers::huggingface::load_device}; + +#[tauri::command] +pub async fn load_model_whisper(window: tauri::WebviewWindow) -> Result<(), String> { + let device = match load_device() { + Ok(device) => device, + Err(e) => { + let error_message = format!("Failed to load device: {}", e); + info!("{}", error_message); + return Err(error_message); + }, + }; + + info!("Loading models..."); + + // Load the traditional whisper models first + if let Err(e) = load_whisper_model(device.clone(), window) { + let error_message = format!("Failed to load whisper models: {}", e); + info!("{}", error_message); + return Err(error_message); + } + + info!("All models loaded successfully"); + Ok(()) +} + +pub fn init() -> TauriPlugin { + PluginBuilder::new("proj-airi-tauri-plugin-audio-transcription") + .setup(|_, _| Ok(())) + .invoke_handler(tauri::generate_handler![load_model_whisper]) + .build() +} diff --git a/apps/stage-tamagotchi/src-tauri/src/plugins/audio_vad/mod.rs b/apps/stage-tamagotchi/src-tauri/src/plugins/audio_vad/mod.rs new file mode 100644 index 000000000..a096028be --- /dev/null +++ b/apps/stage-tamagotchi/src-tauri/src/plugins/audio_vad/mod.rs @@ -0,0 +1,39 @@ +use log::info; +use tauri::{ + Runtime, + plugin::{Builder as PluginBuilder, TauriPlugin}, +}; + +use crate::{app::models::load_vad_model, helpers::huggingface::load_device}; + +#[tauri::command] +pub async fn load_model_silero_vad( + window: tauri::WebviewWindow +) -> Result<(), String> { + let device = match load_device() { + Ok(device) => device, + Err(e) => { + let error_message = format!("Failed to load device: {}", e); + info!("{}", error_message); + return Err(error_message); + }, + }; + + info!("Loading models..."); + + if let Err(e) = load_vad_model(device.clone(), window) { + let error_message = format!("Failed to load VAD model: {}", e); + info!("{}", error_message); + return Err(error_message); + } + + info!("All models loaded successfully"); + Ok(()) +} + +pub fn init() -> TauriPlugin { + PluginBuilder::new("proj-airi-tauri-plugin-audio-vad") + .setup(|_, _| Ok(())) + .invoke_handler(tauri::generate_handler![load_model_silero_vad]) + .build() +} diff --git a/apps/stage-tamagotchi/src-tauri/src/plugins/mod.rs b/apps/stage-tamagotchi/src-tauri/src/plugins/mod.rs index 4917d8e0e..3e876469c 100644 --- a/apps/stage-tamagotchi/src-tauri/src/plugins/mod.rs +++ b/apps/stage-tamagotchi/src-tauri/src/plugins/mod.rs @@ -1,3 +1,5 @@ +pub mod audio_transcription; +pub mod audio_vad; pub mod window; pub mod window_pass_through_on_hover; pub mod window_persistence; diff --git a/apps/stage-tamagotchi/src-tauri/src/whisper/mod.rs b/apps/stage-tamagotchi/src-tauri/src/whisper/mod.rs deleted file mode 100644 index d77b22e44..000000000 --- a/apps/stage-tamagotchi/src-tauri/src/whisper/mod.rs +++ /dev/null @@ -1,4 +0,0 @@ -pub mod progress; -pub mod whisper; -pub mod model_manager; -pub mod vad; diff --git a/apps/stage-tamagotchi/src-tauri/src/whisper/model_manager.rs b/apps/stage-tamagotchi/src-tauri/src/whisper/model_manager.rs deleted file mode 100644 index 14e0cbf1c..000000000 --- a/apps/stage-tamagotchi/src-tauri/src/whisper/model_manager.rs +++ /dev/null @@ -1,43 +0,0 @@ -use log::info; -use anyhow::Ok; - -use crate::whisper::whisper::{WhichWhisperModel, WhisperProcessor}; -use crate::whisper::progress::ModelLoadProgressEmitterManager; -use crate::whisper::vad::VADProcessor; - -pub fn load_device() -> anyhow::Result { - // Determine device to use - let device = if candle_core::utils::cuda_is_available() { - candle_core::Device::new_cuda(0)? - } else if candle_core::utils::metal_is_available() { - candle_core::Device::new_metal(0)? - } else { - candle_core::Device::Cpu - }; - - info!("Using device: {device:?}"); - Ok(device) -} - - -pub fn load_whisper_model(window: tauri::Window, device: candle_core::Device) -> anyhow::Result<()> { - let progress_manager = ModelLoadProgressEmitterManager::new(window); - - let whisper_model = WhichWhisperModel::Tiny; - - info!("Loading whisper model: {:?}", whisper_model); - - let _ = WhisperProcessor::new(whisper_model, device.clone(), progress_manager)?; - Ok(()) -} - -pub fn load_vad_model(window: tauri::Window, device: candle_core::Device) -> anyhow::Result<()> { - let progress_manager = ModelLoadProgressEmitterManager::new(window); - - let whisper_model = WhichWhisperModel::Tiny; - - info!("Loading VAD model: {:?}", whisper_model); - - let _ = VADProcessor::new(device.clone(), 0.3, progress_manager)?; - Ok(()) -} diff --git a/apps/stage-tamagotchi/src-tauri/src/whisper/progress.rs b/apps/stage-tamagotchi/src-tauri/src/whisper/progress.rs deleted file mode 100644 index 74e99d560..000000000 --- a/apps/stage-tamagotchi/src-tauri/src/whisper/progress.rs +++ /dev/null @@ -1,100 +0,0 @@ -use log::error; -use tauri::Emitter; - -#[derive(Clone)] -pub struct ModelLoadProgressEmitterManager { - window: tauri::Window, -} - -impl ModelLoadProgressEmitterManager { - pub fn new(window: tauri::Window) -> Self { - Self { window } - } - - pub fn new_for( - self, - filename: &str, - ) -> ModelLoadProgressEmitter { - ModelLoadProgressEmitter::new(self.window, filename) - } -} - -pub struct ModelLoadProgressEmitter { - filename: String, - size: usize, - total_size: usize, - progress: f32, - window: tauri::Window, -} - -impl ModelLoadProgressEmitter { - fn new( - window: tauri::Window, - filename: &str, - ) -> Self { - Self { - filename: filename.to_string(), - size: 0, - total_size: 0, - progress: 0.0, - window, - } - } -} - -impl hf_hub::api::Progress for ModelLoadProgressEmitter { - fn init( - &mut self, - size: usize, - _: &str, - ) { - self.total_size = size; - self.progress = 0.0; - self - .window - .emit( - "tauri-app:model-load-progress", - (self.filename.clone(), self.progress), - ) - .map_err(|err| { - error!("Failed to emit model-load-progress: {:?}", err); - }) - .unwrap(); - } - - fn update( - &mut self, - size: usize, - ) { - self.size += size; - self.progress = if self.total_size > 0 { - (self.size as f32 / self.total_size as f32 * 100.0).min(100.0) - } else { - 100.0 - }; - self - .window - .emit( - "tauri-app:model-load-progress", - (self.filename.clone(), self.progress), - ) - .map_err(|err| { - error!("Failed to emit model-load-progress: {:?}", err); - }) - .unwrap(); - } - - fn finish(&mut self) { - self.progress = 100.0; - self - .window - .emit( - "tauri-app:model-load-progress", - (self.filename.clone(), self.progress), - ) - .map_err(|err| { - error!("Failed to emit model-load-progress: {:?}", err); - }) - .unwrap(); - } -} diff --git a/apps/stage-tamagotchi/src/composables/tauri.ts b/apps/stage-tamagotchi/src/composables/tauri.ts index 53bfd87f9..e5565cc37 100644 --- a/apps/stage-tamagotchi/src/composables/tauri.ts +++ b/apps/stage-tamagotchi/src/composables/tauri.ts @@ -141,13 +141,16 @@ export function useTauriEvent() { } export interface InvokeMethods { - // Model related - 'load_models': { args: undefined, options: undefined, returns: void } - // app windows 'open_settings_window': { args: undefined, options: undefined, returns: void } 'open_chat_window': { args: undefined, options: undefined, returns: void } + // Plugin - Audio Transcription + 'plugin:proj-airi-tauri-plugin-audio-transcription|load_model_whisper': { args: undefined, options: undefined, returns: void } + + // Plugin - Audio VAD + 'plugin:proj-airi-tauri-plugin-audio-vad|load_model_silero_vad': { args: undefined, options: undefined, returns: void } + // Plugin - Window Pass through on hover 'plugin:proj-airi-tauri-plugin-window-pass-through-on-hover|start_monitor': { args: undefined, options: undefined, returns: void } 'plugin:proj-airi-tauri-plugin-window-pass-through-on-hover|stop_monitor': { args: undefined, options: undefined, returns: void } diff --git a/apps/stage-tamagotchi/src/pages/index.vue b/apps/stage-tamagotchi/src/pages/index.vue index a05ac0b83..78bded2d8 100644 --- a/apps/stage-tamagotchi/src/pages/index.vue +++ b/apps/stage-tamagotchi/src/pages/index.vue @@ -83,7 +83,8 @@ onMounted(async () => { })) // Load models - invoke('load_models') + invoke('plugin:proj-airi-tauri-plugin-audio-transcription|load_model_whisper') + invoke('plugin:proj-airi-tauri-plugin-audio-vad|load_model_silero_vad') if (connected.value) return