feat(stage-tamagotchi): tauri-plugin-audio-transcription & tauri-plugin-audio-vad

apps/stage-tamagotchi/src-tauri/build.rs

@@ -1,6 +1,18 @@
 fn main() {
   tauri_build::try_build(
     tauri_build::Attributes::new()
+      .plugin(
+        "proj-airi-tauri-plugin-audio-transcription",
+        tauri_build::InlinedPlugin::new()
+          .commands(&["load_model_whisper"])
+          .default_permission(tauri_build::DefaultPermissionRule::AllowAllCommands),
+      )
+      .plugin(
+        "proj-airi-tauri-plugin-audio-vad",
+        tauri_build::InlinedPlugin::new()
+          .commands(&["load_model_silero_vad"])
+          .default_permission(tauri_build::DefaultPermissionRule::AllowAllCommands),
+      )
       .plugin(
         "proj-airi-tauri-plugin-window",
         tauri_build::InlinedPlugin::new()

apps/stage-tamagotchi/src-tauri/capabilities/default.json

@@ -23,6 +23,8 @@
     "proj-airi-tauri-plugin-window-pass-through-on-hover:default",
     "proj-airi-tauri-plugin-window-persistence:default",
     "proj-airi-tauri-plugin-window:default",
-    "proj-airi-tauri-plugin-window-router-link:default"
+    "proj-airi-tauri-plugin-window-router-link:default",
+    "proj-airi-tauri-plugin-audio-transcription:default",
+    "proj-airi-tauri-plugin-audio-vad:default"
   ]
 }

apps/stage-tamagotchi/src-tauri/src/app/mod.rs

@@ -1,2 +1,3 @@
 pub mod commands;
+pub mod models;
 pub mod windows;

apps/stage-tamagotchi/src-tauri/src/app/models/mod.rs

@@ -0,0 +1,29 @@
+pub mod silero_vad;
+pub mod whisper;
+
+use log::info;
+use tauri::Runtime;
+
+use crate::app::models::{
+  silero_vad::VADProcessor,
+  whisper::{WhichWhisperModel, WhisperProcessor},
+};
+
+pub fn load_whisper_model<R: Runtime>(
+  device: candle_core::Device,
+  window: tauri::WebviewWindow<R>,
+) -> anyhow::Result<()> {
+  let whisper_model = WhichWhisperModel::Tiny;
+  info!("Loading whisper model: {:?}", whisper_model);
+  let _ = WhisperProcessor::new(whisper_model, device.clone(), window)?;
+  Ok(())
+}
+
+pub fn load_vad_model<R: Runtime>(
+  device: candle_core::Device,
+  window: tauri::WebviewWindow<R>,
+) -> anyhow::Result<()> {
+  info!("Loading VAD model");
+  let _ = VADProcessor::new(device.clone(), 0.3, window)?;
+  Ok(())
+}

apps/stage-tamagotchi/src-tauri/src/app/models/silero_vad/mod.rs

@@ -4,8 +4,9 @@ use anyhow::Result;
 use candle_core::{DType, Device, Tensor};
 use candle_onnx::simple_eval;
 use hf_hub::{Repo, RepoType};
+use tauri::Runtime;
 
-use crate::whisper::progress;
+use crate::helpers::huggingface::create_progress_emitter;
 
 pub struct VADProcessor {
   model:        candle_onnx::onnx::ModelProto,
@@ -19,10 +20,10 @@ pub struct VADProcessor {
 }
 
 impl VADProcessor {
-  pub fn new(
+  pub fn new<R: Runtime>(
     device: Device,
     threshold: f32,
-    manager: progress::ModelLoadProgressEmitterManager,
+    window: tauri::WebviewWindow<R>,
   ) -> Result<Self> {
     let api = hf_hub::api::sync::Api::new()?;
     let repo = api.repo(Repo::with_revision(
@@ -33,7 +34,7 @@ impl VADProcessor {
 
     let model_path = repo.download_with_progress(
       "onnx/model.onnx",
-      manager.clone().new_for("onnx/model.onnx"),
+      create_progress_emitter(window.clone(), "onnx/model.onnx".to_string()),
     )?;
 
     let model = candle_onnx::read_file(model_path)?;
@@ -61,11 +62,19 @@ impl VADProcessor {
       return Ok(0.0);
     }
 
-    let next_context = Tensor::from_slice(&chunk[self.frame_size - self.context_size..], (1, self.context_size), &self.device)?;
+    let next_context = Tensor::from_slice(
+      &chunk[self.frame_size - self.context_size..],
+      (1, self.context_size),
+      &self.device,
+    )?;
     let chunk_tensor = Tensor::from_vec(chunk.to_vec(), (1, self.frame_size), &self.device)?;
 
     let input = Tensor::cat(&[&self.context, &chunk_tensor], 1)?;
-    let inputs: HashMap<String, Tensor> = HashMap::from_iter([("input".to_string(), input), ("sr".to_string(), self.sample_rate.clone()), ("state".to_string(), self.state.clone())]);
+    let inputs: HashMap<String, Tensor> = HashMap::from_iter([
+      ("input".to_string(), input),
+      ("sr".to_string(), self.sample_rate.clone()),
+      ("state".to_string(), self.state.clone()),
+    ]);
 
     let outputs = simple_eval(&self.model, inputs)?;
     let graph = self.model.graph.as_ref().unwrap();

apps/stage-tamagotchi/src-tauri/src/app/models/whisper/mod.rs

@@ -7,9 +7,10 @@ use candle_transformers::models::whisper::{self as whisper_model, Config, audio}
 use clap::ValueEnum;
 use hf_hub::{Repo, RepoType, api::sync::ApiBuilder};
 use log::info;
+use tauri::Runtime;
 use tokenizers::Tokenizer;
 
-use crate::whisper::progress;
+use crate::helpers::huggingface::create_progress_emitter;
 
 pub enum WhisperModel {
   Normal(whisper_model::model::Whisper),
@@ -101,10 +102,10 @@ pub struct WhisperProcessor {
 }
 
 impl WhisperProcessor {
-  pub fn new(
+  pub fn new<R: Runtime>(
     model: WhichWhisperModel,
     device: Device,
-    manager: progress::ModelLoadProgressEmitterManager,
+    window: tauri::WebviewWindow<R>,
   ) -> Result<Self> {
     // Load the Whisper model based on the provided model type
     let api = ApiBuilder::new().with_progress(false).build()?;
@@ -115,15 +116,19 @@ impl WhisperProcessor {
       revision.to_string(),
     ));
 
-    let config_filename =
-      repo.download_with_progress("config.json", manager.clone().new_for("config.json"))?;
+    let config_filename = repo.download_with_progress(
+      "config.json",
+      create_progress_emitter(window.clone(), "config.json".to_string()),
+    )?;
     info!("config_filename: {:?}", config_filename.display());
-    let tokenizer_filename =
-      repo.download_with_progress("tokenizer.json", manager.clone().new_for("tokenizer.json"))?;
+    let tokenizer_filename = repo.download_with_progress(
+      "tokenizer.json",
+      create_progress_emitter(window.clone(), "tokenizer.json".to_string()),
+    )?;
     info!("tokenizer_filename: {:?}", tokenizer_filename.display());
     let model_filename = repo.download_with_progress(
       "model.safetensors",
-      manager.clone().new_for("model.safetensors"),
+      create_progress_emitter(window.clone(), "model.safetensors".to_string()),
     )?;
     info!("model_filename: {:?}", model_filename.display());
 

apps/stage-tamagotchi/src-tauri/src/helpers/huggingface.rs

@@ -0,0 +1,104 @@
+use anyhow::Ok;
+use log::{error, info};
+use tauri::{Emitter, Runtime};
+
+pub fn load_device() -> anyhow::Result<candle_core::Device> {
+  // Determine device to use
+  let device = if candle_core::utils::cuda_is_available() {
+    candle_core::Device::new_cuda(0)?
+  } else if candle_core::utils::metal_is_available() {
+    candle_core::Device::new_metal(0)?
+  } else {
+    candle_core::Device::Cpu
+  };
+
+  info!("Using device: {device:?}");
+  Ok(device)
+}
+
+pub trait ProgressEmitter: Send + Sync {
+  fn emit_progress(
+    &self,
+    filename: String,
+    progress: f32,
+  );
+}
+
+pub struct ModelLoadProgressEmitter {
+  filename:   String,
+  size:       usize,
+  total_size: usize,
+  progress:   f32,
+  emitter:    Box<dyn ProgressEmitter>,
+}
+
+impl ModelLoadProgressEmitter {
+  pub fn new(
+    emitter: Box<dyn ProgressEmitter>,
+    filename: String,
+  ) -> Self {
+    Self {
+      filename,
+      size: 0,
+      total_size: 0,
+      progress: 0.0,
+      emitter,
+    }
+  }
+}
+
+impl<R: Runtime> ProgressEmitter for tauri::WebviewWindow<R> {
+  fn emit_progress(
+    &self,
+    filename: String,
+    progress: f32,
+  ) {
+    if let Err(err) = self.emit("tauri-app:model-load-progress", (filename, progress)) {
+      error!("Failed to emit model-load-progress: {:?}", err);
+    }
+  }
+}
+
+pub fn create_progress_emitter(
+  window: impl ProgressEmitter + 'static,
+  filename: String,
+) -> ModelLoadProgressEmitter {
+  ModelLoadProgressEmitter::new(Box::new(window), filename)
+}
+
+// Remove the generic <R: Runtime> since ModelLoadProgressEmitter no longer has generics
+impl hf_hub::api::Progress for ModelLoadProgressEmitter {
+  fn init(
+    &mut self,
+    size: usize,
+    _: &str,
+  ) {
+    self.total_size = size;
+    self.progress = 0.0;
+    self
+      .emitter
+      .emit_progress(self.filename.clone(), self.progress);
+  }
+
+  fn update(
+    &mut self,
+    size: usize,
+  ) {
+    self.size += size;
+    self.progress = if self.total_size > 0 {
+      (self.size as f32 / self.total_size as f32 * 100.0).min(100.0)
+    } else {
+      100.0
+    };
+    self
+      .emitter
+      .emit_progress(self.filename.clone(), self.progress);
+  }
+
+  fn finish(&mut self) {
+    self.progress = 100.0;
+    self
+      .emitter
+      .emit_progress(self.filename.clone(), self.progress);
+  }
+}

apps/stage-tamagotchi/src-tauri/src/helpers/mod.rs

@@ -0,0 +1 @@
+pub mod huggingface;

apps/stage-tamagotchi/src-tauri/src/lib.rs

@@ -1,4 +1,3 @@
-use log::info;
 use tauri::{
   Emitter,
   Manager,
@@ -10,54 +9,30 @@ use tauri::{
 use tauri_plugin_prevent_default::Flags;
 
 mod app;
+mod helpers;
 mod plugins;
-mod whisper;
-
-fn load_whisper_model(window: tauri::Window) -> anyhow::Result<()> {
-  let device = whisper::model_manager::load_device()?;
-
-  whisper::model_manager::load_whisper_model(window.clone(), device.clone())?;
-  whisper::model_manager::load_vad_model(window.clone(), candle_core::Device::Cpu)?;
-
-  Ok(())
-}
-
-#[tauri::command]
-async fn load_models(window: tauri::Window) -> Result<(), String> {
-  info!("Loading models...");
-
-  load_whisper_model(window).map_or_else(
-    |e| {
-      let error_message = format!("Failed to load models: {}", e);
-      info!("{}", error_message);
-      Err(error_message)
-    },
-    |_| {
-      info!("Models loaded successfully");
-      Ok(())
-    },
-  )
-}
 
 #[cfg_attr(mobile, tauri::mobile_entry_point)]
-#[allow(clippy::missing_panics_doc)]
 pub fn run() {
   let prevent_default_plugin = tauri_plugin_prevent_default::Builder::new()
     .with_flags(Flags::RELOAD)
     .build();
 
-  #[allow(clippy::missing_panics_doc)]
   tauri::Builder::default()
+    // External plugins
     .plugin(prevent_default_plugin)
     .plugin(tauri_plugin_mcp::Builder.build())
     .plugin(tauri_plugin_os::init())
     .plugin(tauri_plugin_global_shortcut::Builder::new().build())
     .plugin(tauri_plugin_window_state::Builder::default().build())
     .plugin(tauri_plugin_positioner::init())
+    // Internal plugins
     .plugin(plugins::window::init())
     .plugin(plugins::window_persistence::init())
     .plugin(plugins::window_pass_through_on_hover::init())
     .plugin(plugins::window_router_link::init())
+    .plugin(plugins::audio_transcription::init())
+    .plugin(plugins::audio_vad::init())
     .setup(|app| {
       let mut builder = WebviewWindowBuilder::new(app, "main", WebviewUrl::default())
         .title("AIRI")
@@ -152,9 +127,7 @@ pub fn run() {
       app::commands::open_settings_window,
       app::commands::open_chat_window,
       app::commands::debug_println,
-      load_models,
     ])
-    .build(tauri::generate_context!())
-    .expect("error while building tauri application")
-    .run(|_, _| {});
+    .run(tauri::generate_context!())
+    .expect("error while running tauri application");
 }

apps/stage-tamagotchi/src-tauri/src/plugins/audio_transcription/mod.rs

@@ -0,0 +1,38 @@
+use log::info;
+use tauri::{
+  Runtime,
+  plugin::{Builder as PluginBuilder, TauriPlugin},
+};
+
+use crate::{app::models::load_whisper_model, helpers::huggingface::load_device};
+
+#[tauri::command]
+pub async fn load_model_whisper<R: Runtime>(window: tauri::WebviewWindow<R>) -> Result<(), String> {
+  let device = match load_device() {
+    Ok(device) => device,
+    Err(e) => {
+      let error_message = format!("Failed to load device: {}", e);
+      info!("{}", error_message);
+      return Err(error_message);
+    },
+  };
+
+  info!("Loading models...");
+
+  // Load the traditional whisper models first
+  if let Err(e) = load_whisper_model(device.clone(), window) {
+    let error_message = format!("Failed to load whisper models: {}", e);
+    info!("{}", error_message);
+    return Err(error_message);
+  }
+
+  info!("All models loaded successfully");
+  Ok(())
+}
+
+pub fn init<R: Runtime>() -> TauriPlugin<R> {
+  PluginBuilder::new("proj-airi-tauri-plugin-audio-transcription")
+    .setup(|_, _| Ok(()))
+    .invoke_handler(tauri::generate_handler![load_model_whisper])
+    .build()
+}

apps/stage-tamagotchi/src-tauri/src/plugins/audio_vad/mod.rs

@@ -0,0 +1,39 @@
+use log::info;
+use tauri::{
+  Runtime,
+  plugin::{Builder as PluginBuilder, TauriPlugin},
+};
+
+use crate::{app::models::load_vad_model, helpers::huggingface::load_device};
+
+#[tauri::command]
+pub async fn load_model_silero_vad<R: Runtime>(
+  window: tauri::WebviewWindow<R>
+) -> Result<(), String> {
+  let device = match load_device() {
+    Ok(device) => device,
+    Err(e) => {
+      let error_message = format!("Failed to load device: {}", e);
+      info!("{}", error_message);
+      return Err(error_message);
+    },
+  };
+
+  info!("Loading models...");
+
+  if let Err(e) = load_vad_model(device.clone(), window) {
+    let error_message = format!("Failed to load VAD model: {}", e);
+    info!("{}", error_message);
+    return Err(error_message);
+  }
+
+  info!("All models loaded successfully");
+  Ok(())
+}
+
+pub fn init<R: Runtime>() -> TauriPlugin<R> {
+  PluginBuilder::new("proj-airi-tauri-plugin-audio-vad")
+    .setup(|_, _| Ok(()))
+    .invoke_handler(tauri::generate_handler![load_model_silero_vad])
+    .build()
+}

apps/stage-tamagotchi/src-tauri/src/plugins/mod.rs

@@ -1,3 +1,5 @@
+pub mod audio_transcription;
+pub mod audio_vad;
 pub mod window;
 pub mod window_pass_through_on_hover;
 pub mod window_persistence;

apps/stage-tamagotchi/src-tauri/src/whisper/mod.rs

@@ -1,4 +0,0 @@
-pub mod progress;
-pub mod whisper;
-pub mod model_manager;
-pub mod vad;

apps/stage-tamagotchi/src-tauri/src/whisper/model_manager.rs

@@ -1,43 +0,0 @@
-use log::info;
-use anyhow::Ok;
-
-use crate::whisper::whisper::{WhichWhisperModel, WhisperProcessor};
-use crate::whisper::progress::ModelLoadProgressEmitterManager;
-use crate::whisper::vad::VADProcessor;
-
-pub fn load_device() -> anyhow::Result<candle_core::Device> {
-  // Determine device to use
-  let device = if candle_core::utils::cuda_is_available() {
-    candle_core::Device::new_cuda(0)?
-  } else if candle_core::utils::metal_is_available() {
-    candle_core::Device::new_metal(0)?
-  } else {
-    candle_core::Device::Cpu
-  };
-
-  info!("Using device: {device:?}");
-  Ok(device)
-}
-
-
-pub fn load_whisper_model(window: tauri::Window, device: candle_core::Device) -> anyhow::Result<()> {
-  let progress_manager = ModelLoadProgressEmitterManager::new(window);
-
-  let whisper_model = WhichWhisperModel::Tiny;
-
-  info!("Loading whisper model: {:?}", whisper_model);
-
-  let _ = WhisperProcessor::new(whisper_model, device.clone(), progress_manager)?;
-  Ok(())
-}
-
-pub fn load_vad_model(window: tauri::Window, device: candle_core::Device) -> anyhow::Result<()> {
-  let progress_manager = ModelLoadProgressEmitterManager::new(window);
-
-  let whisper_model = WhichWhisperModel::Tiny;
-
-  info!("Loading VAD model: {:?}", whisper_model);
-
-  let _ = VADProcessor::new(device.clone(), 0.3, progress_manager)?;
-  Ok(())
-}

apps/stage-tamagotchi/src-tauri/src/whisper/progress.rs

@@ -1,100 +0,0 @@
-use log::error;
-use tauri::Emitter;
-
-#[derive(Clone)]
-pub struct ModelLoadProgressEmitterManager {
-  window: tauri::Window,
-}
-
-impl ModelLoadProgressEmitterManager {
-  pub fn new(window: tauri::Window) -> Self {
-    Self { window }
-  }
-
-  pub fn new_for(
-    self,
-    filename: &str,
-  ) -> ModelLoadProgressEmitter {
-    ModelLoadProgressEmitter::new(self.window, filename)
-  }
-}
-
-pub struct ModelLoadProgressEmitter {
-  filename:   String,
-  size:       usize,
-  total_size: usize,
-  progress:   f32,
-  window:     tauri::Window,
-}
-
-impl ModelLoadProgressEmitter {
-  fn new(
-    window: tauri::Window,
-    filename: &str,
-  ) -> Self {
-    Self {
-      filename: filename.to_string(),
-      size: 0,
-      total_size: 0,
-      progress: 0.0,
-      window,
-    }
-  }
-}
-
-impl hf_hub::api::Progress for ModelLoadProgressEmitter {
-  fn init(
-    &mut self,
-    size: usize,
-    _: &str,
-  ) {
-    self.total_size = size;
-    self.progress = 0.0;
-    self
-      .window
-      .emit(
-        "tauri-app:model-load-progress",
-        (self.filename.clone(), self.progress),
-      )
-      .map_err(|err| {
-        error!("Failed to emit model-load-progress: {:?}", err);
-      })
-      .unwrap();
-  }
-
-  fn update(
-    &mut self,
-    size: usize,
-  ) {
-    self.size += size;
-    self.progress = if self.total_size > 0 {
-      (self.size as f32 / self.total_size as f32 * 100.0).min(100.0)
-    } else {
-      100.0
-    };
-    self
-      .window
-      .emit(
-        "tauri-app:model-load-progress",
-        (self.filename.clone(), self.progress),
-      )
-      .map_err(|err| {
-        error!("Failed to emit model-load-progress: {:?}", err);
-      })
-      .unwrap();
-  }
-
-  fn finish(&mut self) {
-    self.progress = 100.0;
-    self
-      .window
-      .emit(
-        "tauri-app:model-load-progress",
-        (self.filename.clone(), self.progress),
-      )
-      .map_err(|err| {
-        error!("Failed to emit model-load-progress: {:?}", err);
-      })
-      .unwrap();
-  }
-}

apps/stage-tamagotchi/src/composables/tauri.ts

@@ -141,13 +141,16 @@ export function useTauriEvent<ES = Events>() {
 }
 
 export interface InvokeMethods {
-  // Model related
-  'load_models': { args: undefined, options: undefined, returns: void }
-
   // app windows
   'open_settings_window': { args: undefined, options: undefined, returns: void }
   'open_chat_window': { args: undefined, options: undefined, returns: void }
 
+  // Plugin - Audio Transcription
+  'plugin:proj-airi-tauri-plugin-audio-transcription|load_model_whisper': { args: undefined, options: undefined, returns: void }
+
+  // Plugin - Audio VAD
+  'plugin:proj-airi-tauri-plugin-audio-vad|load_model_silero_vad': { args: undefined, options: undefined, returns: void }
+
   // Plugin - Window Pass through on hover
   'plugin:proj-airi-tauri-plugin-window-pass-through-on-hover|start_monitor': { args: undefined, options: undefined, returns: void }
   'plugin:proj-airi-tauri-plugin-window-pass-through-on-hover|stop_monitor': { args: undefined, options: undefined, returns: void }

apps/stage-tamagotchi/src/pages/index.vue

@@ -83,7 +83,8 @@ onMounted(async () => {
   }))
 
   // Load models
-  invoke('load_models')
+  invoke('plugin:proj-airi-tauri-plugin-audio-transcription|load_model_whisper')
+  invoke('plugin:proj-airi-tauri-plugin-audio-vad|load_model_silero_vad')
 
   if (connected.value)
     return