fix(stage-tamagotchi): resource island wasn't working (#347)

apps/stage-tamagotchi/src/components/Widgets/ResourceStatusIsland/index.vue

@@ -7,7 +7,9 @@ import { ref, watch } from 'vue'
 import LoadingModules from './LoadingModules.vue'
 
 import { useResourcesStore } from '../../../stores/resources'
+import { useWindowStore } from '../../../stores/window'
 
+const { shouldHideView } = storeToRefs(useWindowStore())
 const { atLeastOneLoading, atLeastOneLoadingDelay5s, atLeastOneLoadingDelay10s } = storeToRefs(useResourcesStore())
 
 const loadingProgressOpen = ref(false)
@@ -24,7 +26,7 @@ function handleClick() {
 <template>
   <div fixed left-0 top-3 w-full flex flex-col items-center>
     <TooltipProvider v-if="atLeastOneLoadingDelay10s" :delay-duration="150">
-      <TooltipRoot :open="loadingProgressOpen" disable-closing-trigger @update:open="(state) => loadingProgressOpen = state">
+      <TooltipRoot :open="loadingProgressOpen && !shouldHideView" disable-closing-trigger @update:open="(state) => loadingProgressOpen = state">
         <TooltipTrigger>
           <Transition name="fade">
             <div

apps/stage-tamagotchi/src/composables/tauri-window-pass-through-on-hover.ts

@@ -2,8 +2,9 @@ import type { MaybeRefOrGetter } from 'vue'
 
 import type { AiriTamagotchiEvents, Point, WindowFrame } from './tauri'
 
-import { onMounted, ref, toValue, watch } from 'vue'
+import { onMounted, onUnmounted, ref, toValue, watch } from 'vue'
 
+import { startClickThrough, stopClickThrough } from '../utils/windows'
 import { useAppRuntime } from './runtime'
 import { useTauriCore, useTauriEvent } from './tauri'
 
@@ -41,7 +42,17 @@ export function useTauriPointAndWindowFrame() {
     })
   }
 
-  onMounted(() => addListeners())
+  onMounted(() => {
+    addListeners()
+    invoke('plugin:window-pass-through-on-hover|start_tracing_cursor')
+  })
+
+  onUnmounted(() => {
+    unListenFuncs.value.forEach(fn => fn?.())
+    unListenFuncs.value.length = 0
+
+    invoke('plugin:window-pass-through-on-hover|stop_tracing_cursor')
+  })
 
   if (import.meta.hot) { // For better DX
     import.meta.hot.on('vite:beforeUpdate', () => {
@@ -52,8 +63,7 @@ export function useTauriPointAndWindowFrame() {
     })
 
     import.meta.hot.on('vite:afterUpdate', () => {
-      unListenFuncs.value.length === 0 && addListeners()
-
+      addListeners()
       invoke('plugin:window-pass-through-on-hover|start_tracing_cursor')
     })
   }
@@ -108,6 +118,14 @@ export function useTauriWindowClickThrough(live2DLookAtDefault: MaybeRefOrGetter
     }
   })
 
+  onMounted(async () => {
+    await startClickThrough()
+  })
+
+  onUnmounted(async () => {
+    await stopClickThrough()
+  })
+
   return {
     isCursorInside,
     live2dLookAtX,

apps/stage-tamagotchi/src/composables/tauri.ts

@@ -58,9 +58,9 @@ export interface AiriTamagotchiEvents extends Events {
   'tauri-plugins:tauri-plugin-window-pass-through-on-hover:pass-through-enabled': boolean
 
   // from tauri-plugin-ipc-audio-transcription-ort
-  'tauri-plugins:tauri-plugin-ipc-audio-transcription-ort:load-model-silero-vad-progress': [boolean, string, number, number, number]
+  'tauri-plugins:tauri-plugin-ipc-audio-vad-ort:load-model-silero-vad-progress': [boolean, string, number, number, number]
   // from tauri-plugin-ipc-audio-vad-ort
-  'tauri-plugins:tauri-plugin-ipc-audio-vad-ort:load-model-whisper-progress': [boolean, string, number, number, number]
+  'tauri-plugins:tauri-plugin-ipc-audio-transcription-ort:load-model-whisper-progress': [boolean, string, number, number, number]
 
   // from tauri-plugin-rdev
   'tauri-plugins:tauri-plugin-rdev:keydown': { time: { secs_since_epoch: number, nanos_since_epoch: number }, name: string, event_type: { KeyPress: KeyCode | { Unknown: number } } } // similar to 'keydown' events from DOM elements

apps/stage-tamagotchi/src/pages/index.vue

@@ -4,7 +4,6 @@ import type { AiriTamagotchiEvents, Point } from '../composables/tauri'
 import { WidgetStage } from '@proj-airi/stage-ui/components/scenes'
 import { useLive2d, useMcpStore } from '@proj-airi/stage-ui/stores'
 import { connectServer } from '@proj-airi/tauri-plugin-mcp'
-import { useWindowSize } from '@vueuse/core'
 import { storeToRefs } from 'pinia'
 import { computed, onMounted, onUnmounted, ref, watch } from 'vue'
 
@@ -12,32 +11,28 @@ import ResourceStatusIsland from '../components/Widgets/ResourceStatusIsland/ind
 
 import { useTauriCore, useTauriEvent } from '../composables/tauri'
 import { useTauriGlobalShortcuts } from '../composables/tauri-global-shortcuts'
-import { useTauriWindowClickThrough } from '../composables/tauri-window-pass-through-on-hover'
 import { useResourcesStore } from '../stores/resources'
+import { useWindowStore } from '../stores/window'
 import { useWindowControlStore } from '../stores/window-controls'
 import { WindowControlMode } from '../types/window-controls'
-import { startClickThrough, stopClickThrough } from '../utils/windows'
 
 useTauriGlobalShortcuts()
-const windowStore = useWindowControlStore()
+const windowControlStore = useWindowControlStore()
 const resourcesStore = useResourcesStore()
 const mcpStore = useMcpStore()
 
 const { listen } = useTauriEvent<AiriTamagotchiEvents>()
 const { invoke } = useTauriCore()
-const { width, height } = useWindowSize()
 const { connected, serverCmd, serverArgs } = storeToRefs(mcpStore)
 const { scale, positionInPercentageString } = storeToRefs(useLive2d())
 
-const centerPos = computed(() => ({ x: width.value / 2, y: height.value / 2 }))
-const { live2dLookAtX, live2dLookAtY, isCursorInside } = useTauriWindowClickThrough(centerPos)
-const shouldHideView = computed(() => isCursorInside.value && !windowStore.isControlActive && windowStore.isIgnoringMouseEvent)
+const { centerPos, live2dLookAtX, live2dLookAtY, shouldHideView } = storeToRefs(useWindowStore())
 const live2dFocusAt = ref<Point>(centerPos.value)
 
 watch([live2dLookAtX, live2dLookAtY], ([x, y]) => live2dFocusAt.value = { x, y }, { immediate: true })
 
 const modeIndicatorClass = computed(() => {
-  switch (windowStore.controlMode) {
+  switch (windowControlStore.controlMode) {
     case WindowControlMode.MOVE:
       return 'cursor-move'
     case WindowControlMode.RESIZE:
@@ -49,16 +44,6 @@ const modeIndicatorClass = computed(() => {
   }
 })
 
-onMounted(async () => {
-  await invoke('plugin:window-pass-through-on-hover|start_tracing_cursor')
-  await startClickThrough()
-})
-
-onUnmounted(async () => {
-  await stopClickThrough()
-  await invoke('plugin:window-pass-through-on-hover|stop_tracing_cursor')
-})
-
 const unListenFuncs: (() => void)[] = []
 
 function openSettings() {
@@ -69,20 +54,35 @@ function openChat() {
   invoke('open_chat_window')
 }
 
-onMounted(async () => {
+async function setupVADModelLoadingProgressListener() {
   // VAD
-  unListenFuncs.push(await listen('tauri-plugins:tauri-plugin-ipc-audio-transcription-ort:load-model-silero-vad-progress', (event) => {
+  unListenFuncs.push(await listen('tauri-plugins:tauri-plugin-ipc-audio-vad-ort:load-model-silero-vad-progress', (event) => {
     const [_, filename, progress, totalSize, currentSize] = event.payload
     resourcesStore.updateResourceProgress('hearing', 'vad', { filename, progress, totalSize, currentSize })
   }))
-  invoke('plugin:ipc-audio-vad-ort|load_ort_model_silero_vad')
+}
 
+async function setupVADModel() {
+  await setupVADModelLoadingProgressListener()
+  invoke('plugin:ipc-audio-vad-ort|load_ort_model_silero_vad')
+}
+
+async function setupWhisperModelLoadingProgressListener() {
   // Whisper
-  unListenFuncs.push(await listen('tauri-plugins:tauri-plugin-ipc-audio-vad-ort:load-model-whisper-progress', (event) => {
+  unListenFuncs.push(await listen('tauri-plugins:tauri-plugin-ipc-audio-transcription-ort:load-model-whisper-progress', (event) => {
     const [_, filename, progress, totalSize, currentSize] = event.payload
     resourcesStore.updateResourceProgress('hearing', 'whisper', { filename, progress, totalSize, currentSize })
   }))
+}
+
+async function setupWhisperModel() {
+  await setupWhisperModelLoadingProgressListener()
   invoke('plugin:ipc-audio-transcription-ort|load_ort_model_whisper', { modelType: 'medium' })
+}
+
+onMounted(async () => {
+  await setupVADModel()
+  await setupWhisperModel()
 
   if (connected.value)
     return
@@ -106,11 +106,10 @@ if (import.meta.hot) { // For better DX
   import.meta.hot.on('vite:beforeUpdate', () => {
     unListenFuncs.forEach(fn => fn?.())
     unListenFuncs.length = 0
-
-    invoke('plugin:window-pass-through-on-hover|stop_tracing_cursor')
   })
   import.meta.hot.on('vite:afterUpdate', async () => {
-    invoke('plugin:window-pass-through-on-hover|start_tracing_cursor')
+    await setupVADModelLoadingProgressListener()
+    await setupWhisperModelLoadingProgressListener()
   })
 }
 </script>
@@ -137,8 +136,8 @@ if (import.meta.hot) { // For better DX
       <div
         absolute bottom-4 left-4 flex gap-1 op-0 transition="opacity duration-500"
         :class="{
-          'pointer-events-none': windowStore.isControlActive,
-          'show-on-hover': !windowStore.isIgnoringMouseEvent,
+          'pointer-events-none': windowControlStore.isControlActive,
+          'show-on-hover': !windowControlStore.isIgnoringMouseEvent,
         }"
       >
         <div
@@ -162,7 +161,7 @@ if (import.meta.hot) { // For better DX
       </div>
     </div>
     <!-- Debug Mode UI -->
-    <div v-if="windowStore.controlMode === WindowControlMode.DEBUG" class="debug-controls">
+    <div v-if="windowControlStore.controlMode === WindowControlMode.DEBUG" class="debug-controls">
       <!-- Add debug controls here -->
     </div>
   </div>
@@ -175,7 +174,7 @@ if (import.meta.hot) { // For better DX
     leave-to-class="opacity-0"
   >
     <div
-      v-if="windowStore.controlMode === WindowControlMode.MOVE"
+      v-if="windowControlStore.controlMode === WindowControlMode.MOVE"
       data-tauri-drag-region
       class="absolute left-0 top-0 z-999 h-full w-full flex cursor-grab items-center justify-center overflow-hidden"
     >
@@ -200,7 +199,7 @@ if (import.meta.hot) { // For better DX
     leave-to-class="opacity-50"
   >
     <div
-      v-if="windowStore.controlMode === WindowControlMode.RESIZE"
+      v-if="windowControlStore.controlMode === WindowControlMode.RESIZE"
       class="absolute left-0 top-0 z-999 h-full w-full"
     >
       <div h-full w-full animate-flash animate-duration-2.5s animate-count-infinite b-4 b-primary rounded-2xl />

apps/stage-tamagotchi/src/stores/window.ts

@@ -0,0 +1,25 @@
+import { useWindowSize } from '@vueuse/core'
+import { defineStore } from 'pinia'
+import { computed } from 'vue'
+
+import { useTauriWindowClickThrough } from '../composables/tauri-window-pass-through-on-hover'
+import { useWindowControlStore } from './window-controls'
+
+export const useWindowStore = defineStore('tamagotchi-window', () => {
+  const windowControlStore = useWindowControlStore()
+
+  const { width, height } = useWindowSize()
+  const centerPos = computed(() => ({ x: width.value / 2, y: height.value / 2 }))
+  const { live2dLookAtX, live2dLookAtY, isCursorInside } = useTauriWindowClickThrough(centerPos)
+  const shouldHideView = computed(() => isCursorInside.value && !windowControlStore.isControlActive && windowControlStore.isIgnoringMouseEvent)
+
+  return {
+    width,
+    height,
+    centerPos,
+    live2dLookAtX,
+    live2dLookAtY,
+    isCursorInside,
+    shouldHideView,
+  }
+})

crates/tauri-plugin-ipc-audio-transcription-ort/src/models/whisper/whisper.rs

@@ -199,6 +199,22 @@ pub fn whisper_language_to_code(language: &str) -> Result<String> {
   Err(anyhow!("Language '{}' is not supported.", language))
 }
 
+fn get_or_download_file<R: Runtime>(
+  cache_repo: &hf_hub::CacheRepo,
+  repo: &hf_hub::api::sync::ApiRepo,
+  window: tauri::WebviewWindow<R>,
+  file_sub_path: &str,
+  event_name: &str,
+) -> Result<PathBuf, hf_hub::api::sync::ApiError> {
+  match cache_repo.get(file_sub_path) {
+    Some(p) => Ok(p),
+    None => repo.download_with_progress(
+      file_sub_path,
+      create_progress_emitter(window.clone(), event_name, file_sub_path.to_string()),
+    ),
+  }
+}
+
 pub struct Whisper {
   encoder_session: Session,
   decoder_session: Session,
@@ -225,35 +241,44 @@ impl Whisper {
       revision.to_string(),
     ));
 
-    let encoder_model_path_sub_name = "onnx/encoder_model.onnx";
-    let encoder_model_path = match cache_repo.get(encoder_model_path_sub_name) {
+    let encoder_model_path = get_or_download_file(
+      &cache_repo,
+      &repo,
+      window.clone(),
+      "onnx/encoder_model.onnx",
+      "tauri-plugins:tauri-plugin-ipc-audio-transcription-ort:load-model-whisper-progress",
+    )?;
+
+    let decoder_model_path = get_or_download_file(
+      &cache_repo,
+      &repo,
+      window.clone(),
+      "onnx/decoder_model.onnx",
+      "tauri-plugins:tauri-plugin-ipc-audio-transcription-ort:load-model-whisper-progress",
+    )?;
+
+    let config_path = match cache_repo.get("config.json") {
+      Some(path) => path,
       None => repo.download_with_progress(
-        encoder_model_path_sub_name,
+        "config.json",
         create_progress_emitter(
           window.clone(),
           "tauri-plugins:tauri-plugin-ipc-audio-transcription-ort:load-model-whisper-progress",
-          encoder_model_path_sub_name.to_string(),
+          "config.json".to_string(),
         ),
       )?,
-      Some(p) => p,
     };
 
-    let decoder_model_path_sub_name = "onnx/decoder_model.onnx";
-    let decoder_model_path = match cache_repo.get(decoder_model_path_sub_name) {
+    let tokenizer_config_path = match cache_repo.get("tokenizer_config.json") {
       Some(path) => path,
-      None => repo.download(decoder_model_path_sub_name)?,
-    };
-
-    let config_path_sub_name = "config.json";
-    let config_path = match cache_repo.get(config_path_sub_name) {
-      Some(path) => path,
-      None => repo.download(config_path_sub_name)?,
-    };
-
-    let tokenizer_config_path_sub_name = "tokenizer.json";
-    let tokenizer_config_path = match cache_repo.get(tokenizer_config_path_sub_name) {
-      Some(path) => path,
-      None => repo.download(tokenizer_config_path_sub_name)?,
+      None => repo.download_with_progress(
+        "tokenizer_config.json",
+        create_progress_emitter(
+          window.clone(),
+          "tauri-plugins:tauri-plugin-ipc-audio-transcription-ort:load-model-whisper-progress",
+          "tokenizer_config.json".to_string(),
+        ),
+      )?,
     };
 
     let encoder_session = Self::create_optimized_session(encoder_model_path)?;
@@ -453,18 +478,33 @@ impl WhisperPipeline {
     revision: &str,
     window: tauri::WebviewWindow<R>,
   ) -> Result<Self> {
-    let model = Whisper::new(model_id, revision, window)?;
+    let model = Whisper::new(model_id, revision, window.clone())?;
 
     // Initialize our new processor
     let processor = WhisperProcessor::new(which_model)?;
 
+    let cache_api = hf_hub::Cache::from_env();
+    let cache_repo = cache_api.repo(Repo::with_revision(
+      model_id.to_string(),
+      RepoType::Model,
+      revision.to_string(),
+    ));
+
     let api = Api::new()?;
     let repo = api.repo(hf_hub::Repo::with_revision(
       model_id.to_string(),
       hf_hub::RepoType::Model,
       revision.to_string(),
     ));
-    let tokenizer_path = repo.get("tokenizer.json")?;
+
+    let tokenizer_path = get_or_download_file(
+      &cache_repo,
+      &repo,
+      window.clone(),
+      "tokenizer.json",
+      "tauri-plugins:tauri-plugin-ipc-audio-transcription-ort:load-model-whisper-progress",
+    )?;
+
     let tokenizer = Tokenizer::from_file(tokenizer_path)
       .map_err(|e| anyhow!("Failed to load tokenizer: {}", e))?;
 

cspell.config.yaml

@@ -18,6 +18,7 @@ words:
   - Attributify
   - attw
   - audioworklet
+  - autoplay
   - awilix
   - Ayaka
   - baichuan
@@ -26,6 +27,7 @@ words:
   - bigserial
   - Bitstream
   - browserbasehq
+  - buildless
   - bumpp
   - byteorder
   - catppuccin