From 6071331033baac2224c731956a7106f311f45d09 Mon Sep 17 00:00:00 2001
From: Neko Ayaka <neko@ayaka.moe>
Date: Thu, 30 Oct 2025 16:58:57 +0800
Subject: [PATCH] feat(stage-tamagotchi): speech recovered

---
 .../components/HearingPermissionStatus.vue    |  78 +++++++++++++
 .../Widgets/ControlsIsland/index.vue          |  11 +-
 .../src/renderer/pages/index.vue              | 110 +++++++++++++++++-
 .../src/shared/electron/system-preferences.ts |   4 +-
 .../audio-input/hearing-config-dialog.vue     |   2 +
 .../dialogs/audio-input/hearing-config.vue    |  30 ++++-
 6 files changed, 227 insertions(+), 8 deletions(-)
 create mode 100644 apps/stage-tamagotchi/src/renderer/components/HearingPermissionStatus.vue

diff --git a/apps/stage-tamagotchi/src/renderer/components/HearingPermissionStatus.vue b/apps/stage-tamagotchi/src/renderer/components/HearingPermissionStatus.vue
new file mode 100644
index 000000000..3fe288635
--- /dev/null
+++ b/apps/stage-tamagotchi/src/renderer/components/HearingPermissionStatus.vue
@@ -0,0 +1,78 @@
+<script setup lang="ts">
+import { onMounted, ref } from 'vue'
+
+import { electron } from '../../shared/electron'
+import { useElectronEventaInvoke } from '../composables/electron-vueuse'
+
+const status = ref<string>('unknown')
+const loading = ref(false)
+const error = ref<string>('')
+
+const getMediaAccessStatus = useElectronEventaInvoke(electron.systemPreferences.getMediaAccessStatus)
+const askForMediaAccess = useElectronEventaInvoke(electron.systemPreferences.askForMediaAccess)
+
+async function refreshStatus() {
+  try {
+    loading.value = true
+    error.value = ''
+    const s = await getMediaAccessStatus(['microphone'])
+    status.value = String(s)
+  }
+  catch (e) {
+    error.value = e instanceof Error ? e.message : String(e)
+  }
+  finally {
+    loading.value = false
+  }
+}
+
+async function requestAccess() {
+  try {
+    loading.value = true
+    error.value = ''
+    await askForMediaAccess(['microphone'])
+    await refreshStatus()
+  }
+  catch (e) {
+    error.value = e instanceof Error ? e.message : String(e)
+  }
+  finally {
+    loading.value = false
+  }
+}
+
+onMounted(() => {
+  if (window?.electron)
+    refreshStatus()
+})
+</script>
+
+<template>
+  <div class="mt-4 border border-neutral-200 rounded-lg p-3 dark:border-neutral-700">
+    <div class="mb-2 text-sm text-neutral-600 font-medium dark:text-neutral-300">
+      Microphone Permission
+    </div>
+    <div class="flex items-center gap-3">
+      <div v-if="loading" class="i-solar:spinner-line-duotone animate-spin text-neutral-500" />
+      <div
+        v-else :class="{
+          'text-green-600 dark:text-green-400': status === 'granted',
+          'text-red-600 dark:text-red-400': status === 'denied' || status === 'restricted',
+          'text-amber-600 dark:text-amber-400': status === 'not-determined' || status === 'unknown',
+        }"
+      >
+        Status: {{ status }}
+      </div>
+      <button
+        v-if="status !== 'granted'"
+        class="ml-auto rounded-md bg-neutral-200 px-3 py-1 text-sm text-neutral-800 dark:bg-neutral-800 hover:bg-neutral-300 dark:text-neutral-200 dark:hover:bg-neutral-700"
+        @click="requestAccess"
+      >
+        Request Access
+      </button>
+    </div>
+    <div v-if="error" class="mt-2 text-xs text-red-500">
+      {{ error }}
+    </div>
+  </div>
+</template>
diff --git a/apps/stage-tamagotchi/src/renderer/components/Widgets/ControlsIsland/index.vue b/apps/stage-tamagotchi/src/renderer/components/Widgets/ControlsIsland/index.vue
index c80e0de97..6836791f9 100644
--- a/apps/stage-tamagotchi/src/renderer/components/Widgets/ControlsIsland/index.vue
+++ b/apps/stage-tamagotchi/src/renderer/components/Widgets/ControlsIsland/index.vue
@@ -5,7 +5,9 @@ import { defineInvoke } from '@unbird/eventa'
 import { createContext } from '@unbird/eventa/adapters/electron/renderer'
 import { useDark, useToggle } from '@vueuse/core'
 import { storeToRefs } from 'pinia'
+import { ref } from 'vue'
 
+import HearingPermissionStatus from '../../../components/HearingPermissionStatus.vue'
 import ControlButton from './ControlButton.vue'
 import ControlButtonTooltip from './ControlButtonTooltip.vue'
 
@@ -28,6 +30,10 @@ const openSettings = defineInvoke(context, electronOpenSettings)
  * See `apps/stage-tamagotchi/src/main/windows/main/index.ts` for handler definition
  */
 const startDraggingWindow = !isLinux ? defineInvoke(context, electronStartDraggingWindow) : undefined
+
+// Expose whether hearing dialog is open so parent can disable click-through
+const hearingDialogOpen = ref(false)
+defineExpose({ hearingDialogOpen })
 </script>
 
 <template>
@@ -44,13 +50,16 @@ const startDraggingWindow = !isLinux ? defineInvoke(context, electronStartDraggi
       </ControlButtonTooltip>
 
       <ControlButtonTooltip>
-        <HearingConfigDialog>
+        <HearingConfigDialog v-model:show="hearingDialogOpen">
           <ControlButton>
             <Transition name="fade" mode="out-in">
               <div v-if="isAudioEnabled" i-ph:microphone size-5 text="neutral-800 dark:neutral-300" />
               <div v-else i-ph:microphone-slash size-5 text="neutral-800 dark:neutral-300" />
             </Transition>
           </ControlButton>
+          <template #extra>
+            <HearingPermissionStatus />
+          </template>
         </HearingConfigDialog>
 
         <template #tooltip>
diff --git a/apps/stage-tamagotchi/src/renderer/pages/index.vue b/apps/stage-tamagotchi/src/renderer/pages/index.vue
index 1d738584d..335abbd35 100644
--- a/apps/stage-tamagotchi/src/renderer/pages/index.vue
+++ b/apps/stage-tamagotchi/src/renderer/pages/index.vue
@@ -1,10 +1,21 @@
 <script setup lang="ts">
+import type { ChatProvider } from '@xsai-ext/shared-providers'
+
+import workletUrl from '@proj-airi/stage-ui/workers/vad/process.worklet?worker&url'
+
 import { WidgetStage } from '@proj-airi/stage-ui/components/scenes'
+import { useAudioRecorder } from '@proj-airi/stage-ui/composables/audio/audio-recorder'
 import { useCanvasPixelIsTransparentAtPoint } from '@proj-airi/stage-ui/composables/canvas-alpha'
+import { useVAD } from '@proj-airi/stage-ui/stores/ai/models/vad'
+import { useChatStore } from '@proj-airi/stage-ui/stores/chat'
 import { useLive2d } from '@proj-airi/stage-ui/stores/live2d'
+import { useConsciousnessStore } from '@proj-airi/stage-ui/stores/modules/consciousness'
+import { useHearingSpeechInputPipeline } from '@proj-airi/stage-ui/stores/modules/hearing'
+import { useProvidersStore } from '@proj-airi/stage-ui/stores/providers'
+import { useSettingsAudioDevice } from '@proj-airi/stage-ui/stores/settings'
 import { debouncedRef, watchPausable } from '@vueuse/core'
 import { storeToRefs } from 'pinia'
-import { ref, toRef, watch } from 'vue'
+import { computed, onUnmounted, ref, toRef, watch } from 'vue'
 
 import ControlsIsland from '../components/Widgets/ControlsIsland/index.vue'
 import ResourceStatusIsland from '../components/Widgets/ResourceStatusIsland/index.vue'
@@ -49,7 +60,18 @@ const { pause, resume } = watchPausable(isTransparent, (transparent) => {
   shouldFadeOnCursorWithin.value = !transparent
 }, { immediate: true })
 
-watch([isOutsideFor250Ms, isAroundWindowBorderFor250Ms, isOutsideWindow, isTransparent], () => {
+const hearingDialogOpen = computed(() => controlsIslandRef.value?.hearingDialogOpen ?? false)
+
+watch([isOutsideFor250Ms, isAroundWindowBorderFor250Ms, isOutsideWindow, isTransparent, hearingDialogOpen], () => {
+  if (hearingDialogOpen.value) {
+    // Hearing dialog/drawer is open; keep window interactive
+    isIgnoringMouseEvents.value = false
+    shouldFadeOnCursorWithin.value = false
+    setIgnoreMouseEvents([false, { forward: true }])
+    pause()
+    return
+  }
+
   const insideControls = !isOutsideFor250Ms.value
   const nearBorder = isAroundWindowBorderFor250Ms.value
 
@@ -70,6 +92,90 @@ watch([isOutsideFor250Ms, isAroundWindowBorderFor250Ms, isOutsideWindow, isTrans
     resume()
   }
 })
+
+const settingsAudioDeviceStore = useSettingsAudioDevice()
+const { stream, enabled } = storeToRefs(settingsAudioDeviceStore)
+const { startRecord, stopRecord, onStopRecord } = useAudioRecorder(stream)
+const { transcribeForRecording } = useHearingSpeechInputPipeline()
+const providersStore = useProvidersStore()
+const consciousnessStore = useConsciousnessStore()
+const { activeProvider: activeChatProvider, activeModel: activeChatModel } = storeToRefs(consciousnessStore)
+const chatStore = useChatStore()
+
+const {
+  init: initVAD,
+  dispose: disposeVAD,
+  start: startVAD,
+  loaded: vadLoaded,
+} = useVAD(workletUrl, {
+  threshold: ref(0.6),
+  onSpeechStart: () => startRecord(),
+  onSpeechEnd: () => stopRecord(),
+})
+
+let stopOnStopRecord: (() => void) | undefined
+
+async function startAudioInteraction() {
+  try {
+    await initVAD()
+    if (stream.value)
+      await startVAD(stream.value)
+
+    // Hook once
+    stopOnStopRecord = onStopRecord(async (recording) => {
+      const text = await transcribeForRecording(recording)
+      if (!text || !text.trim())
+        return
+
+      try {
+        const provider = await providersStore.getProviderInstance(activeChatProvider.value)
+        if (!provider || !activeChatModel.value)
+          return
+
+        await chatStore.send(text, { model: activeChatModel.value, chatProvider: provider as ChatProvider })
+      }
+      catch (err) {
+        console.error('Failed to send chat from voice:', err)
+      }
+    })
+  }
+  catch (e) {
+    console.error('Audio interaction init failed:', e)
+  }
+}
+
+function stopAudioInteraction() {
+  try {
+    stopOnStopRecord?.()
+    stopOnStopRecord = undefined
+    disposeVAD()
+  }
+  catch {}
+}
+
+watch(enabled, async (val) => {
+  if (val) {
+    await startAudioInteraction()
+  }
+  else {
+    stopAudioInteraction()
+  }
+}, { immediate: true })
+
+onUnmounted(() => {
+  stopAudioInteraction()
+})
+
+watch([stream, () => vadLoaded.value], async ([s, loaded]) => {
+  if (enabled.value && loaded && s) {
+    try {
+      await startVAD(s)
+    }
+    catch (e) {
+      console.error('Failed to start VAD with stream:', e)
+    }
+  }
+})
 </script>
 
 <template>
diff --git a/apps/stage-tamagotchi/src/shared/electron/system-preferences.ts b/apps/stage-tamagotchi/src/shared/electron/system-preferences.ts
index c65b2bb0e..3631104a0 100644
--- a/apps/stage-tamagotchi/src/shared/electron/system-preferences.ts
+++ b/apps/stage-tamagotchi/src/shared/electron/system-preferences.ts
@@ -2,8 +2,8 @@ import type { systemPreferences as electronSystemPreferences } from 'electron'
 
 import { defineInvokeEventa } from '@unbird/eventa'
 
-const getMediaAccessStatus = defineInvokeEventa<ReturnType<typeof electronSystemPreferences.getMediaAccessStatus>, Parameters<typeof electronSystemPreferences.getMediaAccessStatus>[0]>('eventa:invoke:electron:system-preferences:get-media-access-status')
-const askForMediaAccess = defineInvokeEventa<ReturnType<typeof electronSystemPreferences.askForMediaAccess>, Parameters<typeof electronSystemPreferences.askForMediaAccess>[0]>('eventa:invoke:electron:system-preferences:ask-for-media-access')
+const getMediaAccessStatus = defineInvokeEventa<ReturnType<typeof electronSystemPreferences.getMediaAccessStatus>, [Parameters<typeof electronSystemPreferences.getMediaAccessStatus>[0]]>('eventa:invoke:electron:system-preferences:get-media-access-status')
+const askForMediaAccess = defineInvokeEventa<ReturnType<typeof electronSystemPreferences.askForMediaAccess>, [Parameters<typeof electronSystemPreferences.askForMediaAccess>[0]]>('eventa:invoke:electron:system-preferences:ask-for-media-access')
 
 export const systemPreferences = {
   getMediaAccessStatus,
diff --git a/packages/stage-ui/src/components/scenarios/dialogs/audio-input/hearing-config-dialog.vue b/packages/stage-ui/src/components/scenarios/dialogs/audio-input/hearing-config-dialog.vue
index f46ead40f..f7c11c9b6 100644
--- a/packages/stage-ui/src/components/scenarios/dialogs/audio-input/hearing-config-dialog.vue
+++ b/packages/stage-ui/src/components/scenarios/dialogs/audio-input/hearing-config-dialog.vue
@@ -37,6 +37,7 @@ onMounted(() => screenSafeArea.update())
           <DialogTitle>Hearing Input</DialogTitle>
         </VisuallyHidden>
         <HearingConfig @close="showDialog = false" />
+        <slot name="extra" />
       </DialogContent>
     </DialogPortal>
   </DialogRoot>
@@ -49,6 +50,7 @@ onMounted(() => screenSafeArea.update())
       <DrawerContent class="fixed bottom-0 left-0 right-0 z-1000 mt-20 h-full max-h-[50%] flex flex-col rounded-t-2xl bg-neutral-50 px-4 pt-4 outline-none backdrop-blur-md dark:bg-neutral-900/95" :style="{ paddingBottom: `${Math.max(Number.parseFloat(screenSafeArea.bottom.value.replace('px', '')), 24)}px` }">
         <DrawerHandle />
         <HearingConfig @close="showDialog = false" />
+        <slot name="extra" />
       </DrawerContent>
     </DrawerPortal>
   </DrawerRoot>
diff --git a/packages/stage-ui/src/components/scenarios/dialogs/audio-input/hearing-config.vue b/packages/stage-ui/src/components/scenarios/dialogs/audio-input/hearing-config.vue
index 9039119f2..afda3e7e0 100644
--- a/packages/stage-ui/src/components/scenarios/dialogs/audio-input/hearing-config.vue
+++ b/packages/stage-ui/src/components/scenarios/dialogs/audio-input/hearing-config.vue
@@ -1,22 +1,46 @@
 <script setup lang="ts">
+import { FieldSelect } from '@proj-airi/ui'
 import { useToggle } from '@vueuse/core'
 import { storeToRefs } from 'pinia'
+import { computed } from 'vue'
 
 import Button from '../../../misc/Button.vue'
 
 import { useSettingsAudioDevice } from '../../../../stores/settings'
 
 const settingsAudioDeviceStore = useSettingsAudioDevice()
-const { enabled } = storeToRefs(settingsAudioDeviceStore)
+const { enabled, audioInputs, selectedAudioInput } = storeToRefs(settingsAudioDeviceStore)
 const toggleAudioDevice = useToggle(enabled)
+
+function requestPermission() {
+  // Generic permission request via mediaDevices; in Electron, renderer wrappers may also call OS APIs.
+  settingsAudioDeviceStore.askPermission()
+}
+
+const audioInputOptions = computed(() => audioInputs.value.map(input => ({
+  label: input.label || input.deviceId,
+  value: input.deviceId,
+})))
 </script>
 
 <template>
-  <div>
-    <div>
+  <div class="space-y-4">
+    <div class="flex items-center gap-2">
       <Button @click="() => toggleAudioDevice()">
         <span>{{ enabled ? 'Disable Microphone' : 'Enable Microphone' }}</span>
       </Button>
+      <Button variant="secondary" @click="requestPermission">
+        Request Microphone Access
+      </Button>
     </div>
+
+    <FieldSelect
+      v-model="selectedAudioInput"
+      label="Audio Input Device"
+      description="Select the audio input device"
+      :options="audioInputOptions"
+      placeholder="Select an audio input device"
+      layout="vertical"
+    />
   </div>
 </template>