feat(stage-web): use tooltip for hearing adhoc desktop config

apps/stage-tamagotchi/src/renderer/components/StageIslands/ControlsIsland/ControlsIslandHearingConfig.vue

@@ -51,12 +51,9 @@ onUnmounted(async () => {
     v-model:enabled="enabled"
     v-model:selected-audio-input="selectedAudioInput"
     :granted="mediaAccessStatus !== 'denied' && mediaAccessStatus !== 'restricted'"
-    :audio-input-options="audioInputs"
+    :audio-inputs="audioInputs"
     :volume-level="volumeLevel"
   >
     <slot />
-    <template #extra>
-      <slot name="extra" />
-    </template>
   </HearingConfigDialog>
 </template>

apps/stage-tamagotchi/src/renderer/components/StageIslands/ControlsIsland/index.vue

@@ -19,7 +19,7 @@ const toggleDark = useToggle(isDark)
 
 const settingsAudioDeviceStore = useSettingsAudioDevice()
 const context = useElectronEventaContext()
-const { enabled: isAudioEnabled } = storeToRefs(settingsAudioDeviceStore)
+const { enabled } = storeToRefs(settingsAudioDeviceStore)
 const openSettings = useElectronEventaInvoke(electronOpenSettings)
 const openChat = useElectronEventaInvoke(electronOpenChat)
 
@@ -64,7 +64,7 @@ defineExpose({ hearingDialogOpen })
           <div class="relative">
             <ControlButton>
               <Transition name="fade" mode="out-in">
-                <IndicatorMicVolume v-if="isAudioEnabled" size-5 />
+                <IndicatorMicVolume v-if="enabled" size-5 />
                 <div v-else i-ph:microphone-slash size-5 text="neutral-800 dark:neutral-300" />
               </Transition>
             </ControlButton>

apps/stage-tamagotchi/src/renderer/pages/index.vue

@@ -17,8 +17,8 @@ import { debouncedRef, useBroadcastChannel, watchPausable } from '@vueuse/core'
 import { storeToRefs } from 'pinia'
 import { computed, onUnmounted, ref, toRef, watch } from 'vue'
 
-import ControlsIsland from '../components/Widgets/ControlsIsland/index.vue'
-import ResourceStatusIsland from '../components/Widgets/ResourceStatusIsland/index.vue'
+import ControlsIsland from '../components/StageIslands/ControlsIsland/index.vue'
+import ResourceStatusIsland from '../components/StageIslands/ResourceStatusIsland/index.vue'
 
 import { electron } from '../../shared/electron'
 import {

apps/stage-web/package.json

@@ -97,6 +97,7 @@
     "@iconify-json/eos-icons": "^1.2.4",
     "@iconify-json/lucide": "^1.2.71",
     "@iconify-json/mingcute": "^1.2.5",
+    "@iconify-json/ph": "^1.2.2",
     "@iconify-json/simple-icons": "^1.2.56",
     "@iconify-json/solar": "^1.2.5",
     "@iconify-json/svg-spinners": "^1.2.4",

apps/stage-web/src/components/Layouts/InteractiveArea.vue

@@ -1,24 +1,24 @@
 <script setup lang="ts">
 import type { ChatProvider } from '@xsai-ext/shared-providers'
 
-import { HearingConfigDialog } from '@proj-airi/stage-ui/components'
 import { useAudioAnalyzer } from '@proj-airi/stage-ui/composables'
 import { useAudioContext } from '@proj-airi/stage-ui/stores/audio'
 import { useChatStore } from '@proj-airi/stage-ui/stores/chat'
 import { useConsciousnessStore } from '@proj-airi/stage-ui/stores/modules/consciousness'
 import { useProvidersStore } from '@proj-airi/stage-ui/stores/providers'
 import { useSettings, useSettingsAudioDevice } from '@proj-airi/stage-ui/stores/settings'
-import { BasicTextarea } from '@proj-airi/ui'
+import { BasicTextarea, FieldSelect } from '@proj-airi/ui'
 import { useDark } from '@vueuse/core'
 import { storeToRefs } from 'pinia'
-import { onMounted, ref, watch } from 'vue'
+import { TooltipContent, TooltipProvider, TooltipRoot, TooltipTrigger } from 'reka-ui'
+import { computed, onUnmounted, ref, watch } from 'vue'
 import { useI18n } from 'vue-i18n'
 
 import ChatHistory from '../Widgets/ChatHistory.vue'
 import IndicatorMicVolume from '../Widgets/IndicatorMicVolume.vue'
 
 const messageInput = ref('')
-const hearingDialogOpen = ref(false)
+const hearingTooltipOpen = ref(false)
 const isComposing = ref(false)
 
 const providersStore = useProvidersStore()
@@ -59,9 +59,7 @@ async function handleSend() {
   }
 }
 
-// No inline VAD/whisper here; see pages/index.vue pipeline
-
-watch(hearingDialogOpen, async (value) => {
+watch(hearingTooltipOpen, async (value) => {
   if (value) {
     await askPermission()
   }
@@ -73,17 +71,18 @@ watch([activeProvider, activeModel], async () => {
   }
 })
 
-onMounted(() => {})
-
 onAfterMessageComposed(async () => {
   messageInput.value = ''
 })
 
 const { startAnalyzer, stopAnalyzer, volumeLevel } = useAudioAnalyzer()
+const normalizedVolume = computed(() => Math.min(1, Math.max(0, (volumeLevel.value ?? 0) / 100)))
 let analyzerSource: MediaStreamAudioSourceNode | undefined
 
 function teardownAnalyzer() {
-  try { analyzerSource?.disconnect() }
+  try {
+    analyzerSource?.disconnect()
+  }
   catch {}
   analyzerSource = undefined
   stopAnalyzer()
@@ -91,7 +90,7 @@ function teardownAnalyzer() {
 
 async function setupAnalyzer() {
   teardownAnalyzer()
-  if (!hearingDialogOpen.value || !enabled.value || !stream.value)
+  if (!hearingTooltipOpen.value || !enabled.value || !stream.value)
     return
   if (audioContext.state === 'suspended')
     await audioContext.resume()
@@ -102,9 +101,13 @@ async function setupAnalyzer() {
   analyzerSource.connect(analyser)
 }
 
-watch([hearingDialogOpen, enabled, stream], () => {
+watch([hearingTooltipOpen, enabled, stream], () => {
   setupAnalyzer()
 }, { immediate: true })
+
+onUnmounted(() => {
+  teardownAnalyzer()
+})
 </script>
 
 <template>
@@ -113,54 +116,107 @@ watch([hearingDialogOpen, enabled, stream], () => {
       <div
         flex="~ col"
         border="solid 4 primary-200/20 dark:primary-400/20"
-        h-full w-full overflow-scroll rounded-xl
+        h-full w-full rounded-xl
         bg="primary-50/50 dark:primary-950/70" backdrop-blur-md
       >
         <ChatHistory h-full flex-1 w="full" max-h="<md:[60%]" />
         <div h="<md:full" flex gap-2>
-          <BasicTextarea
-            v-model="messageInput"
-            :placeholder="t('stage.message')"
-            text="primary-500 hover:primary-600 dark:primary-300/50 dark:hover:primary-500 placeholder:primary-400 placeholder:hover:primary-500 placeholder:dark:primary-300/50 placeholder:dark:hover:primary-500"
-            bg="primary-200/20 dark:primary-400/20"
-            min-h="[100px]" max-h="[300px]" w-full
-            rounded-t-xl p-4 font-medium
-            outline-none transition="all duration-250 ease-in-out placeholder:all placeholder:duration-250 placeholder:ease-in-out"
-            :class="{
-              'transition-colors-none placeholder:transition-colors-none': themeColorsHueDynamic,
-            }"
-            @submit="handleSend"
-            @compositionstart="isComposing = true"
-            @compositionend="isComposing = false"
-          />
-
-          <HearingConfigDialog
-            v-model:show="hearingDialogOpen"
-            :overlay-dim="true"
-            :overlay-blur="true"
-            :enabled="enabled"
-            :audio-input-options="audioInputs"
-            :selected-audio-input="selectedAudioInput"
-            :has-devices="(audioInputs || []).length > 0"
-            :volume-level="volumeLevel"
-            @toggle-enabled="enabled = !enabled"
-            @update:selected-audio-input="val => selectedAudioInput = val as string"
+          <div
+            :class="[
+              'relative',
+              'w-full',
+              'bg-primary-200/20 dark:bg-primary-400/20',
+            ]"
           >
-            <button
-              class="max-h-[10lh] min-h-[1lh]"
-              bg="neutral-100 dark:neutral-800"
-              text="lg neutral-500 dark:neutral-400"
-              :class="{ 'ring-2 ring-primary-400/60 ring-offset-2 dark:ring-offset-neutral-900': enabled }"
-              flex items-center justify-center rounded-md p-2 outline-none
-              transition="colors duration-200, transform duration-100" active:scale-95
-              :title="t('settings.hearing.title')"
-            >
-              <Transition name="fade" mode="out-in">
-                <IndicatorMicVolume v-if="enabled" />
-                <div v-else class="i-ph:microphone-slash" />
-              </Transition>
-            </button>
-          </HearingConfigDialog>
+            <BasicTextarea
+              v-model="messageInput"
+              :placeholder="t('stage.message')"
+              text="primary-500 hover:primary-600 dark:primary-300/50 dark:hover:primary-500 placeholder:primary-400 placeholder:hover:primary-500 placeholder:dark:primary-300/50 placeholder:dark:hover:primary-500"
+              bg="transparent"
+              min-h="[100px]" max-h="[300px]" w-full
+              rounded-t-xl p-4 font-medium
+              outline-none transition="all duration-250 ease-in-out placeholder:all placeholder:duration-250 placeholder:ease-in-out"
+              :class="{
+                'transition-colors-none placeholder:transition-colors-none': themeColorsHueDynamic,
+              }"
+              @submit="handleSend"
+              @compositionstart="isComposing = true"
+              @compositionend="isComposing = false"
+            />
+
+            <div>
+              <TooltipProvider :delay-duration="0" :skip-delay-duration="0">
+                <TooltipRoot v-model:open="hearingTooltipOpen">
+                  <TooltipTrigger as-child>
+                    <button
+                      class="max-h-[10lh] min-h-[1lh]"
+                      text="lg neutral-500 dark:neutral-400"
+                      flex items-center justify-center rounded-md p-2 outline-none
+                      transition="colors duration-200, transform duration-100" active:scale-95
+                      :title="t('settings.hearing.title')"
+                    >
+                      <Transition name="fade" mode="out-in">
+                        <IndicatorMicVolume v-if="enabled" />
+                        <div v-else class="i-ph:microphone-slash" />
+                      </Transition>
+                    </button>
+                  </TooltipTrigger>
+                  <Transition name="fade">
+                    <TooltipContent
+                      side="top"
+                      :side-offset="8"
+                      :class="[
+                        'w-72 max-w-[18rem] rounded-xl border border-neutral-200/60 bg-neutral-50/90 p-4',
+                        'shadow-lg backdrop-blur-md dark:border-neutral-800/30 dark:bg-neutral-900/80',
+                        'flex flex-col gap-3',
+                      ]"
+                    >
+                      <div class="flex flex-col items-center justify-center">
+                        <div class="relative h-28 w-28 select-none">
+                          <div
+                            class="absolute left-1/2 top-1/2 h-20 w-20 rounded-full transition-all duration-150 -translate-x-1/2 -translate-y-1/2"
+                            :style="{ transform: `translate(-50%, -50%) scale(${1 + normalizedVolume * 0.35})`, opacity: String(0.25 + normalizedVolume * 0.25) }"
+                            :class="enabled ? 'bg-primary-500/15 dark:bg-primary-600/20' : 'bg-neutral-300/20 dark:bg-neutral-700/20'"
+                          />
+                          <div
+                            class="absolute left-1/2 top-1/2 h-24 w-24 rounded-full transition-all duration-200 -translate-x-1/2 -translate-y-1/2"
+                            :style="{ transform: `translate(-50%, -50%) scale(${1.2 + normalizedVolume * 0.55})`, opacity: String(0.15 + normalizedVolume * 0.2) }"
+                            :class="enabled ? 'bg-primary-500/10 dark:bg-primary-600/15' : 'bg-neutral-300/10 dark:bg-neutral-700/10'"
+                          />
+                          <div
+                            class="absolute left-1/2 top-1/2 h-28 w-28 rounded-full transition-all duration-300 -translate-x-1/2 -translate-y-1/2"
+                            :style="{ transform: `translate(-50%, -50%) scale(${1.5 + normalizedVolume * 0.8})`, opacity: String(0.08 + normalizedVolume * 0.15) }"
+                            :class="enabled ? 'bg-primary-500/5 dark:bg-primary-600/10' : 'bg-neutral-300/5 dark:bg-neutral-700/5'"
+                          />
+                          <button
+                            class="absolute left-1/2 top-1/2 grid h-16 w-16 place-items-center rounded-full shadow-md outline-none transition-all duration-200 -translate-x-1/2 -translate-y-1/2"
+                            :class="enabled
+                              ? 'bg-primary-500 text-white hover:bg-primary-600 active:scale-95'
+                              : 'bg-neutral-200 text-neutral-600 hover:bg-neutral-300 active:scale-95 dark:bg-neutral-700 dark:text-neutral-200'"
+                            @click="enabled = !enabled"
+                          >
+                            <div :class="enabled ? 'i-ph:microphone' : 'i-ph:microphone-slash'" class="h-6 w-6" />
+                          </button>
+                        </div>
+                        <p class="mt-3 text-xs text-neutral-500 dark:text-neutral-400">
+                          {{ enabled ? 'Microphone enabled' : 'Microphone disabled' }}
+                        </p>
+                      </div>
+
+                      <FieldSelect
+                        v-model="selectedAudioInput"
+                        label="Input device"
+                        description="Select the microphone you want to use."
+                        :options="audioInputs.map(device => ({ label: device.label || 'Unknown Device', value: device.deviceId }))"
+                        layout="vertical"
+                        placeholder="Select microphone"
+                      />
+                    </TooltipContent>
+                  </Transition>
+                </TooltipRoot>
+              </TooltipProvider>
+            </div>
+          </div>
         </div>
       </div>
     </div>

apps/stage-web/src/components/Layouts/MobileInteractiveArea.vue

@@ -2,7 +2,8 @@
 import type { ChatProvider } from '@xsai-ext/shared-providers'
 
 import { HearingConfigDialog } from '@proj-airi/stage-ui/components'
-import { useMicVAD } from '@proj-airi/stage-ui/composables'
+import { useAudioAnalyzer } from '@proj-airi/stage-ui/composables'
+import { useAudioContext } from '@proj-airi/stage-ui/stores/audio'
 import { useChatStore } from '@proj-airi/stage-ui/stores/chat'
 import { useConsciousnessStore } from '@proj-airi/stage-ui/stores/modules/consciousness'
 import { useProvidersStore } from '@proj-airi/stage-ui/stores/providers'
@@ -10,10 +11,11 @@ import { useSettings, useSettingsAudioDevice } from '@proj-airi/stage-ui/stores/
 import { BasicTextarea } from '@proj-airi/ui'
 import { useDark, useResizeObserver, useScreenSafeArea } from '@vueuse/core'
 import { storeToRefs } from 'pinia'
-import { onMounted, ref, useTemplateRef, watch } from 'vue'
+import { onMounted, onUnmounted, ref, useTemplateRef, watch } from 'vue'
 import { useI18n } from 'vue-i18n'
 import { RouterLink } from 'vue-router'
 
+import IndicatorMicVolume from '../Widgets/IndicatorMicVolume.vue'
 import MobileChatHistory from '../Widgets/MobileChatHistory.vue'
 import ActionAbout from './InteractiveArea/Actions/About.vue'
 import ActionViewControls from './InteractiveArea/Actions/ViewControls.vue'
@@ -26,7 +28,6 @@ const viewControlsActiveMode = ref<'x' | 'y' | 'z' | 'scale'>('scale')
 const viewControlsInputsRef = useTemplateRef<InstanceType<typeof ViewControlInputs>>('viewControlsInputs')
 
 const messageInput = ref('')
-const listening = ref(false)
 const isComposing = ref(false)
 
 const screenSafeArea = useScreenSafeArea()
@@ -34,13 +35,15 @@ const providersStore = useProvidersStore()
 const { activeProvider, activeModel } = storeToRefs(useConsciousnessStore())
 
 useResizeObserver(document.documentElement, () => screenSafeArea.update())
-
-// const { askPermission } = useSettingsAudioDevice()
 const { themeColorsHueDynamic, stageViewControlsEnabled } = storeToRefs(useSettings())
-const { enabled, selectedAudioInput } = storeToRefs(useSettingsAudioDevice())
+const settingsAudioDevice = useSettingsAudioDevice()
+const { enabled, selectedAudioInput, stream, audioInputs } = storeToRefs(settingsAudioDevice)
 const { send, onAfterMessageComposed, discoverToolsCompatibility, cleanupMessages } = useChatStore()
 const { messages } = storeToRefs(useChatStore())
 const { t } = useI18n()
+const { audioContext } = useAudioContext()
+const { startAnalyzer, stopAnalyzer, volumeLevel } = useAudioAnalyzer()
+let analyzerSource: MediaStreamAudioSourceNode | undefined
 
 function isMobileDevice() {
   return /Mobi|Android|iPhone|iPad|iPod|BlackBerry|IEMobile|Opera Mini/i.test(navigator.userAgent)
@@ -75,40 +78,35 @@ async function handleSend() {
   }
 }
 
-const { destroy, start } = useMicVAD(selectedAudioInput, {
-  onSpeechStart: () => {
-    // TODO: interrupt the playback
-    // TODO: interrupt any of the ongoing TTS
-    // TODO: interrupt any of the ongoing LLM requests
-    // TODO: interrupt any of the ongoing animation of Live2D or VRM
-    // TODO: once interrupted, we should somehow switch to listen or thinking
-    //       emotion / expression?
-    listening.value = true
-  },
-  // VAD misfire means while speech end is detected but
-  // the frames of the segment of the audio buffer
-  // is not enough to be considered as a speech segment
-  // which controlled by the `minSpeechFrames` parameter
-  onVADMisfire: () => {
-    // TODO: do audio buffer send to whisper
-    listening.value = false
-  },
-  onSpeechEnd: (buffer) => {
-    // TODO: do audio buffer send to whisper
-    listening.value = false
-    handleTranscription(buffer)
-  },
-  auto: false,
-})
-
-function handleTranscription(_buffer: Float32Array<ArrayBufferLike>) {
-  // eslint-disable-next-line no-alert
-  alert('Transcription is not implemented yet')
+function teardownAnalyzer() {
+  try {
+    analyzerSource?.disconnect()
+  }
+  catch {}
+  analyzerSource = undefined
+  stopAnalyzer()
 }
 
-watch(enabled, async (value) => {
-  if (value === false) {
-    destroy()
+async function setupAnalyzer() {
+  teardownAnalyzer()
+  if (!hearingDialogOpen.value || !enabled.value || !stream.value)
+    return
+  if (audioContext.state === 'suspended')
+    await audioContext.resume()
+  const analyser = startAnalyzer(audioContext)
+  if (!analyser)
+    return
+  analyzerSource = audioContext.createMediaStreamSource(stream.value)
+  analyzerSource.connect(analyser)
+}
+
+watch([hearingDialogOpen, enabled, stream], () => {
+  setupAnalyzer()
+}, { immediate: true })
+
+watch(hearingDialogOpen, (value) => {
+  if (value) {
+    settingsAudioDevice.askPermission()
   }
 })
 
@@ -122,8 +120,11 @@ watch([activeProvider, activeModel], async () => {
   }
 })
 
+onUnmounted(() => {
+  teardownAnalyzer()
+})
+
 onMounted(() => {
-  start()
   screenSafeArea.update()
 })
 </script>
@@ -142,7 +143,14 @@ onMounted(() => {
       <div translate-y="[-100%]" absolute right-0 w-full px-3 pb-3 font-sans>
         <div flex="~ col" w-full gap-1>
           <ActionAbout />
-          <HearingConfigDialog v-model:show="hearingDialogOpen">
+          <HearingConfigDialog
+            v-model:show="hearingDialogOpen"
+            v-model:enabled="enabled"
+            v-model:selected-audio-input="selectedAudioInput"
+            :audio-inputs="audioInputs"
+            :volume-level="volumeLevel"
+            :granted="true"
+          >
             <button
               border="2 solid neutral-100/60 dark:neutral-800/30"
               bg="neutral-50/70 dark:neutral-800/70"
@@ -150,7 +158,7 @@ onMounted(() => {
               title="Hearing"
             >
               <Transition name="fade" mode="out-in">
-                <div v-if="enabled" i-solar:microphone-3-bold-duotone size-5 text="neutral-500 dark:neutral-400" />
+                <IndicatorMicVolume v-if="enabled" size-5 color-class="text-neutral-500 dark:text-neutral-400" />
                 <div v-else i-solar:microphone-3-outline size-5 text="neutral-500 dark:neutral-400" />
               </Transition>
             </button>

packages/stage-ui/src/components/scenarios/dialogs/audio-input/hearing-config-dialog.vue

@@ -10,7 +10,7 @@ const props = defineProps<{
   overlayDim?: boolean
   overlayBlur?: boolean
   granted?: boolean
-  audioInputOptions?: MediaDeviceInfo[]
+  audioInputs?: MediaDeviceInfo[]
   volumeLevel?: number
 }>()
 
@@ -45,7 +45,7 @@ onMounted(() => screenSafeArea.update())
         <HearingConfig
           v-model:enabled="enabled"
           v-model:selected-audio-input="selectedAudioInput"
-          :audio-input-options="props.audioInputOptions"
+          :audio-inputs="props.audioInputs"
           :granted="props.granted"
           :volume-level="props.volumeLevel"
         />
@@ -64,7 +64,7 @@ onMounted(() => screenSafeArea.update())
         <HearingConfig
           v-model:enabled="enabled"
           v-model:selected-audio-input="selectedAudioInput"
-          :audio-input-options="props.audioInputOptions"
+          :audio-inputs="props.audioInputs"
           :granted="props.granted"
           :volume-level="props.volumeLevel"
         />

packages/stage-ui/src/components/scenarios/dialogs/audio-input/hearing-config.vue

@@ -7,12 +7,12 @@ import Callout from '../../../layouts/Callout.vue'
 const props = withDefaults(defineProps<{
   enabled?: boolean
   granted?: boolean
-  audioInputOptions?: MediaDeviceInfo[]
+  audioInputs?: MediaDeviceInfo[]
   volumeLevel?: number
 }>(), {
   enabled: false,
   granted: false,
-  audioInputOptions: () => [],
+  audioInputs: () => [],
   volumeLevel: 0,
 })
 
@@ -78,7 +78,7 @@ const ringEnabledClass = computed(() => enabled.value
         v-model="selectedAudioInput"
         label="Input device"
         description="Select the microphone you want to use."
-        :options="props.audioInputOptions.map(device => ({ label: device.label || 'Unknown Device', value: device.deviceId }))"
+        :options="props.audioInputs.map(device => ({ label: device.label || 'Unknown Device', value: device.deviceId }))"
         placeholder="Select microphone"
         layout="vertical"
       />

pnpm-lock.yaml

@@ -889,6 +889,9 @@ importers:
       '@iconify-json/mingcute':
         specifier: ^1.2.5
         version: 1.2.5
+      '@iconify-json/ph':
+        specifier: ^1.2.2
+        version: 1.2.2
       '@iconify-json/simple-icons':
         specifier: ^1.2.56
         version: 1.2.56