From d56823f75c3062a7938423d26a2f9f5eb1524c18 Mon Sep 17 00:00:00 2001 From: Neko Ayaka Date: Fri, 25 Jul 2025 03:58:27 +0800 Subject: [PATCH] fix(stage-*): async audio implementation --- .../src/components/InteractiveArea.vue | 13 +- apps/stage-tamagotchi/src/pages/index.vue | 18 ++- .../src/pages/settings/modules/hearing.vue | 115 ++++++++++++++---- .../components/Layouts/InteractiveArea.vue | 15 ++- .../Layouts/MobileInteractiveArea.vue | 14 +-- .../src/pages/settings/modules/hearing.vue | 115 ++++++++++++++---- .../Providers/TranscriptionPlayground.vue | 53 +++----- .../src/composables/audio/audio-recorder.ts | 6 +- .../stage-ui/src/composables/audio/device.ts | 39 ++++++ .../stage-ui/src/composables/audio/index.ts | 1 + packages/stage-ui/src/stores/audio.ts | 54 +++++++- .../stage-ui/src/stores/modules/hearing.ts | 24 +++- packages/stage-ui/src/stores/settings.ts | 68 +++++++---- pnpm-lock.yaml | 115 ++++++++++++++++-- 14 files changed, 491 insertions(+), 159 deletions(-) create mode 100644 packages/stage-ui/src/composables/audio/device.ts diff --git a/apps/stage-tamagotchi/src/components/InteractiveArea.vue b/apps/stage-tamagotchi/src/components/InteractiveArea.vue index fd9e7231f..7ca5de62d 100644 --- a/apps/stage-tamagotchi/src/components/InteractiveArea.vue +++ b/apps/stage-tamagotchi/src/components/InteractiveArea.vue @@ -2,7 +2,7 @@ import type { ChatProvider } from '@xsai-ext/shared-providers' import { useMicVAD } from '@proj-airi/stage-ui/composables' -import { useChatStore, useConsciousnessStore, useProvidersStore, useSettings } from '@proj-airi/stage-ui/stores' +import { useChatStore, useConsciousnessStore, useProvidersStore, useSettingsAudioDevice } from '@proj-airi/stage-ui/stores' import { BasicTextarea } from '@proj-airi/ui' import { storeToRefs } from 'pinia' import { onMounted, ref, watch } from 'vue' @@ -13,9 +13,8 @@ import TamagotchiChatHistory from './ChatHistory.vue' const messageInput = ref('') const listening = ref(false) -// const { audioInputs } = useDevicesList({ constraints: { audio: true }, requestPermissions: true }) -// const { selectedAudioDevice, isAudioInputOn, selectedAudioDeviceId } = storeToRefs(useSettings()) -const { isAudioInputOn, selectedAudioDeviceId } = storeToRefs(useSettings()) +// const { askPermission } = useSettingsAudioDevice() +const { enabled, selectedAudioInput } = storeToRefs(useSettingsAudioDevice()) const { send, onAfterSend, discoverToolsCompatibility } = useChatStore() const { messages } = storeToRefs(useChatStore()) const { t } = useI18n() @@ -44,7 +43,7 @@ async function handleSend() { } } -const { destroy, start } = useMicVAD(selectedAudioDeviceId, { +const { destroy, start } = useMicVAD(selectedAudioInput, { onSpeechStart: () => { // TODO: interrupt the playback // TODO: interrupt any of the ongoing TTS @@ -86,8 +85,8 @@ function handleTranscription(_buffer: Float32Array) { // selectedAudioDevice.value = found // } -watch(isAudioInputOn, async (value) => { - if (value === 'false') { +watch(enabled, async (value) => { + if (value === false) { destroy() } }) diff --git a/apps/stage-tamagotchi/src/pages/index.vue b/apps/stage-tamagotchi/src/pages/index.vue index e44bae238..b882071e1 100644 --- a/apps/stage-tamagotchi/src/pages/index.vue +++ b/apps/stage-tamagotchi/src/pages/index.vue @@ -20,21 +20,19 @@ import { startClickThrough, stopClickThrough } from '../utils/windows' useTauriGlobalShortcuts() const windowStore = useWindowControlStore() -const { width, height } = useWindowSize() - -const centerPos = computed(() => ({ x: width.value / 2, y: height.value / 2 })) -const live2dFocusAt = ref(centerPos.value) +const resourcesStore = useResourcesStore() +const mcpStore = useMcpStore() const { listen } = useTauriEvent() const { invoke } = useTauriCore() +const { width, height } = useWindowSize() +const { connected, serverCmd, serverArgs } = storeToRefs(mcpStore) +const { scale, positionInPercentageString } = storeToRefs(useLive2d()) + +const centerPos = computed(() => ({ x: width.value / 2, y: height.value / 2 })) const { live2dLookAtX, live2dLookAtY, isCursorInside } = useTauriWindowClickThrough(centerPos) const shouldHideView = computed(() => isCursorInside.value && !windowStore.isControlActive && windowStore.isIgnoringMouseEvent) - -const resourcesStore = useResourcesStore() -const mcpStore = useMcpStore() -const { connected, serverCmd, serverArgs } = storeToRefs(mcpStore) - -const { scale, positionInPercentageString } = storeToRefs(useLive2d()) +const live2dFocusAt = ref(centerPos.value) watch([live2dLookAtX, live2dLookAtY], ([x, y]) => live2dFocusAt.value = { x, y }, { immediate: true }) diff --git a/apps/stage-tamagotchi/src/pages/settings/modules/hearing.vue b/apps/stage-tamagotchi/src/pages/settings/modules/hearing.vue index e31765379..da1232ba9 100644 --- a/apps/stage-tamagotchi/src/pages/settings/modules/hearing.vue +++ b/apps/stage-tamagotchi/src/pages/settings/modules/hearing.vue @@ -1,13 +1,12 @@