diff --git a/apps/stage-tamagotchi/src/renderer/components/stage-islands/controls-island/controls-island-hearing-config.vue b/apps/stage-tamagotchi/src/renderer/components/stage-islands/controls-island/controls-island-hearing-config.vue index f6cbcfd71..99f04c979 100644 --- a/apps/stage-tamagotchi/src/renderer/components/stage-islands/controls-island/controls-island-hearing-config.vue +++ b/apps/stage-tamagotchi/src/renderer/components/stage-islands/controls-island/controls-island-hearing-config.vue @@ -12,7 +12,6 @@ const show = defineModel('show', { type: Boolean, default: false }) const settingsAudioDeviceStore = useSettingsAudioDevice() const { enabled, selectedAudioInput, stream, audioInputs } = storeToRefs(settingsAudioDeviceStore) -const { startStream, stopStream } = settingsAudioDeviceStore const getMediaAccessStatus = useElectronEventaInvoke(electron.systemPreferences.getMediaAccessStatus) const { state: mediaAccessStatus, execute: refreshMediaAccessStatus } = useAsyncState(() => getMediaAccessStatus(['microphone']), 'not-determined') @@ -20,13 +19,25 @@ const { state: mediaAccessStatus, execute: refreshMediaAccessStatus } = useAsync const { audioContext, initialize, dispose, pause } = useAudioContextFromStream(stream) const { volumeLevel, startAnalyzer, stopAnalyzer } = useAudioAnalyzer() -watch(enabled, (val) => { - if (val) { - startStream() - initialize().then(() => startAnalyzer(audioContext.value!)) +// NOTICE: Do not call `startStream()` / `stopStream()` from this component. +// +// `useSettingsAudioDevice()` already owns the mic stream lifecycle via the persisted `enabled` state. +// We previously toggled the stream here as well, which introduced a second lifecycle controller: the +// dialog could recreate the MediaStream while the page-level transcription pipeline still believed +// the old session was active. +// +// That produced the "VAD still works, but no transcript arrives" failure after retoggling the mic. +// +// This component should only react to the current stream to drive analyzer UI state. +watch([enabled, stream], ([isEnabled, currentStream]) => { + if (isEnabled && currentStream) { + initialize().then(() => { + if (audioContext.value) + return startAnalyzer(audioContext.value) + }) } else { - stopStream() + stopAnalyzer() pause() } }, { immediate: true }) diff --git a/apps/stage-tamagotchi/src/renderer/pages/index.vue b/apps/stage-tamagotchi/src/renderer/pages/index.vue index 1f4783de6..9d7afb65c 100644 --- a/apps/stage-tamagotchi/src/renderer/pages/index.vue +++ b/apps/stage-tamagotchi/src/renderer/pages/index.vue @@ -240,6 +240,7 @@ const { init: initVAD, dispose: disposeVAD, start: startVAD, loaded: vadLoaded } }) let stopOnStopRecord: (() => void) | undefined +const audioInteractionStarting = ref(false) // Caption overlay broadcast channel type CaptionChannelEvent @@ -247,6 +248,37 @@ type CaptionChannelEvent | { type: 'caption-assistant', text: string } const { post: postCaption } = useBroadcastChannel({ name: 'airi-caption-overlay' }) +function handleStreamingSentenceEnd(delta: string) { + console.info('[Main Page] Received transcription delta:', delta) + const finalText = delta + if (!finalText || !finalText.trim()) { + return + } + + postCaption({ type: 'caption-speaker', text: finalText }) + + void (async () => { + try { + const provider = await providersStore.getProviderInstance(activeChatProvider.value) + if (!provider || !activeChatModel.value) { + console.warn('[Main Page] No provider or model available, skipping chat send') + return + } + + console.info('[Main Page] Sending transcription to chat:', finalText) + await chatStore.ingest(finalText, { model: activeChatModel.value, chatProvider: provider as ChatProvider }) + } + catch (err) { + console.error('[Main Page] Failed to send chat from voice:', err) + } + })() +} + +function handleStreamingSpeechEnd(text: string) { + console.info('[Main Page] Speech ended, final text:', text) + postCaption({ type: 'caption-speaker', text }) +} + async function handleSpeechStart() { if (shouldUseStreamInput.value) { console.info('Speech detected - transcription session should already be active') @@ -266,6 +298,19 @@ async function handleSpeechEnd() { } async function startAudioInteraction() { + if (audioInteractionStarting.value) + return + + // NOTICE: `stopOnStopRecord` only tracks whether the non-stream recording hook was registered. + // + // It does NOT guarantee that the current realtime transcription session is still attached to the + // latest `MediaStream`. We previously used it as a generic "already started" guard, which broke + // the hearing-config retoggle path: the mic stream was recreated, VAD restarted on the new stream, + // but `transcribeForMediaStream()` never reattached so speech was detected without any transcript. + // + // Keep the startup guard scoped to "startup in progress" only, and let stream changes restart the + // transcription binding when a new stream arrives. + audioInteractionStarting.value = true try { console.info('[Main Page] Starting audio interaction...') @@ -291,35 +336,8 @@ async function startAudioInteraction() { // Use sentence deltas for live captions and speech end for final text. await transcribeForMediaStream(stream.value, { - onSentenceEnd: (delta) => { - console.info('[Main Page] Received transcription delta:', delta) - const finalText = delta - if (!finalText || !finalText.trim()) { - return - } - - postCaption({ type: 'caption-speaker', text: finalText }) - - void (async () => { - try { - const provider = await providersStore.getProviderInstance(activeChatProvider.value) - if (!provider || !activeChatModel.value) { - console.warn('[Main Page] No provider or model available, skipping chat send') - return - } - - console.info('[Main Page] Sending transcription to chat:', finalText) - await chatStore.ingest(finalText, { model: activeChatModel.value, chatProvider: provider as ChatProvider }) - } - catch (err) { - console.error('[Main Page] Failed to send chat from voice:', err) - } - })() - }, - onSpeechEnd: (text) => { - console.info('[Main Page] Speech ended, final text:', text) - postCaption({ type: 'caption-speaker', text }) - }, + onSentenceEnd: handleStreamingSentenceEnd, + onSpeechEnd: handleStreamingSpeechEnd, }) console.info('[Main Page] Streaming transcription started successfully') @@ -332,45 +350,58 @@ async function startAudioInteraction() { }) } - // Hook once - stopOnStopRecord = onStopRecord(async (recording) => { - if (shouldUseStreamInput.value) - return - - const text = await transcribeForRecording(recording) - if (!text || !text.trim()) - return - - // Update caption overlay speaker text via BroadcastChannel - postCaption({ type: 'caption-speaker', text }) - - try { - const provider = await providersStore.getProviderInstance(activeChatProvider.value) - if (!provider || !activeChatModel.value) + // NOTICE: This hook is only for record-then-transcribe providers. + // + // Streaming providers use the active `MediaStream` directly, so this callback must not be treated + // as proof that a realtime session is alive. Future refactors should keep recorder-hook bookkeeping + // separate from stream transcription state, otherwise mic/device re-toggles can leave VAD active + // but transcription detached. + // + // Hook once for non-streaming providers. + if (!stopOnStopRecord) { + stopOnStopRecord = onStopRecord(async (recording) => { + if (shouldUseStreamInput.value) return - await chatStore.ingest(text, { model: activeChatModel.value, chatProvider: provider as ChatProvider }) - } - catch (err) { - console.error('Failed to send chat from voice:', err) - } - }) + const text = await transcribeForRecording(recording) + if (!text || !text.trim()) + return + + // Update caption overlay speaker text via BroadcastChannel + postCaption({ type: 'caption-speaker', text }) + + try { + const provider = await providersStore.getProviderInstance(activeChatProvider.value) + if (!provider || !activeChatModel.value) + return + + await chatStore.ingest(text, { model: activeChatModel.value, chatProvider: provider as ChatProvider }) + } + catch (err) { + console.error('Failed to send chat from voice:', err) + } + }) + } } catch (e) { console.error('Audio interaction init failed:', e) } + finally { + audioInteractionStarting.value = false + } } function stopAudioInteraction() { tryCatch(() => { stopOnStopRecord?.() stopOnStopRecord = undefined + audioInteractionStarting.value = false void stopStreamingTranscription(true) disposeVAD() }) } -watch([enabled, stream], async ([val]) => { +watch(enabled, async (val) => { console.info('[Main Page] Audio enabled changed:', val, 'stream available:', !!stream.value) if (val) { await askPermission() @@ -395,6 +426,18 @@ onUnmounted(() => { stopAudioInteraction() }) +watch(stream, async (currentStream) => { + if (!enabled.value || !currentStream || audioInteractionStarting.value) + return + + // NOTICE: The controls-island mic toggle and device changes can replace the underlying MediaStream + // without reloading the page. When that happens, VAD may successfully restart against the new stream, + // but any existing transcription transport is still bound to the old one. Always allow the page to + // re-run `startAudioInteraction()` for a newly available stream unless startup is already underway. + console.info('[Main Page] Stream became available, ensuring audio interaction is started') + await startAudioInteraction() +}) + watch([stream, () => vadLoaded.value], async ([s, loaded]) => { if (enabled.value && loaded && s) { try { diff --git a/packages/stage-ui/src/stores/modules/hearing.ts b/packages/stage-ui/src/stores/modules/hearing.ts index d03e9e259..9898dbcae 100644 --- a/packages/stage-ui/src/stores/modules/hearing.ts +++ b/packages/stage-ui/src/stores/modules/hearing.ts @@ -2,7 +2,7 @@ import type { TranscriptionProviderWithExtraOptions } from '@xsai-ext/providers/ import type { WithUnknown } from '@xsai/shared' import type { StreamTranscriptionResult, StreamTranscriptionOptions as XSAIStreamTranscriptionOptions } from '@xsai/stream-transcription' -import { tryCatch } from '@moeru/std' +import { errorMessageFrom, tryCatch } from '@moeru/std' import { useLocalStorageManualReset } from '@proj-airi/stage-shared/composables' import { refManualReset } from '@vueuse/core' import { generateTranscription } from '@xsai/generate-transcription' @@ -16,7 +16,7 @@ import { streamAliyunTranscription } from '../providers/aliyun/stream-transcript import { streamWebSpeechAPITranscription } from '../providers/web-speech-api' function errorMessage(err: unknown): string { - const msg = err instanceof Error ? err.message : String(err) + const msg = errorMessageFrom(err) ?? String(err) // Browsers hide the real reason (CORS, timeout, DNS, …) behind this generic string. if (msg === 'Failed to fetch' || msg === 'Load failed') { return `${msg} — check the browser console (Network tab) for the exact reason (e.g. CORS, network timeout, DNS failure).` @@ -24,6 +24,33 @@ function errorMessage(err: unknown): string { return msg } +// NOTICE: Realtime transcription intentionally uses `AbortError` as a control-flow signal when the +// current stream session is being stopped on purpose. +// +// This happens in `stopStreamingTranscription()`, +// which aborts the session with one of the DOMException messages below when the user disables the mic, +// the page tears down audio interaction, callbacks are intentionally rebound, or the idle timeout closes +// an inactive stream. Those cases should not be surfaced as provider failures because the session was +// explicitly asked to stop. If a future abort is noisy or unexpected, inspect the abort source first: +// `stopStreamingTranscription()` in this file is the primary origin, and provider-specific teardown +// bridges such as `packages/stage-ui/src/stores/providers/aliyun/stream-transcription.ts` propagate the +// same reason through the transport. Only treat an abort as "expected" if it is one of these known +// shutdown paths; any other `AbortError` should still be investigated as a real lifecycle bug or a +// provider/runtime failure. +function isExpectedStreamStopError(err: unknown): boolean { + return err instanceof DOMException + && err.name === 'AbortError' + && (err.message === 'Stopped' || err.message === 'Aborted' || err.message === 'Closed' || err.message === 'Idle timeout') +} + +function haveStreamingCallbacksChanged( + previous: { onSentenceEnd?: (delta: string) => void, onSpeechEnd?: (text: string) => void } | undefined, + next: { onSentenceEnd?: (delta: string) => void, onSpeechEnd?: (text: string) => void }, +): boolean { + return (next.onSentenceEnd !== undefined && next.onSentenceEnd !== previous?.onSentenceEnd) + || (next.onSpeechEnd !== undefined && next.onSpeechEnd !== previous?.onSpeechEnd) +} + export interface StreamTranscriptionFileInputOptions extends Omit { file: Blob fileName?: string @@ -357,6 +384,9 @@ export const useHearingSpeechInputPipeline = defineStore('modules:hearing:speech return text } catch (err) { + if (isExpectedStreamStopError(err)) + return + error.value = errorMessage(err) console.error('Error getting transcription result:', error.value) } @@ -402,6 +432,9 @@ export const useHearingSpeechInputPipeline = defineStore('modules:hearing:speech return text } catch (err) { + if (isExpectedStreamStopError(err)) + return + error.value = errorMessage(err) console.error('Error generating transcription:', error.value) } @@ -460,15 +493,15 @@ export const useHearingSpeechInputPipeline = defineStore('modules:hearing:speech // Check if session already exists and reuse it const existingSession = streamingSession.value if (existingSession && existingSession.providerId === 'browser-web-speech-api') { + const nextCallbacks = { + onSentenceEnd: options?.onSentenceEnd, + onSpeechEnd: options?.onSpeechEnd, + } // For Web Speech API, if callbacks are provided and different, we need to restart // because recognition instance callbacks are set once and can't be changed - // However, if no new callbacks are provided, we can just reuse the session - const hasNewCallbacks = !!(options?.onSentenceEnd || options?.onSpeechEnd) + const hasNewCallbacks = haveStreamingCallbacksChanged(existingSession.callbacks, nextCallbacks) if (hasNewCallbacks) { - // We need to restart to use new callbacks, but only if they're actually different - // Since we can't compare functions, we'll just always restart if new callbacks are provided - // This ensures callbacks are always up-to-date console.info('Web Speech API: New callbacks provided, restarting session to use them') await stopStreamingTranscription(false, existingSession.providerId) // Continue to create new session below @@ -580,7 +613,8 @@ export const useHearingSpeechInputPipeline = defineStore('modules:hearing:speech } } catch (err) { - console.error('Error reading text stream:', err) + if (!isExpectedStreamStopError(err)) + console.error('Error reading text stream:', err) } })() } @@ -600,9 +634,10 @@ export const useHearingSpeechInputPipeline = defineStore('modules:hearing:speech // require restarting the session to create a new reader. const existingSession = streamingSession.value if (existingSession) { - const hasNewCallbacks - = options?.onSentenceEnd !== undefined - || options?.onSpeechEnd !== undefined + const hasNewCallbacks = haveStreamingCallbacksChanged(existingSession.callbacks, { + onSentenceEnd: options?.onSentenceEnd, + onSpeechEnd: options?.onSpeechEnd, + }) if (hasNewCallbacks) { console.info('[Hearing Pipeline] New callbacks provided, restarting session') @@ -699,7 +734,8 @@ export const useHearingSpeechInputPipeline = defineStore('modules:hearing:speech } } catch (err) { - console.error('Error reading text stream:', err) + if (!isExpectedStreamStopError(err)) + console.error('Error reading text stream:', err) } finally { // Use captured callbacks to avoid cross-session leakage @@ -709,6 +745,9 @@ export const useHearingSpeechInputPipeline = defineStore('modules:hearing:speech } } catch (err) { + if (isExpectedStreamStopError(err)) + return + error.value = errorMessage(err) console.error('Error generating transcription:', error.value) }