From 62a01d83b9ff1063e7431ca464e7f11512831653 Mon Sep 17 00:00:00 2001 From: Lovehsigure_520 <62863834+Neko-233@users.noreply.github.com> Date: Sat, 11 Jul 2026 00:25:07 +0800 Subject: [PATCH] feat(stage-tamagotchi,stage-ui,pipelines-audio): integrate voice input composable into tamagotchi (#2004) --------- Authored-by-agent: Codex --- apps/stage-tamagotchi/package.json | 1 + .../src/renderer/pages/index.vue | 533 ++++++++++++------ .../utils/voice-input-lifecycle.test.ts | 85 +++ .../renderer/utils/voice-input-lifecycle.ts | 85 +++ .../utils/voice-input-suppression.test.ts | 44 ++ .../renderer/utils/voice-input-suppression.ts | 42 ++ packages/pipelines-audio/README.md | 30 + packages/pipelines-audio/src/index.ts | 1 + .../src/transcript-buffer.test.ts | 169 ++++++ .../pipelines-audio/src/transcript-buffer.ts | 125 ++++ .../src/stores/settings/audio-device.test.ts | 59 +- .../src/stores/settings/audio-device.ts | 18 +- pnpm-lock.yaml | 3 + 13 files changed, 1007 insertions(+), 188 deletions(-) create mode 100644 apps/stage-tamagotchi/src/renderer/utils/voice-input-lifecycle.test.ts create mode 100644 apps/stage-tamagotchi/src/renderer/utils/voice-input-lifecycle.ts create mode 100644 apps/stage-tamagotchi/src/renderer/utils/voice-input-suppression.test.ts create mode 100644 apps/stage-tamagotchi/src/renderer/utils/voice-input-suppression.ts create mode 100644 packages/pipelines-audio/README.md create mode 100644 packages/pipelines-audio/src/transcript-buffer.test.ts create mode 100644 packages/pipelines-audio/src/transcript-buffer.ts diff --git a/apps/stage-tamagotchi/package.json b/apps/stage-tamagotchi/package.json index 90e822467..6b1726889 100644 --- a/apps/stage-tamagotchi/package.json +++ b/apps/stage-tamagotchi/package.json @@ -62,6 +62,7 @@ "@proj-airi/font-cjkfonts-allseto": "workspace:^", "@proj-airi/font-xiaolai": "workspace:^", "@proj-airi/i18n": "workspace:^", + "@proj-airi/pipelines-audio": "workspace:^", "@proj-airi/plugin-sdk-tamagotchi": "workspace:^", "@proj-airi/server-sdk": "workspace:*", "@proj-airi/stage-layouts": "workspace:^", diff --git a/apps/stage-tamagotchi/src/renderer/pages/index.vue b/apps/stage-tamagotchi/src/renderer/pages/index.vue index 09821c8f6..aadea7902 100644 --- a/apps/stage-tamagotchi/src/renderer/pages/index.vue +++ b/apps/stage-tamagotchi/src/renderer/pages/index.vue @@ -3,9 +3,7 @@ import type { ModelSettingsRuntimeSnapshot } from '@proj-airi/stage-ui/component import type { ModelSettingsRuntimeChannelEvent } from '../../shared/model-settings-runtime' -import workletUrl from '@proj-airi/stage-ui/workers/vad/process.worklet?worker&url' - -import { tryCatch } from '@moeru/std' +import { errorMessageFrom, tryCatch } from '@moeru/std' import { electron } from '@proj-airi/electron-eventa' import { useElectronEventaInvoke, @@ -14,6 +12,7 @@ import { useElectronMouseInWindow, useElectronRelativeMouse, } from '@proj-airi/electron-vueuse' +import { createTranscriptBuffer } from '@proj-airi/pipelines-audio' import { IS_DEV } from '@proj-airi/stage-shared' import { useModelStore, useThreeSceneIsTransparentAtPoint } from '@proj-airi/stage-ui-three' import { HoloCoupon } from '@proj-airi/stage-ui/components' @@ -22,15 +21,16 @@ import { resolveComponentStateToRuntimePhase, } from '@proj-airi/stage-ui/components/scenarios/settings/model-settings/runtime' import { WidgetStage } from '@proj-airi/stage-ui/components/scenes' -import { useAudioRecorder } from '@proj-airi/stage-ui/composables/audio/audio-recorder' +import { useVoiceInputSession } from '@proj-airi/stage-ui/composables' import { useCanvasPixelIsTransparentAtPoint } from '@proj-airi/stage-ui/composables/canvas-alpha' -import { useVAD } from '@proj-airi/stage-ui/stores/ai/models/vad' -import { useHearingSpeechInputPipeline } from '@proj-airi/stage-ui/stores/modules/hearing' +import { useSpeakingStore } from '@proj-airi/stage-ui/stores/audio' +import { useHearingSpeechInputPipeline, useHearingStore } from '@proj-airi/stage-ui/stores/modules/hearing' import { useOnboardingStore } from '@proj-airi/stage-ui/stores/onboarding' import { useSettings, useSettingsAudioDevice } from '@proj-airi/stage-ui/stores/settings' import { refDebounced, useBroadcastChannel } from '@vueuse/core' import { storeToRefs } from 'pinia' -import { computed, onMounted, onUnmounted, ref, toRef, watch } from 'vue' +import { computed, onMounted, onUnmounted, ref, shallowRef, toRef, watch } from 'vue' +import { toast } from 'vue-sonner' import ControlsIsland from '../components/stage-islands/controls-island/index.vue' import ResourceStatusIsland from '../components/stage-islands/resource-status-island/index.vue' @@ -42,6 +42,12 @@ import { useChatSyncStore } from '../stores/chat-sync' import { useControlsIslandStore } from '../stores/controls-island' import { useStageWindowLifecycleStore } from '../stores/stage-window-lifecycle' import { shouldSampleStageTransparency } from '../utils/stage-three-transparency' +import { createVoiceInputInteractionLifecycle } from '../utils/voice-input-lifecycle' +import { + assistantSpeechCooldownDeadline, + DEFAULT_ASSISTANT_SPEECH_INPUT_COOLDOWN_MS, + shouldSuppressVoiceInput, +} from '../utils/voice-input-suppression' const controlsIslandRef = ref>() const statusIslandRef = ref>() @@ -222,39 +228,60 @@ watch([isOutsideFor250Ms, isOutsideStatusIslandFor250Ms, isAroundWindowBorderFor }) // Emit runtime snapshot on change and on request from settings panel +/** + * Sends model-settings runtime events without letting closed HMR channels break the stage. + */ +function postModelSettingsRuntimeEvent(event: ModelSettingsRuntimeChannelEvent) { + const { error } = tryCatch(() => postModelSettingsRuntimeChannelEvent(event)) + if (error) + console.warn('[Main Page] Failed to post model settings runtime event:', error) +} + watch(modelSettingsRuntimeSnapshot, (snapshot) => { - postModelSettingsRuntimeChannelEvent({ type: 'snapshot', snapshot }) + postModelSettingsRuntimeEvent({ type: 'snapshot', snapshot }) }, { immediate: true }) watch(modelSettingsRuntimeChannelEvent, (event) => { if (event?.type !== 'request-current') return - postModelSettingsRuntimeChannelEvent({ type: 'snapshot', snapshot: modelSettingsRuntimeSnapshot.value }) + postModelSettingsRuntimeEvent({ type: 'snapshot', snapshot: modelSettingsRuntimeSnapshot.value }) }) const settingsAudioDeviceStore = useSettingsAudioDevice() const { stream, enabled } = storeToRefs(settingsAudioDeviceStore) -const { askPermission } = settingsAudioDeviceStore -const { startRecord, stopRecord, onStopRecord } = useAudioRecorder(stream) +const { askPermission, startStream, stopStream } = settingsAudioDeviceStore +const { nowSpeaking } = storeToRefs(useSpeakingStore()) +const hearingStore = useHearingStore() +const { activeTranscriptionModel, activeTranscriptionProvider } = storeToRefs(hearingStore) const hearingPipeline = useHearingSpeechInputPipeline() -const { transcribeForRecording, transcribeForMediaStream, stopStreamingTranscription } = hearingPipeline -const { supportsStreamInput } = storeToRefs(hearingPipeline) +const { transcribeForMediaStream, stopStreamingTranscription } = hearingPipeline +const { error: transcriptionError, supportsStreamInput } = storeToRefs(hearingPipeline) const chatSyncStore = useChatSyncStore() -const shouldUseStreamInput = computed(() => supportsStreamInput.value && !!stream.value) - -const { init: initVAD, dispose: disposeVAD, start: startVAD, loaded: vadLoaded } = useVAD(workletUrl, { - threshold: ref(0.6), - onSpeechStart: () => { - void handleSpeechStart() - }, - onSpeechEnd: () => { - void handleSpeechEnd() +const streamingTranscriptionUnavailable = ref(false) +const shouldUseStreamInput = computed(() => supportsStreamInput.value && !!stream.value && !streamingTranscriptionUnavailable.value) +const voiceTranscriptBuffer = createTranscriptBuffer({ + flushDelayMs: 1200, + maxBufferedTextLength: 90, + async flush(text) { + await sendVoiceInputTextToChat(text) }, }) -let stopOnStopRecord: (() => void) | undefined -const audioInteractionStarting = ref(false) +const assistantSpeechSuppressedUntil = shallowRef(0) +const assistantSpeechResumeTimer = shallowRef>() +let voiceInputGeneration = 0 + +/** Controls transcript cleanup while voice input stops. */ +interface StopAudioInteractionOptions { + /** Flushes pending transcript text to chat before stop completes. */ + flushTranscript?: boolean +} + +const voiceInputInteractionLifecycle = createVoiceInputInteractionLifecycle({ + start: startAudioInteractionConsumers, + stop: stopAudioInteractionConsumers, +}) // Caption overlay broadcast channel type CaptionChannelEvent @@ -262,160 +289,319 @@ type CaptionChannelEvent | { type: 'caption-assistant', text: string } const { post: postCaption } = useBroadcastChannel({ name: 'airi-caption-overlay' }) -function handleStreamingSentenceEnd(delta: string) { - console.info('[Main Page] Received transcription delta:', delta) - const finalText = delta - if (!finalText || !finalText.trim()) { +/** + * Reports a voice input pipeline failure to both the console and visible app UI. + */ +function reportVoiceInputFailure(action: string, error: unknown) { + const reason = errorMessageFrom(error) + const message = reason + ? `Voice input failed to ${action}: ${reason}` + : `Voice input failed to ${action}.` + console.error(`[Main Page] ${message}`, error) + toast.error(message) +} + +/** + * Checks whether current voice input should be ignored to avoid assistant self-transcription. + */ +function isVoiceInputSuppressed(now = Date.now()) { + return shouldSuppressVoiceInput({ + assistantSpeaking: nowSpeaking.value, + suppressedUntil: assistantSpeechSuppressedUntil.value, + }, now) +} + +/** + * Captures whether a queued VAD segment can still leave the app for ASR. + */ +function inspectVoiceInputProviderRequestGate(generation: unknown) { + const current = generation === voiceInputGeneration + const audioEnabled = enabled.value + const suppressed = isVoiceInputSuppressed() + let reason: string | undefined + if (!current) + reason = 'Skipped stale voice input segment' + else if (!audioEnabled) + reason = 'Skipped voice input segment because audio input is disabled' + else if (suppressed) + reason = 'Skipped voice input segment while assistant speech is active or cooling down' + + return { + generation, + activeGeneration: voiceInputGeneration, + current, + enabled: audioEnabled, + suppressed, + reason, + skip: !current || !audioEnabled || suppressed, + } +} + +/** + * Captures whether live microphone audio can still leave the app for streaming ASR. + */ +function inspectVoiceInputStreamingRequestGate() { + const audioEnabled = enabled.value + const suppressed = isVoiceInputSuppressed() + + return { + enabled: audioEnabled, + suppressed, + skip: !audioEnabled || suppressed, + } +} + +/** + * Clears the pending assistant-speech resume timer. + */ +function clearAssistantSpeechResumeTimer() { + if (!assistantSpeechResumeTimer.value) return + + clearTimeout(assistantSpeechResumeTimer.value) + assistantSpeechResumeTimer.value = undefined +} + +/** + * Restarts voice input after assistant playback tail audio should be gone. + */ +function scheduleAssistantSpeechResume() { + clearAssistantSpeechResumeTimer() + + if (!enabled.value) + return + + const remainingCooldownMs = Math.max( + 0, + assistantSpeechSuppressedUntil.value + ? assistantSpeechSuppressedUntil.value - Date.now() + : DEFAULT_ASSISTANT_SPEECH_INPUT_COOLDOWN_MS, + ) + const cooldownMs = nowSpeaking.value + ? DEFAULT_ASSISTANT_SPEECH_INPUT_COOLDOWN_MS + : remainingCooldownMs + + assistantSpeechResumeTimer.value = setTimeout(() => { + assistantSpeechResumeTimer.value = undefined + if (!enabled.value || isVoiceInputSuppressed()) + return + + void voiceInputInteractionLifecycle.start().catch(error => reportVoiceInputFailure('resume listening', error)) + }, cooldownMs) +} + +/** + * Ensures the microphone stream has a live audio track before binding recorder or VAD. + */ +async function ensureLiveAudioInputStream() { + if (!enabled.value) + return false + + if (stream.value?.getAudioTracks().some(track => track.readyState === 'live')) + return true + + stopStream() + + if (!enabled.value) + return false + + await askPermission() + + if (!enabled.value) + return false + + await startStream() + + if (!enabled.value) { + stopStream() + return false } - postCaption({ type: 'caption-speaker', text: finalText }) + if (stream.value?.getAudioTracks().some(track => track.readyState === 'live')) + return true - void (async () => { - try { - console.info('[Main Page] Sending transcription to chat:', finalText) - await chatSyncStore.requestIngest({ text: finalText }) - } - catch (err) { - console.error('[Main Page] Failed to send chat from voice:', err) - } - })() + throw new Error('Microphone stream did not provide a live audio track') } -function handleStreamingSpeechEnd(text: string) { - console.info('[Main Page] Speech ended, final text:', text) - postCaption({ type: 'caption-speaker', text }) +/** + * Sends voice captions as best-effort overlay updates without interrupting chat ingestion. + */ +function postSpeakerCaption(text: string) { + const { error } = tryCatch(() => postCaption({ type: 'caption-speaker', text })) + if (error) + console.warn('[Main Page] Failed to post voice input caption:', error) } -async function handleSpeechStart() { - if (shouldUseStreamInput.value) { - console.info('Speech detected - transcription session should already be active') - return - } - - startRecord() -} - -async function handleSpeechEnd() { - if (shouldUseStreamInput.value) { - // Keep streaming session alive; idle timer in pipeline will handle teardown. - return - } - - stopRecord() -} - -async function startAudioInteraction() { - if (audioInteractionStarting.value) - return - - // NOTICE: `stopOnStopRecord` only tracks whether the non-stream recording hook was registered. - // - // It does NOT guarantee that the current realtime transcription session is still attached to the - // latest `MediaStream`. We previously used it as a generic "already started" guard, which broke - // the hearing-config retoggle path: the mic stream was recreated, VAD restarted on the new stream, - // but `transcribeForMediaStream()` never reattached so speech was detected without any transcript. - // - // Keep the startup guard scoped to "startup in progress" only, and let stream changes restart the - // transcription binding when a new stream arrives. - audioInteractionStarting.value = true +/** + * Sends buffered voice input text to the active chat session. + */ +async function sendVoiceInputTextToChat(text: string) { try { - console.info('[Main Page] Starting audio interaction...') + await chatSyncStore.requestIngest({ text }) + } + catch (err) { + reportVoiceInputFailure('send to chat', err) + } +} - initVAD().then(() => { - if (stream.value) { - console.info('[Main Page] VAD initialized successfully, starting with stream input') - return startVAD(stream.value) - } - }).catch((err) => { - console.warn('[Main Page] VAD initialization failed (non-critical for Web Speech API):', err) +/** Sends completed streaming-ASR sentences to captions and chat. */ +function handleStreamingSentenceEnd(delta: string) { + if (isVoiceInputSuppressed()) + return + + const finalText = delta + if (!finalText || !finalText.trim()) + return + + postSpeakerCaption(finalText) + void sendVoiceInputTextToChat(finalText) +} + +/** Publishes the provider's final streaming-ASR text to the caption overlay. */ +function handleStreamingSpeechEnd(text: string) { + if (isVoiceInputSuppressed()) + return + + postSpeakerCaption(text) +} + +/** Reads the listening generation attached to recorder-backed transcription metadata. */ +function getVoiceInputGeneration(metadata?: Record) { + return typeof metadata?.generation === 'number' ? metadata.generation : undefined +} + +const voiceInputSession = useVoiceInputSession(stream, { + shouldUseStreamInput, + canStartSegment: () => enabled.value && !isVoiceInputSuppressed(), + inspectBeforeTranscription: ({ metadata }) => inspectVoiceInputProviderRequestGate(getVoiceInputGeneration(metadata)), + inspectAfterTranscription: ({ metadata }) => inspectVoiceInputProviderRequestGate(getVoiceInputGeneration(metadata)), + onRecordingReady: () => ({ generation: voiceInputGeneration }), + onTranscriptionResult: ({ text }) => { + postSpeakerCaption(text) + toast(`Voice input transcribed: ${text}`) + voiceTranscriptBuffer.push(text) + }, + onTranscriptionEmpty: () => { + if (transcriptionError.value) { + reportVoiceInputFailure('transcribe speech', transcriptionError.value) + return + } + + toast('Voice input transcribed no text.') + }, + onTranscriptionError: ({ error }) => { + reportVoiceInputFailure('transcribe speech', error) + }, +}) + +/** Starts the active streaming or recorder-backed voice-input consumers. */ +async function startAudioInteractionConsumers() { + if (isVoiceInputSuppressed()) { + scheduleAssistantSpeechResume() + return + } + + if (!await ensureLiveAudioInputStream()) + return + + if (shouldUseStreamInput.value) { + const currentStream = stream.value + if (!currentStream) + throw new Error('Microphone stream is unavailable for streaming transcription') + + const requestGate = inspectVoiceInputStreamingRequestGate() + if (requestGate.skip) + return + + await transcribeForMediaStream(currentStream, { + onSentenceEnd: handleStreamingSentenceEnd, + onSpeechEnd: handleStreamingSpeechEnd, }) - if (shouldUseStreamInput.value) { - console.info('[Main Page] Starting streaming transcription...', { - supportsStreamInput: supportsStreamInput.value, - hasStream: !!stream.value, - }) - - if (!stream.value) { - console.warn('[Main Page] Stream not available despite shouldUseStreamInput being true') - return - } - - // Use sentence deltas for live captions and speech end for final text. - await transcribeForMediaStream(stream.value, { - onSentenceEnd: handleStreamingSentenceEnd, - onSpeechEnd: handleStreamingSpeechEnd, - }) - - console.info('[Main Page] Streaming transcription started successfully') - } - else { - console.warn('[Main Page] Not starting streaming transcription:', { - shouldUseStreamInput: shouldUseStreamInput.value, - hasStream: !!stream.value, - supportsStreamInput: supportsStreamInput.value, - }) + if (inspectVoiceInputStreamingRequestGate().skip) { + await stopStreamingTranscription(true) + return } - // NOTICE: This hook is only for record-then-transcribe providers. - // - // Streaming providers use the active `MediaStream` directly, so this callback must not be treated - // as proof that a realtime session is alive. Future refactors should keep recorder-hook bookkeeping - // separate from stream transcription state, otherwise mic/device re-toggles can leave VAD active - // but transcription detached. - // - // Hook once for non-streaming providers. - if (!stopOnStopRecord) { - stopOnStopRecord = onStopRecord(async (recording) => { - if (shouldUseStreamInput.value) - return - - const text = await transcribeForRecording(recording) - if (!text || !text.trim()) - return - - // Update caption overlay speaker text via BroadcastChannel - postCaption({ type: 'caption-speaker', text }) - - try { - await chatSyncStore.requestIngest({ text }) - } - catch (err) { - console.error('Failed to send chat from voice:', err) - } - }) + if (transcriptionError.value) { + streamingTranscriptionUnavailable.value = true + await stopStreamingTranscription(true) + console.warn('[Main Page] Streaming transcription unavailable; using recorder-backed fallback:', transcriptionError.value) } } - catch (e) { - console.error('Audio interaction init failed:', e) - } - finally { - audioInteractionStarting.value = false - } + + if (!shouldUseStreamInput.value) + await voiceInputSession.startAutoSegmentation() } -function stopAudioInteraction() { - tryCatch(() => { - stopOnStopRecord?.() - stopOnStopRecord = undefined - audioInteractionStarting.value = false - void stopStreamingTranscription(true) - disposeVAD() - }) +/** + * Stops active microphone consumers before the stage binds to another audio stream. + */ +async function stopAudioInteractionConsumers(options: StopAudioInteractionOptions = {}) { + const flushTranscript = options.flushTranscript ?? true + + clearAssistantSpeechResumeTimer() + voiceInputGeneration += 1 + + await Promise.all([ + stopStreamingTranscription(true), + voiceInputSession.stop({ flushActiveRecording: false }), + ]) + + if (flushTranscript) + await voiceTranscriptBuffer.dispose() + else + voiceTranscriptBuffer.clear() } watch(enabled, async (val) => { - console.info('[Main Page] Audio enabled changed:', val, 'stream available:', !!stream.value) - if (val) { - await askPermission() - await startAudioInteraction() + try { + if (val) { + await askPermission() + await voiceInputInteractionLifecycle.start() + } + else { + await voiceInputInteractionLifecycle.stop() + } } - else { - stopAudioInteraction() + catch (error) { + reportVoiceInputFailure(val ? 'start listening' : 'stop listening', error) + if (val) + enabled.value = false } }, { immediate: true }) +watch([activeTranscriptionProvider, activeTranscriptionModel, supportsStreamInput], async () => { + streamingTranscriptionUnavailable.value = false + if (!enabled.value) + return + + try { + await voiceInputInteractionLifecycle.stop({ flushTranscript: false }) + await voiceInputInteractionLifecycle.start() + } + catch (error) { + reportVoiceInputFailure('restart after transcription settings changed', error) + enabled.value = false + } +}) + +watch(nowSpeaking, async (speaking) => { + if (speaking) { + clearAssistantSpeechResumeTimer() + try { + await voiceInputInteractionLifecycle.stop({ flushTranscript: false }) + } + catch (error) { + reportVoiceInputFailure('pause while assistant is speaking', error) + } + return + } + + assistantSpeechSuppressedUntil.value = assistantSpeechCooldownDeadline() + scheduleAssistantSpeechResume() +}) + onMounted(() => { if (onboardingStore.needsOnboarding) { openOnboarding() @@ -423,33 +609,29 @@ onMounted(() => { }) onUnmounted(() => { - postModelSettingsRuntimeChannelEvent({ + postModelSettingsRuntimeEvent({ type: 'owner-gone', ownerInstanceId: modelSettingsRuntimeOwnerInstanceId, }) - stopAudioInteraction() + clearAssistantSpeechResumeTimer() + void voiceInputInteractionLifecycle.stop().catch(error => reportVoiceInputFailure('stop listening', error)) }) watch(stream, async (currentStream) => { - if (!enabled.value || !currentStream || audioInteractionStarting.value) + if (!enabled.value || !currentStream || voiceInputInteractionLifecycle.isStarting() || voiceInputInteractionLifecycle.isStopping() || isVoiceInputSuppressed()) return // NOTICE: The controls-island mic toggle and device changes can replace the underlying MediaStream // without reloading the page. When that happens, VAD may successfully restart against the new stream, // but any existing transcription transport is still bound to the old one. Always allow the page to - // re-run `startAudioInteraction()` for a newly available stream unless startup is already underway. - console.info('[Main Page] Stream became available, ensuring audio interaction is started') - await startAudioInteraction() -}) - -watch([stream, () => vadLoaded.value], async ([s, loaded]) => { - if (enabled.value && loaded && s) { - try { - await startVAD(s) - } - catch (e) { - console.error('Failed to start VAD with stream:', e) - } + // restart voice input for a newly available stream unless another lifecycle operation is underway. + try { + await voiceInputInteractionLifecycle.stop() + await voiceInputInteractionLifecycle.start() + } + catch (error) { + reportVoiceInputFailure('restart after microphone changed', error) + enabled.value = false } }) @@ -497,9 +679,7 @@ const cursorPosition = computed(() => ({ :paused="stagePaused" /> - + @@ -544,7 +724,14 @@ const cursorPosition = computed(() => ({ bg="white/80 dark:neutral-950/80" backdrop-blur="md" >
-
+
DRAG HERE TO MOVE
diff --git a/apps/stage-tamagotchi/src/renderer/utils/voice-input-lifecycle.test.ts b/apps/stage-tamagotchi/src/renderer/utils/voice-input-lifecycle.test.ts new file mode 100644 index 000000000..1e96deda6 --- /dev/null +++ b/apps/stage-tamagotchi/src/renderer/utils/voice-input-lifecycle.test.ts @@ -0,0 +1,85 @@ +import { describe, expect, it } from 'vitest' + +import { createVoiceInputInteractionLifecycle } from './voice-input-lifecycle' + +/** + * @example + * Voice-input lifecycle calls are serialized across overlapping UI toggles. + */ +describe('createVoiceInputInteractionLifecycle', () => { + // https://github.com/moeru-ai/airi/pull/2004#discussion_r3480058474 + // ROOT CAUSE: + // + // If a start begins while an earlier stop is flushing buffered text, the older stop can + // tear down the newly-started microphone consumers after the toggle is already enabled. + // + // Before: start and stop operations ran concurrently. + // After: a start waits for the active stop operation before binding microphone consumers. + /** + * @example + * A start requested during stop waits until stop releases its pending work. + */ + it('serializes a start requested while stop is still in progress', async () => { + const calls: string[] = [] + let finishStop!: () => void + const lifecycle = createVoiceInputInteractionLifecycle({ + async start() { + calls.push('start') + }, + async stop() { + calls.push('stop') + await new Promise((resolve) => { + finishStop = resolve + }) + }, + }) + + const stopping = lifecycle.stop() + await Promise.resolve() + const starting = lifecycle.start() + await Promise.resolve() + + /** + * @example + * The pending start has not entered the start operation yet. + */ + expect(calls).toEqual(['stop']) + + finishStop() + await Promise.all([stopping, starting]) + + /** + * @example + * The start operation runs only after stop completes. + */ + expect(calls).toEqual(['stop', 'start']) + }) + + // https://github.com/moeru-ai/airi/pull/2004#discussion_r3480058478 + // ROOT CAUSE: + // + // If microphone startup throws, swallowing the error prevents the enabled watcher from + // rolling the persisted toggle back to disabled. + // + // Before: startup failures were reported and then returned as a successful promise. + // After: the lifecycle preserves the rejection for the watcher to handle. + /** + * @example + * A rejected microphone start remains rejected for the enabled watcher. + */ + it('propagates startup failures to its caller', async () => { + const error = new DOMException('Selected microphone is unavailable', 'NotFoundError') + const lifecycle = createVoiceInputInteractionLifecycle({ + async start() { + throw error + }, + async stop() {}, + }) + + /** + * @example + * Callers can catch the original startup error and roll back UI state. + */ + await expect(lifecycle.start()).rejects.toBe(error) + }) +}) diff --git a/apps/stage-tamagotchi/src/renderer/utils/voice-input-lifecycle.ts b/apps/stage-tamagotchi/src/renderer/utils/voice-input-lifecycle.ts new file mode 100644 index 000000000..ddd757cb0 --- /dev/null +++ b/apps/stage-tamagotchi/src/renderer/utils/voice-input-lifecycle.ts @@ -0,0 +1,85 @@ +/** + * Operations that bind and release the main-stage voice-input consumers. + * + * @param TStopOptions Configuration accepted by the stop operation. + */ +export interface VoiceInputInteractionOperations { + /** Starts the streaming or recorder-backed microphone consumers. */ + start: () => Promise + /** Stops active microphone consumers and applies the requested flush policy. */ + stop: (options?: TStopOptions) => Promise +} + +/** + * Serialized lifecycle for main-stage voice input. + * + * Use when: + * - Mic toggles can overlap asynchronous start and stop operations. + * - A restart must wait until the previous listener has fully stopped. + * + * Expects: + * - Operations own the actual microphone, transcription, and transcript-buffer work. + * - Stop remains safe after a failed start. + * + * Returns: + * - Start and stop actions that preserve operation errors and prevent overlapping lifecycles. + */ +export function createVoiceInputInteractionLifecycle( + operations: VoiceInputInteractionOperations, +) { + let startPromise: Promise | undefined + let stopPromise: Promise | undefined + + /** Starts after any active stop and deduplicates concurrent starts. */ + async function start() { + if (stopPromise) + await stopPromise + + if (startPromise) + return startPromise + + const operation = Promise.resolve().then(operations.start) + startPromise = operation + try { + await operation + } + finally { + if (startPromise === operation) + startPromise = undefined + } + } + + /** Stops after any active start and deduplicates concurrent stops. */ + async function stop(options?: TStopOptions) { + if (stopPromise) + return stopPromise + + const operation = (async () => { + if (startPromise) { + try { + await startPromise + } + catch { + // A failed start still needs its partially-created microphone consumers released. + } + } + + await operations.stop(options) + })() + stopPromise = operation + try { + await operation + } + finally { + if (stopPromise === operation) + stopPromise = undefined + } + } + + return { + start, + stop, + isStarting: () => startPromise !== undefined, + isStopping: () => stopPromise !== undefined, + } +} diff --git a/apps/stage-tamagotchi/src/renderer/utils/voice-input-suppression.test.ts b/apps/stage-tamagotchi/src/renderer/utils/voice-input-suppression.test.ts new file mode 100644 index 000000000..605974b65 --- /dev/null +++ b/apps/stage-tamagotchi/src/renderer/utils/voice-input-suppression.test.ts @@ -0,0 +1,44 @@ +import { describe, expect, it } from 'vitest' + +import { + assistantSpeechCooldownDeadline, + DEFAULT_ASSISTANT_SPEECH_INPUT_COOLDOWN_MS, + shouldSuppressVoiceInput, +} from './voice-input-suppression' + +describe('shouldSuppressVoiceInput', () => { + it('suppresses voice input while assistant speech is active', () => { + const result = shouldSuppressVoiceInput({ + assistantSpeaking: true, + suppressedUntil: 0, + }, 1000) + + expect(result).toBe(true) + }) + + it('suppresses voice input during the assistant speech cooldown', () => { + const result = shouldSuppressVoiceInput({ + assistantSpeaking: false, + suppressedUntil: 1800, + }, 1200) + + expect(result).toBe(true) + }) + + it('allows voice input after assistant speech cooldown ends', () => { + const result = shouldSuppressVoiceInput({ + assistantSpeaking: false, + suppressedUntil: 1800, + }, 1800) + + expect(result).toBe(false) + }) +}) + +describe('assistantSpeechCooldownDeadline', () => { + it('returns the default cooldown deadline after assistant speech ends', () => { + const result = assistantSpeechCooldownDeadline(1000) + + expect(result).toBe(1000 + DEFAULT_ASSISTANT_SPEECH_INPUT_COOLDOWN_MS) + }) +}) diff --git a/apps/stage-tamagotchi/src/renderer/utils/voice-input-suppression.ts b/apps/stage-tamagotchi/src/renderer/utils/voice-input-suppression.ts new file mode 100644 index 000000000..d2f1e5595 --- /dev/null +++ b/apps/stage-tamagotchi/src/renderer/utils/voice-input-suppression.ts @@ -0,0 +1,42 @@ +export const DEFAULT_ASSISTANT_SPEECH_INPUT_COOLDOWN_MS = 800 + +export interface VoiceInputSuppressionOptions { + assistantSpeaking: boolean + suppressedUntil: number +} + +/** + * Decides whether voice input should be ignored while assistant audio can leak into the microphone. + * + * Use when: + * - The assistant is actively playing TTS. + * - The assistant just stopped speaking and speaker echo may still be captured. + * + * Expects: + * - `suppressedUntil` is a timestamp in milliseconds. + * + * Returns: + * - `true` when capture, transcription, and ingestion should be skipped. + */ +export function shouldSuppressVoiceInput(options: VoiceInputSuppressionOptions, now = Date.now()) { + return options.assistantSpeaking || now < options.suppressedUntil +} + +/** + * Calculates the timestamp until which voice input should stay muted after assistant speech. + * + * Use when: + * - Assistant playback has ended and the microphone may still receive speaker tail audio. + * + * Expects: + * - `endedAt` is the playback end timestamp in milliseconds. + * + * Returns: + * - A timestamp in milliseconds after the configured cooldown. + */ +export function assistantSpeechCooldownDeadline( + endedAt = Date.now(), + cooldownMs = DEFAULT_ASSISTANT_SPEECH_INPUT_COOLDOWN_MS, +) { + return endedAt + cooldownMs +} diff --git a/packages/pipelines-audio/README.md b/packages/pipelines-audio/README.md new file mode 100644 index 000000000..21023afae --- /dev/null +++ b/packages/pipelines-audio/README.md @@ -0,0 +1,30 @@ +# @proj-airi/pipelines-audio + +Shared audio-pipeline orchestration for AIRI. The package owns reusable streaming, playback, text-chunking, and transcript-buffering policies without depending on an application UI. + +## Use it for + +- Building and scheduling speech playback pipelines. +- Parsing streaming-control events. +- Grouping nearby ASR fragments with `createTranscriptBuffer` before a product sends one spoken turn downstream. + +## Do not use it for + +- Vue or Electron lifecycle state. +- Provider credentials and product-specific error UI. +- Raw audio encoding utilities, which belong in `@proj-airi/audio`. + +## Transcript buffering + +```ts +import { createTranscriptBuffer } from '@proj-airi/pipelines-audio' + +const buffer = createTranscriptBuffer({ + flushDelayMs: 1200, + flush: async text => sendToChat(text), +}) + +buffer.push('hello') +buffer.push('world') +await buffer.dispose() +``` diff --git a/packages/pipelines-audio/src/index.ts b/packages/pipelines-audio/src/index.ts index fbb9ed6f8..aba156969 100644 --- a/packages/pipelines-audio/src/index.ts +++ b/packages/pipelines-audio/src/index.ts @@ -6,4 +6,5 @@ export * from './processors/tts-chunker' export * from './speech-pipeline' export * from './stream' export * from './timeline' +export * from './transcript-buffer' export * from './types' diff --git a/packages/pipelines-audio/src/transcript-buffer.test.ts b/packages/pipelines-audio/src/transcript-buffer.test.ts new file mode 100644 index 000000000..6182ae126 --- /dev/null +++ b/packages/pipelines-audio/src/transcript-buffer.test.ts @@ -0,0 +1,169 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +import { createTranscriptBuffer } from './transcript-buffer' + +/** + * @example + * Recorder-backed ASR fragments are grouped into one spoken turn. + */ +describe('createTranscriptBuffer', () => { + beforeEach(() => { + vi.useFakeTimers() + }) + + afterEach(() => { + vi.useRealTimers() + }) + + /** + * @example + * Adjacent CJK fragments are joined without an artificial space. + */ + it('merges adjacent transcription fragments before flushing', async () => { + const flushed: string[] = [] + const buffer = createTranscriptBuffer({ + flushDelayMs: 1200, + async flush(text) { + flushed.push(text) + }, + }) + + buffer.push('我今天想测试一下') + vi.advanceTimersByTime(800) + buffer.push('长句识别会不会好一点') + await vi.advanceTimersByTimeAsync(1200) + + /** + * @example + * The two CJK fragments form one chat turn. + */ + expect(flushed).toEqual(['我今天想测试一下长句识别会不会好一点']) + }) + + /** + * @example + * Adjacent Latin fragments keep a readable word separator. + */ + it('keeps a separator between Latin fragments', async () => { + const flushed: string[] = [] + const buffer = createTranscriptBuffer({ + flushDelayMs: 1200, + async flush(text) { + flushed.push(text) + }, + }) + + buffer.push('hello') + buffer.push('world') + await vi.advanceTimersByTimeAsync(1200) + + /** + * @example + * Latin words remain separated after grouping. + */ + expect(flushed).toEqual(['hello world']) + }) + + /** + * @example + * Long transcripts flush without waiting for the pause timer. + */ + it('flushes immediately at the configured text-length limit', async () => { + const flushed: string[] = [] + const buffer = createTranscriptBuffer({ + flushDelayMs: 1200, + maxBufferedTextLength: 6, + async flush(text) { + flushed.push(text) + }, + }) + + buffer.push('这是一段很长的文字') + await Promise.resolve() + + /** + * @example + * The long fragment bypasses the delayed timer. + */ + expect(flushed).toEqual(['这是一段很长的文字']) + }) + + /** + * @example + * Dispose flushes pending text before the microphone session stops. + */ + it('flushes pending text when disposed', async () => { + const flushed: string[] = [] + const buffer = createTranscriptBuffer({ + flushDelayMs: 1200, + async flush(text) { + flushed.push(text) + }, + }) + + buffer.push('关闭之前还有一句话') + await buffer.dispose() + + /** + * @example + * Pending text is delivered exactly once during disposal. + */ + expect(flushed).toEqual(['关闭之前还有一句话']) + }) + + /** + * @example + * Clearing a paused voice session discards pending text. + */ + it('clears pending text without flushing it', async () => { + const flushed: string[] = [] + const buffer = createTranscriptBuffer({ + flushDelayMs: 1200, + async flush(text) { + flushed.push(text) + }, + }) + + buffer.push('这句话不应该发送') + buffer.clear() + await vi.advanceTimersByTimeAsync(1200) + + /** + * @example + * No transcript is delivered after clear. + */ + expect(flushed).toEqual([]) + }) + + /** + * @example + * A failed network send does not prevent a later transcript from flushing. + */ + it('continues flushing after a previous flush rejects', async () => { + const flush = vi.fn<(text: string) => Promise>() + .mockRejectedValueOnce(new Error('temporary send failure')) + .mockResolvedValue(undefined) + const buffer = createTranscriptBuffer({ + flushDelayMs: 1200, + flush, + }) + + buffer.push('first transcript') + + // ROOT CAUSE: + // + // The rejected first flush was stored as the shared serialization chain. + // Every later flush chained from that rejection and skipped the callback entirely. + // We fixed this by retaining the caller-facing rejection while recovering the internal chain. + /** @example The failed item still reports its delivery error to the caller. */ + await expect(buffer.flushNow()).rejects.toThrow('temporary send failure') + + buffer.push('second transcript') + await buffer.flushNow() + + /** @example Both items reach the callback even though the first delivery failed. */ + expect(flush).toHaveBeenNthCalledWith(1, 'first transcript') + /** @example The recovered chain delivers the next buffered transcript. */ + expect(flush).toHaveBeenNthCalledWith(2, 'second transcript') + }) +}) diff --git a/packages/pipelines-audio/src/transcript-buffer.ts b/packages/pipelines-audio/src/transcript-buffer.ts new file mode 100644 index 000000000..2db2dc5bf --- /dev/null +++ b/packages/pipelines-audio/src/transcript-buffer.ts @@ -0,0 +1,125 @@ +/** Options for grouping nearby ASR fragments into one transcript. */ +export interface TranscriptBufferOptions { + /** Delay after the latest fragment before the buffer flushes. */ + flushDelayMs: number + /** + * Maximum buffered text length before an immediate flush. + * + * @default 80 + */ + maxBufferedTextLength?: number + /** Receives serialized, normalized transcript text. */ + flush: (text: string) => Promise | void +} + +const DEFAULT_MAX_BUFFERED_TEXT_LENGTH = 80 +const CJK_BOUNDARY_RE = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]$/u +const CJK_START_RE = /^[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]/u + +/** + * Normalizes the boundary between two ASR transcript fragments. + * + * Before: + * - `"你好"`, `"世界"` + * - `"hello"`, `"world"` + * + * After: + * - `"你好世界"` + * - `"hello world"` + */ +function joinTranscriptFragments(previous: string, next: string) { + if (!previous) + return next + + if (CJK_BOUNDARY_RE.test(previous) && CJK_START_RE.test(next)) + return `${previous}${next}` + + return `${previous} ${next}` +} + +/** + * Groups nearby ASR fragments into serialized transcript flushes. + * + * Use when: + * - Record-then-transcribe providers emit one result per VAD segment. + * - Natural pauses should remain part of one spoken turn. + * + * Expects: + * - `flushDelayMs` covers the pause window that should remain in one turn. + * - The flush callback may be asynchronous and must run in transcript order. + * + * Returns: + * - Actions to push fragments, flush or discard pending text, and dispose the buffer. + */ +export function createTranscriptBuffer(options: TranscriptBufferOptions) { + let pendingText = '' + let flushTimer: ReturnType | undefined + let flushChain = Promise.resolve() + const maxBufferedTextLength = options.maxBufferedTextLength ?? DEFAULT_MAX_BUFFERED_TEXT_LENGTH + + /** Clears the pending delayed flush timer. */ + function clearFlushTimer() { + if (!flushTimer) + return + + clearTimeout(flushTimer) + flushTimer = undefined + } + + /** Sends the current transcript through the serialized flush chain. */ + function flushNow() { + clearFlushTimer() + + const text = pendingText.trim() + pendingText = '' + if (!text) + return flushChain + + const delivery = flushChain.then(() => options.flush(text)) + + // Keep the failed delivery observable to its caller without poisoning later queued flushes. + flushChain = delivery.catch(() => {}) + return delivery + } + + /** Schedules a delayed flush after the configured pause window. */ + function scheduleFlush() { + clearFlushTimer() + flushTimer = setTimeout(() => { + void flushNow() + }, options.flushDelayMs) + } + + /** Adds one normalized ASR fragment to the pending spoken turn. */ + function push(text: string) { + const trimmed = text.trim() + if (!trimmed) + return + + pendingText = joinTranscriptFragments(pendingText, trimmed) + if (pendingText.length >= maxBufferedTextLength) { + void flushNow() + return + } + + scheduleFlush() + } + + /** Discards pending text without invoking the flush callback. */ + function clear() { + clearFlushTimer() + pendingText = '' + } + + /** Flushes pending text and prevents its delayed timer from firing later. */ + async function dispose() { + await flushNow() + } + + return { + push, + flushNow, + clear, + dispose, + } +} diff --git a/packages/stage-ui/src/stores/settings/audio-device.test.ts b/packages/stage-ui/src/stores/settings/audio-device.test.ts index fe11ef4cd..a3e254adb 100644 --- a/packages/stage-ui/src/stores/settings/audio-device.test.ts +++ b/packages/stage-ui/src/stores/settings/audio-device.test.ts @@ -68,7 +68,7 @@ describe('store settings-audio-devices', () => { storageMock.values.clear() audioDeviceMock.audioInputs.value = [] audioDeviceMock.selectedAudioInput.value = '' - vi.clearAllMocks() + vi.resetAllMocks() }) afterEach(() => { @@ -100,19 +100,15 @@ describe('store settings-audio-devices', () => { expect(storageMock.values.get('settings/audio/input')).toBe('microphone-1') }) - it('ignores stale microphone startup failures after a newer start succeeds', async () => { + /** @example A rapid off/on toggle keeps using the pending browser microphone request. */ + it('reuses a pending microphone start after disable and re-enable', async () => { const { useSettingsAudioDevice } = await import('./audio-device') const store = useSettingsAudioDevice() - let rejectFirstStart!: (error: unknown) => void - let resolveSecondStart!: () => void - audioDeviceMock.startStream - .mockImplementationOnce(() => new Promise((_resolve, reject) => { - rejectFirstStart = reject - })) - .mockImplementationOnce(() => new Promise((resolve) => { - resolveSecondStart = resolve - })) + let resolveStart!: () => void + audioDeviceMock.startStream.mockImplementation(() => new Promise((resolve) => { + resolveStart = resolve + })) store.enabled = true await nextTick() @@ -123,14 +119,49 @@ describe('store settings-audio-devices', () => { store.enabled = true await nextTick() - resolveSecondStart() - await Promise.resolve() + /** @example Rapid toggles do not allocate another stream while the first request is pending. */ + expect(audioDeviceMock.startStream).toHaveBeenCalledTimes(1) - rejectFirstStart(new Error('old startup failed')) + resolveStart() await Promise.resolve() await nextTick() + /** @example The re-enabled microphone remains active after the shared request succeeds. */ expect(store.enabled).toBe(true) + /** @example Disabling still stops the previously requested stream once. */ expect(audioDeviceMock.stopStream).toHaveBeenCalledTimes(1) }) + + /** + * @example + * Concurrent page and store consumers share one pending browser microphone request. + */ + // https://github.com/moeru-ai/airi/pull/2004#discussion_r3560276717 + it('reuses an in-flight microphone start for PR #2004', async () => { + const { useSettingsAudioDevice } = await import('./audio-device') + const store = useSettingsAudioDevice() + + let resolveStart!: () => void + const pendingStart = new Promise((resolve) => { + resolveStart = resolve + }) + audioDeviceMock.startStream.mockReturnValue(pendingStart) + + const firstStart = store.startStream() + const secondStart = store.startStream() + + // ROOT CAUSE: + // + // The store previously forwarded every caller to VueUse while stream.value was still empty. + // VueUse only guards completed streams, so concurrent calls created separate getUserMedia requests. + // We fixed this by sharing the pending store-owned startup promise until it settles. + /** @example Only one getUserMedia-backed operation starts while it remains pending. */ + expect(audioDeviceMock.startStream).toHaveBeenCalledTimes(1) + + resolveStart() + await Promise.all([firstStart, secondStart]) + + /** @example Both callers complete through the same underlying startup. */ + expect(audioDeviceMock.startStream).toHaveBeenCalledTimes(1) + }) }) diff --git a/packages/stage-ui/src/stores/settings/audio-device.ts b/packages/stage-ui/src/stores/settings/audio-device.ts index 0d7e0ffac..8e945d2e4 100644 --- a/packages/stage-ui/src/stores/settings/audio-device.ts +++ b/packages/stage-ui/src/stores/settings/audio-device.ts @@ -20,6 +20,7 @@ export const useSettingsAudioDevice = defineStore('settings-audio-devices', () = const selectedAudioInputPersist = useLocalStorageManualReset('settings/audio/input', selectedAudioInputNonPersist.value) const audioInputEnabled = useLocalStorageManualReset('settings/audio/input/enabled', false) let audioInputStartGeneration = 0 + let audioInputStart: ReturnType | undefined function syncSelectedAudioInputFromRuntime() { if (selectedAudioInputPersist.value !== selectedAudioInputNonPersist.value) @@ -46,9 +47,24 @@ export const useSettingsAudioDevice = defineStore('settings-audio-devices', () = audioInputStartGeneration += 1 } + /** Reuses the active browser request so concurrent consumers cannot allocate duplicate streams. */ + function getOrStartAudioInputStream() { + if (audioInputStart) + return audioInputStart + + const currentStart = startAudioInputStream() + audioInputStart = currentStart + const clearCurrentStart = () => { + if (audioInputStart === currentStart) + audioInputStart = undefined + } + void currentStart.then(clearCurrentStart, clearCurrentStart) + return currentStart + } + async function startStreamForGeneration(generation: number) { syncSelectedAudioInputToRuntime() - await startAudioInputStream() + await getOrStartAudioInputStream() if (generation === audioInputStartGeneration) syncSelectedAudioInputFromRuntime() diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0a8564322..84a502339 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -2026,6 +2026,9 @@ importers: '@proj-airi/i18n': specifier: workspace:^ version: link:../../packages/i18n + '@proj-airi/pipelines-audio': + specifier: workspace:^ + version: link:../../packages/pipelines-audio '@proj-airi/plugin-sdk-tamagotchi': specifier: workspace:^ version: link:../../packages/plugin-sdk-tamagotchi