diff --git a/packages/stage-pages/src/pages/settings/modules/hearing.vue b/packages/stage-pages/src/pages/settings/modules/hearing.vue index f16706af6..b63b57432 100644 --- a/packages/stage-pages/src/pages/settings/modules/hearing.vue +++ b/packages/stage-pages/src/pages/settings/modules/hearing.vue @@ -3,7 +3,7 @@ import workletUrl from '@proj-airi/stage-ui/workers/vad/process.worklet?worker&u import { errorMessageFromValue } from '@proj-airi/stage-shared' import { Alert, ErrorContainer, LevelMeter, RadioCardManySelect, RadioCardSimple, TestDummyMarker, ThresholdMeter, TimeSeriesChart } from '@proj-airi/stage-ui/components' -import { useAnalytics, useAudioAnalyzer, useAudioRecorder } from '@proj-airi/stage-ui/composables' +import { useAnalytics, useAudioAnalyzer, useAudioRecorder, useVoiceInputSession } from '@proj-airi/stage-ui/composables' import { useVAD } from '@proj-airi/stage-ui/stores/ai/models/vad' import { useAudioContext } from '@proj-airi/stage-ui/stores/audio' import { CONFIDENCE_THRESHOLD_DISABLED, useHearingSpeechInputPipeline, useHearingStore } from '@proj-airi/stage-ui/stores/modules/hearing' @@ -81,12 +81,67 @@ const useVADThreshold = ref(0.6) // 0.1 - 0.9 const useVADMinSilenceDurationMs = ref(800) const useVADModel = ref(true) // Toggle between VAD and volume-based detection const shouldUseStreamInput = computed(() => supportsStreamInput.value && !!stream.value) +let sttTestStopTimer: ReturnType | undefined + +const sttTestVoiceInputSession = useVoiceInputSession(stream, { + shouldUseStreamInput, + // Manual Settings tests own their 3s recording window; automatic volume segmentation + // would race that timer and make provider diagnostics harder to interpret. + volumeFallback: { + enabled: false, + }, + onSegmentStart: () => { + testStatusMessage.value = 'Recording audio for transcription... (3 seconds)' + }, + onTranscriptionStart: () => { + testStatusMessage.value = 'Transcribing recording...' + isTranscribing.value = true + }, + onTranscriptionResult: ({ text }) => { + testTranscriptionText.value = text + testStatusMessage.value = 'Transcription complete!' + isTranscribing.value = false + isTestingSTT.value = false + console.info('STT test transcription result:', text) + }, + onTranscriptionEmpty: () => { + testTranscriptionError.value = transcriptionPipelineError.value || 'No transcription result returned from provider' + testStatusMessage.value = 'Transcription failed' + isTranscribing.value = false + isTestingSTT.value = false + }, + onRecordingSkipped: ({ gate }) => { + testTranscriptionError.value = gate?.reason || transcriptionPipelineError.value || 'No recording captured from microphone' + testStatusMessage.value = 'Transcription failed' + isTranscribing.value = false + isTestingSTT.value = false + }, + onTranscriptionError: ({ error }) => { + testTranscriptionError.value = errorMessageFromValue(error) + testStatusMessage.value = `Error: ${testTranscriptionError.value}` + isTranscribing.value = false + isTestingSTT.value = false + console.error('STT test transcription error:', error) + }, +}) + +async function resetSttTestVoiceInputSession() { + if (sttTestStopTimer) { + clearTimeout(sttTestStopTimer) + sttTestStopTimer = undefined + } + + await sttTestVoiceInputSession.stop({ flushActiveRecording: false }) +} function formatVADThreshold(value: number) { return value.toFixed(2) } async function handleSpeechStart() { + if (isTestingSTT.value) + return + if (shouldUseStreamInput.value && stream.value) { // Use both callbacks to support incremental updates and final transcript replacement. // ChatArea uses only onSentenceEnd to avoid re-adding deleted text. @@ -105,6 +160,9 @@ async function handleSpeechStart() { } async function handleSpeechEnd() { + if (isTestingSTT.value) + return + if (shouldUseStreamInput.value) { // For streaming providers, keep the session alive; idle timer will handle teardown. return @@ -264,38 +322,12 @@ onStopRecord(async (recording) => { if (shouldUseStreamInput.value) return + if (isTestingSTT.value) + return + if (!recording || recording.size === 0) return - // Handle STT test transcription directly here - if (isTestingSTT.value) { - testStatusMessage.value = 'Transcribing recording...' - isTranscribing.value = true - - try { - const result = await transcribeForRecording(recording) - if (result) { - testTranscriptionText.value = result - testStatusMessage.value = 'Transcription complete!' - console.info('STT test transcription result:', result) - } - else { - testTranscriptionError.value = transcriptionPipelineError.value || 'No transcription result returned from provider' - testStatusMessage.value = 'Transcription failed' - } - } - catch (err) { - testTranscriptionError.value = errorMessageFromValue(err) - testStatusMessage.value = `Error: ${testTranscriptionError.value}` - console.error('STT test transcription error:', err) - } - finally { - isTranscribing.value = false - isTestingSTT.value = false - } - return - } - // Normal monitoring mode - add to audios and transcribe audios.value.push(recording) @@ -393,12 +425,29 @@ async function startSTTTest() { testStatusMessage.value = 'Recording audio for transcription... (3 seconds)' console.info('Starting STT test with recording-based transcription for provider:', activeTranscriptionProvider.value) - startRecord() + const recordingStarted = await sttTestVoiceInputSession.startSegment('manual') + if (!recordingStarted) { + if (!testTranscriptionError.value) + testStatusMessage.value = 'Recording did not start' + isTranscribing.value = false + isTestingSTT.value = false + return + } // Wait a bit for recording to start, then stop it after a delay - setTimeout(async () => { - stopRecord() + sttTestStopTimer = setTimeout(async () => { + sttTestStopTimer = undefined testStatusMessage.value = 'Processing transcription...' + try { + await sttTestVoiceInputSession.stopSegment('manual') + } + catch (err) { + testTranscriptionError.value = errorMessageFromValue(err) + testStatusMessage.value = `Error: ${testTranscriptionError.value}` + isTranscribing.value = false + isTestingSTT.value = false + console.error('STT test stop timer error:', err) + } }, 3000) // Record for 3 seconds } } @@ -422,7 +471,7 @@ async function stopSTTTest() { await stopStreamingTranscription(false, activeTranscriptionProvider.value) } else { - stopRecord() + await resetSttTestVoiceInputSession() } } catch (err) { @@ -446,9 +495,6 @@ async function stopSTTTest() { } } -// Note: STT test transcription is now handled directly in onStopRecord handler above -// This watch is kept for potential future use but is no longer needed for STT tests - watch(selectedAudioInput, async () => isMonitoring.value && await setupAudioMonitoring()) function handleStreamStartError() { diff --git a/packages/stage-ui/src/components/scenes/Stage.vue b/packages/stage-ui/src/components/scenes/Stage.vue index 9eab85946..301da1420 100644 --- a/packages/stage-ui/src/components/scenes/Stage.vue +++ b/packages/stage-ui/src/components/scenes/Stage.vue @@ -36,6 +36,7 @@ import { useSpeechPipelineAnalytics } from '../../composables/use-speech-pipelin import { Emotion, EMOTION_EmotionMotionName_value, EMOTION_VRMExpressionName_value, EmotionThinkMotionName } from '../../constants/emotions' import { getDefaultStreamingModel, getDefinedProvider } from '../../libs/providers/providers' import { OFFICIAL_SPEECH_PROVIDER_ID } from '../../libs/providers/providers/official' +import { bindSpeakingStateToPlaybackManager } from '../../libs/speech/playback-speaking-state' import { createStageTtsSession } from '../../libs/speech/tts-session' import { useAudioContext, useSpeakingStore } from '../../stores/audio' import { useBackgroundStore } from '../../stores/background' @@ -519,29 +520,36 @@ speechPipeline.on('onTurnCancel', ({ turnId }) => { streamingControl.cancelTurn(turnId) }) -playbackManager.onEnd(() => { +function resetSpeakingState() { nowSpeaking.value = false mouthOpenSize.value = 0 -}) +} -playbackManager.onStart(({ item }) => { - nowSpeaking.value = true - // NOTICE: postCaption and postPresent may throw errors if the BroadcastChannel is closed - // (e.g., when navigating away from the page). We wrap these in try-catch to prevent - // breaking playback when the channel is unavailable. - assistantCaption.value += ` ${item.text}` - try { - postCaption({ type: 'caption-assistant', text: item.text }) - } - catch { - // BroadcastChannel may be closed - don't break playback - } - try { - postPresent({ type: 'assistant-append', text: item.text }) - } - catch { - // BroadcastChannel may be closed - don't break playback - } +bindSpeakingStateToPlaybackManager(playbackManager, { + setSpeaking: (speaking) => { + if (!speaking) + resetSpeakingState() + else + nowSpeaking.value = true + }, + onStart: ({ item }) => { + // NOTICE: postCaption and postPresent may throw errors if the BroadcastChannel is closed + // (e.g., when navigating away from the page). We wrap these in try-catch to prevent + // breaking playback when the channel is unavailable. + assistantCaption.value += ` ${item.text}` + try { + postCaption({ type: 'caption-assistant', text: item.text }) + } + catch { + // BroadcastChannel may be closed - don't break playback + } + try { + postPresent({ type: 'assistant-append', text: item.text }) + } + catch { + // BroadcastChannel may be closed - don't break playback + } + }, }) function startLipSyncLoop() { diff --git a/packages/stage-ui/src/composables/audio/audio-device.test.ts b/packages/stage-ui/src/composables/audio/audio-device.test.ts new file mode 100644 index 000000000..c9948a756 --- /dev/null +++ b/packages/stage-ui/src/composables/audio/audio-device.test.ts @@ -0,0 +1,110 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' + +const vueUseMock = vi.hoisted(() => ({ + audioInputs: undefined as unknown as { value: MediaDeviceInfo[] }, + ensurePermissions: vi.fn(async () => {}), + startUserMediaStream: vi.fn(), + stopStream: vi.fn(), +})) + +vi.mock('@vueuse/core', async () => { + const vue = await vi.importActual('vue') + vueUseMock.audioInputs = vue.ref([]) + + return { + useDevicesList: () => ({ + audioInputs: vueUseMock.audioInputs, + permissionGranted: vue.ref(false), + ensurePermissions: vueUseMock.ensurePermissions, + }), + useUserMedia: ({ constraints }: { constraints: { value: MediaStreamConstraints } }) => ({ + stream: vue.shallowRef(), + stop: vueUseMock.stopStream, + start: () => vueUseMock.startUserMediaStream(constraints.value), + }), + } +}) + +function createAudioInput(deviceId: string): MediaDeviceInfo { + return { + deviceId, + groupId: '', + kind: 'audioinput', + label: deviceId, + toJSON: () => ({}), + } +} + +function createDeviceNotFoundError() { + const error = new Error('Requested device not found') + error.name = 'NotFoundError' + return error +} + +describe('useAudioDevice', () => { + afterEach(() => { + vueUseMock.audioInputs.value = [] + vi.clearAllMocks() + }) + + it('recognizes browser device-not-found errors that are not Error instances', async () => { + const { isMissingAudioInputDeviceError } = await import('./audio-device') + + expect(isMissingAudioInputDeviceError({ name: 'NotFoundError' })).toBe(true) + expect(isMissingAudioInputDeviceError({ message: 'Requested device not found' })).toBe(true) + }) + + it('retries with the system default microphone when a persisted device id is stale', async () => { + const { useAudioDevice } = await import('./audio-device') + const { selectedAudioInput, startStream } = useAudioDevice() + selectedAudioInput.value = 'stale-device-id' + + vueUseMock.startUserMediaStream + .mockRejectedValueOnce(createDeviceNotFoundError()) + .mockResolvedValueOnce(undefined) + + await startStream() + + expect(selectedAudioInput.value).toBe('') + expect(vueUseMock.startUserMediaStream).toHaveBeenNthCalledWith(1, { + audio: { + autoGainControl: true, + deviceId: { exact: 'stale-device-id' }, + echoCancellation: true, + noiseSuppression: true, + }, + }) + expect(vueUseMock.startUserMediaStream).toHaveBeenNthCalledWith(2, { + audio: { + autoGainControl: true, + echoCancellation: true, + noiseSuppression: true, + }, + }) + }) + + it('prefers an enumerated default input before falling back to unconstrained audio', async () => { + vueUseMock.audioInputs.value = [ + createAudioInput('default'), + createAudioInput('microphone-1'), + ] + + const { useAudioDevice } = await import('./audio-device') + const { selectedAudioInput, startStream } = useAudioDevice() + selectedAudioInput.value = 'stale-device-id' + + vueUseMock.startUserMediaStream.mockResolvedValueOnce(undefined) + + await startStream() + + expect(selectedAudioInput.value).toBe('default') + expect(vueUseMock.startUserMediaStream).toHaveBeenCalledWith({ + audio: { + autoGainControl: true, + deviceId: { exact: 'default' }, + echoCancellation: true, + noiseSuppression: true, + }, + }) + }) +}) diff --git a/packages/stage-ui/src/composables/audio/audio-device.ts b/packages/stage-ui/src/composables/audio/audio-device.ts index f0c1f6eb5..2f847cec2 100644 --- a/packages/stage-ui/src/composables/audio/audio-device.ts +++ b/packages/stage-ui/src/composables/audio/audio-device.ts @@ -1,32 +1,58 @@ import { useDevicesList, useUserMedia } from '@vueuse/core' import { computed, nextTick, ref, watch } from 'vue' +function resolvePreferredAudioInput(audioInputs: MediaDeviceInfo[]) { + return audioInputs.find(device => device.deviceId === 'default')?.deviceId || audioInputs[0]?.deviceId || '' +} + +export function isMissingAudioInputDeviceError(error: unknown) { + if (!error || typeof error !== 'object') + return false + + const { message, name } = error as { message?: unknown, name?: unknown } + + return name === 'NotFoundError' + || name === 'OverconstrainedError' + || (typeof message === 'string' && message.includes('Requested device not found')) +} + export function useAudioDevice(requestPermission: boolean = false) { const { audioInputs, permissionGranted, ensurePermissions } = useDevicesList({ constraints: { audio: true }, requestPermissions: requestPermission }) const selectedAudioInput = ref(audioInputs.value.find(device => device.deviceId === 'default')?.deviceId || '') + function selectAvailableAudioInput() { + if (!audioInputs.value.length) + return + + const selectedIsAvailable = audioInputs.value.some(device => device.deviceId === selectedAudioInput.value) + if (!selectedAudioInput.value || !selectedIsAvailable) + selectedAudioInput.value = resolvePreferredAudioInput(audioInputs.value) + } + const deviceConstraints = computed(() => ({ - audio: { - deviceId: { exact: selectedAudioInput.value }, - autoGainControl: true, - echoCancellation: true, - noiseSuppression: true, - }, + audio: selectedAudioInput.value + ? { + deviceId: { exact: selectedAudioInput.value }, + autoGainControl: true, + echoCancellation: true, + noiseSuppression: true, + } + : { + autoGainControl: true, + echoCancellation: true, + noiseSuppression: true, + }, })) - const { stream, stop: stopStream, start: startStream } = useUserMedia({ constraints: deviceConstraints, enabled: false, autoSwitch: true }) + const { stream, stop: stopStream, start: startUserMediaStream } = useUserMedia({ constraints: deviceConstraints, enabled: false, autoSwitch: true }) watch(audioInputs, () => { - if (selectedAudioInput.value === '' && audioInputs.value.length > 0) { - selectedAudioInput.value = audioInputs.value.find(input => input.deviceId === 'default')?.deviceId || audioInputs.value[0].deviceId - } + selectAvailableAudioInput() }) function askPermission() { return ensurePermissions() .then(() => nextTick()) .then(() => { - if (audioInputs.value.length > 0 && !selectedAudioInput.value) { - selectedAudioInput.value = audioInputs.value.find(input => input.deviceId === 'default')?.deviceId || audioInputs.value[0].deviceId - } + selectAvailableAudioInput() }) .catch((error) => { console.error('Error ensuring permissions:', error) @@ -34,6 +60,30 @@ export function useAudioDevice(requestPermission: boolean = false) { }) } + async function startStream() { + selectAvailableAudioInput() + + try { + return await startUserMediaStream() + } + catch (error) { + const fallbackDeviceId = resolvePreferredAudioInput(audioInputs.value) + if (fallbackDeviceId && fallbackDeviceId !== selectedAudioInput.value) { + selectedAudioInput.value = fallbackDeviceId + await nextTick() + return await startUserMediaStream() + } + + if (selectedAudioInput.value && isMissingAudioInputDeviceError(error)) { + selectedAudioInput.value = '' + await nextTick() + return await startUserMediaStream() + } + + throw error + } + } + return { audioInputs, selectedAudioInput, diff --git a/packages/stage-ui/src/composables/audio/audio-recorder.test.ts b/packages/stage-ui/src/composables/audio/audio-recorder.test.ts new file mode 100644 index 000000000..483083ca9 --- /dev/null +++ b/packages/stage-ui/src/composables/audio/audio-recorder.test.ts @@ -0,0 +1,147 @@ +import { describe, expect, it, vi } from 'vitest' +import { shallowRef } from 'vue' + +const mediabunnyMock = vi.hoisted(() => { + const audioSources: Array<{ track: MediaStreamTrack, encodingConfig: { codec: string, bitrate: number } }> = [] + const outputs: Array<{ target: { buffer?: Uint8Array }, finalized: boolean }> = [] + let startFailuresRemaining = 0 + + class FakeBufferTarget { + buffer?: Uint8Array + } + + class FakeWavOutputFormat {} + + class FakeMediaStreamAudioTrackSource { + errorPromise = new Promise(() => {}) + + constructor(track: MediaStreamTrack, encodingConfig: { codec: string, bitrate: number }) { + audioSources.push({ track, encodingConfig }) + } + } + + class FakeOutput { + target: FakeBufferTarget + finalized = false + + constructor(options: { target: FakeBufferTarget }) { + this.target = options.target + outputs.push(this) + } + + addAudioTrack() {} + + async getMimeType() { + return 'audio/wav' + } + + async start() { + if (startFailuresRemaining > 0) { + startFailuresRemaining -= 1 + throw new Error('start failed') + } + + this.target.buffer = new Uint8Array([outputs.length]) + } + + async finalize() { + this.finalized = true + } + } + + return { + audioSources, + outputs, + failNextStart: () => { + startFailuresRemaining += 1 + }, + FakeBufferTarget, + FakeMediaStreamAudioTrackSource, + FakeOutput, + FakeWavOutputFormat, + } +}) + +vi.mock('mediabunny', () => ({ + BufferTarget: mediabunnyMock.FakeBufferTarget, + MediaStreamAudioTrackSource: mediabunnyMock.FakeMediaStreamAudioTrackSource, + Output: mediabunnyMock.FakeOutput, + QUALITY_MEDIUM: 1, + WavOutputFormat: mediabunnyMock.FakeWavOutputFormat, +})) + +function createMediaStream() { + return { + getAudioTracks: () => ([{} as MediaStreamTrack]), + } as MediaStream +} + +describe('useAudioRecorder', () => { + it('records WAV audio with 16-bit PCM for transcription providers', async () => { + const { useAudioRecorder } = await import('./audio-recorder') + const stream = shallowRef(createMediaStream()) + + const { startRecord } = useAudioRecorder(stream) + + await startRecord() + + expect(mediabunnyMock.audioSources.at(-1)?.encodingConfig).toEqual({ + codec: 'pcm-s16', + bitrate: 1, + }) + }) + + it('keeps a new recording active while previous stop hooks finish', async () => { + const { useAudioRecorder } = await import('./audio-recorder') + const stream = shallowRef(createMediaStream()) + + const { startRecord, stopRecord, onStopRecord, isRecording } = useAudioRecorder(stream) + + let resolveFirstHook!: () => void + let shouldBlockHook = true + onStopRecord(async () => { + if (!shouldBlockHook) + return + + shouldBlockHook = false + await new Promise((resolve) => { + resolveFirstHook = resolve + }) + }) + + await startRecord() + expect(isRecording.value).toBe(true) + + const firstStop = stopRecord() + await Promise.resolve() + expect(isRecording.value).toBe(false) + + await startRecord() + expect(isRecording.value).toBe(true) + + const activeSecondOutput = mediabunnyMock.outputs.at(-1) + + resolveFirstHook() + await firstStop + + await stopRecord() + expect(isRecording.value).toBe(false) + + expect(activeSecondOutput?.finalized).toBe(true) + }) + + it('resets recorder state after startup fails so recording can be retried', async () => { + const { useAudioRecorder } = await import('./audio-recorder') + const stream = shallowRef(createMediaStream()) + + const { startRecord, isRecording } = useAudioRecorder(stream) + mediabunnyMock.failNextStart() + + await expect(startRecord()).rejects.toThrow('start failed') + expect(isRecording.value).toBe(false) + + await startRecord() + + expect(isRecording.value).toBe(true) + }) +}) diff --git a/packages/stage-ui/src/composables/audio/audio-recorder.ts b/packages/stage-ui/src/composables/audio/audio-recorder.ts index 2ad1e444a..0f0406869 100644 --- a/packages/stage-ui/src/composables/audio/audio-recorder.ts +++ b/packages/stage-ui/src/composables/audio/audio-recorder.ts @@ -2,8 +2,13 @@ import type { MaybeRefOrGetter } from 'vue' import { until } from '@vueuse/core' import { BufferTarget, MediaStreamAudioTrackSource, Output, QUALITY_MEDIUM, WavOutputFormat } from 'mediabunny' -import { ref, shallowRef, toRef } from 'vue' +import { computed, ref, shallowRef, toRef } from 'vue' +const TRANSCRIPTION_WAV_CODEC = 'pcm-s16' + +/** + * Returns the first audio track from the active microphone stream. + */ function getMediaStreamTrack(stream: MediaStream) { const tracks = stream.getAudioTracks() if (!tracks.length) @@ -11,17 +16,24 @@ function getMediaStreamTrack(stream: MediaStream) { return tracks[0] } +/** + * Records microphone input into short WAV blobs for transcription providers. + */ export function useAudioRecorder( media: MaybeRefOrGetter, ) { const mediaRef = toRef(media) const recording = shallowRef() - const mediaOutput = ref() - const mediaFormat = ref() + const mediaOutput = shallowRef() + const mediaFormat = shallowRef() + const isRecording = computed(() => !!mediaOutput.value) const onStopRecordHooks = ref Promise>>([]) + /** + * Registers a callback that receives each finalized recording blob. + */ function onStopRecord(callback: (recording: Blob | undefined) => Promise) { onStopRecordHooks.value.push(callback) // Return unsubscribe function to prevent memory leaks @@ -30,29 +42,55 @@ export function useAudioRecorder( } } + /** + * Starts recording from the current microphone stream if no recording is active. + */ async function startRecord() { + if (mediaOutput.value) + return + await until(mediaRef).toBeTruthy() const track = await getMediaStreamTrack(mediaRef.value!) - mediaOutput.value = new Output({ format: new WavOutputFormat(), target: new BufferTarget() }) + const output = new Output({ format: new WavOutputFormat(), target: new BufferTarget() }) + mediaOutput.value = output - const audioSource = new MediaStreamAudioTrackSource(track, { codec: 'pcm-f32', bitrate: QUALITY_MEDIUM }) - audioSource.errorPromise.catch(console.error) - mediaOutput.value.addAudioTrack(audioSource) + try { + const audioSource = new MediaStreamAudioTrackSource(track, { codec: TRANSCRIPTION_WAV_CODEC, bitrate: QUALITY_MEDIUM }) + audioSource.errorPromise.catch(console.error) + output.addAudioTrack(audioSource) - mediaFormat.value = await mediaOutput.value.getMimeType() - await mediaOutput.value.start() + mediaFormat.value = await output.getMimeType() + await output.start() + } + catch (error) { + if (mediaOutput.value === output) { + mediaOutput.value = undefined + mediaFormat.value = undefined + } + throw error + } } + /** + * Finalizes the active recording and runs stop hooks without blocking the next recording. + */ async function stopRecord() { - if (!mediaOutput.value) { + const activeOutput = mediaOutput.value + const activeFormat = mediaFormat.value + if (!activeOutput) { return } - await mediaOutput.value.finalize() - const bufferTarget = mediaOutput.value.target as BufferTarget | undefined + // Clear the active output before running transcription hooks so VAD can start the next utterance + // while the previous blob is still being sent to the ASR provider. + mediaOutput.value = undefined + mediaFormat.value = undefined + + await activeOutput.finalize() + const bufferTarget = activeOutput.target as BufferTarget | undefined const buffer = bufferTarget?.buffer - const audioBlob = buffer ? new Blob([buffer], { type: mediaFormat.value }) : undefined + const audioBlob = buffer ? new Blob([buffer], { type: activeFormat }) : undefined recording.value = audioBlob @@ -66,8 +104,6 @@ export function useAudioRecorder( } } - mediaOutput.value = undefined - return audioBlob } @@ -76,6 +112,7 @@ export function useAudioRecorder( stopRecord, onStopRecord, + isRecording, recording, } } diff --git a/packages/stage-ui/src/composables/audio/index.ts b/packages/stage-ui/src/composables/audio/index.ts index b56ed309b..fa710b272 100644 --- a/packages/stage-ui/src/composables/audio/index.ts +++ b/packages/stage-ui/src/composables/audio/index.ts @@ -2,3 +2,4 @@ export * from './audio-analyzer' export * from './audio-context' export * from './audio-device' export * from './audio-recorder' +export * from './voice-input-session' diff --git a/packages/stage-ui/src/composables/audio/voice-input-segment.test.ts b/packages/stage-ui/src/composables/audio/voice-input-segment.test.ts new file mode 100644 index 000000000..7241b40a5 --- /dev/null +++ b/packages/stage-ui/src/composables/audio/voice-input-segment.test.ts @@ -0,0 +1,23 @@ +import { describe, expect, it } from 'vitest' + +import { + createVoiceInputRecordingSegment, + resolveActiveVoiceInputRecordingSegmentAfterStop, +} from './voice-input-segment' + +describe('voice input recording segment tracking', () => { + it('does not clear a newer active segment when an older segment finishes stopping', () => { + const stoppedSegment = createVoiceInputRecordingSegment(1, 'volume') + const newerActiveSegment = createVoiceInputRecordingSegment(2, 'vad') + + expect(resolveActiveVoiceInputRecordingSegmentAfterStop(newerActiveSegment, stoppedSegment)) + .toBe(newerActiveSegment) + }) + + it('clears the active segment when the stopped segment is still current', () => { + const stoppedSegment = createVoiceInputRecordingSegment(1, 'manual') + + expect(resolveActiveVoiceInputRecordingSegmentAfterStop(stoppedSegment, stoppedSegment)) + .toBeUndefined() + }) +}) diff --git a/packages/stage-ui/src/composables/audio/voice-input-segment.ts b/packages/stage-ui/src/composables/audio/voice-input-segment.ts new file mode 100644 index 000000000..33c39b2f0 --- /dev/null +++ b/packages/stage-ui/src/composables/audio/voice-input-segment.ts @@ -0,0 +1,26 @@ +export type VoiceInputSessionTrigger = 'manual' | 'vad' | 'volume' + +export interface VoiceInputRecordingSegment { + id: number + trigger: VoiceInputSessionTrigger +} + +export function createVoiceInputRecordingSegment(id: number, trigger: VoiceInputSessionTrigger): VoiceInputRecordingSegment { + return { id, trigger } +} + +function isSameVoiceInputRecordingSegment( + left: VoiceInputRecordingSegment | undefined, + right: VoiceInputRecordingSegment | undefined, +) { + return !!left && !!right && left.id === right.id +} + +export function resolveActiveVoiceInputRecordingSegmentAfterStop( + activeSegment: VoiceInputRecordingSegment | undefined, + stoppedSegment: VoiceInputRecordingSegment | undefined, +) { + return isSameVoiceInputRecordingSegment(activeSegment, stoppedSegment) + ? undefined + : activeSegment +} diff --git a/packages/stage-ui/src/composables/audio/voice-input-session.test.ts b/packages/stage-ui/src/composables/audio/voice-input-session.test.ts new file mode 100644 index 000000000..a79c59bab --- /dev/null +++ b/packages/stage-ui/src/composables/audio/voice-input-session.test.ts @@ -0,0 +1,292 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { ref, shallowRef } from 'vue' + +const audioRecorderMock = vi.hoisted(() => ({ + isRecording: undefined as unknown as { value: boolean }, + startRecord: vi.fn(), + stopRecord: vi.fn(), +})) + +vi.mock('../../workers/vad/process.worklet?worker&url', () => ({ + default: 'vad-worklet-url', +})) + +vi.mock('../../stores/ai/models/vad', async () => { + const vue = await vi.importActual('vue') + + return { + useVAD: () => ({ + init: vi.fn(), + dispose: vi.fn(), + start: vi.fn(), + loaded: vue.ref(true), + isSpeech: vue.ref(false), + isSpeechProb: vue.ref(0), + isSpeechHistory: vue.ref([]), + inferenceError: vue.ref(), + }), + } +}) + +vi.mock('../../stores/modules/hearing', () => ({ + useHearingSpeechInputPipeline: () => ({ + transcribeForRecording: vi.fn(async () => ''), + }), +})) + +vi.mock('./audio-recorder', async () => { + const vue = await vi.importActual('vue') + audioRecorderMock.isRecording = vue.ref(false) + + return { + useAudioRecorder: () => ({ + isRecording: audioRecorderMock.isRecording, + startRecord: audioRecorderMock.startRecord, + stopRecord: audioRecorderMock.stopRecord, + onStopRecord: vi.fn(), + }), + } +}) + +function createMediaStream() { + return { + getAudioTracks: () => ([{} as MediaStreamTrack]), + } as MediaStream +} + +describe('useVoiceInputSession', () => { + afterEach(() => { + audioRecorderMock.isRecording.value = false + vi.useRealTimers() + vi.unstubAllGlobals() + vi.clearAllMocks() + }) + + it('clears the active recorder segment when discarding fails during stop', async () => { + const { useVoiceInputSession } = await import('./voice-input-session') + + audioRecorderMock.startRecord.mockImplementation(async () => { + audioRecorderMock.isRecording.value = true + }) + audioRecorderMock.stopRecord.mockImplementationOnce(async () => { + audioRecorderMock.isRecording.value = false + throw new Error('finalize failed') + }) + + const session = useVoiceInputSession(shallowRef(createMediaStream()), { + volumeFallback: { enabled: false }, + }) + + await expect(session.startSegment('manual')).resolves.toBe(true) + expect(session.activeRecordingTrigger.value).toBe('manual') + + await expect(session.stop({ flushActiveRecording: false })).rejects.toThrow('finalize failed') + + expect(session.activeRecordingTrigger.value).toBeUndefined() + }) + + it('reports a failed recorder start without leaving an active segment', async () => { + const { useVoiceInputSession } = await import('./voice-input-session') + const startupError = new Error('start failed') + + audioRecorderMock.startRecord.mockRejectedValueOnce(startupError) + + const session = useVoiceInputSession(shallowRef(createMediaStream()), { + volumeFallback: { enabled: false }, + }) + + await expect(session.startSegment('manual')).resolves.toBe(false) + + expect(session.activeRecordingTrigger.value).toBeUndefined() + expect(session.lastError.value).toBe(startupError) + }) + + it('clears the active segment when the caller start gate rejects', async () => { + const { useVoiceInputSession } = await import('./voice-input-session') + const gateError = new Error('gate failed') + + const session = useVoiceInputSession(shallowRef(createMediaStream()), { + volumeFallback: { enabled: false }, + canStartSegment: vi.fn() + .mockRejectedValueOnce(gateError) + .mockResolvedValueOnce(true), + }) + + await expect(session.startSegment('manual')).resolves.toBe(false) + + expect(session.activeRecordingTrigger.value).toBeUndefined() + expect(session.lastError.value).toBe(gateError) + + audioRecorderMock.startRecord.mockImplementationOnce(async () => { + audioRecorderMock.isRecording.value = true + }) + + await expect(session.startSegment('manual')).resolves.toBe(true) + expect(session.activeRecordingTrigger.value).toBe('manual') + }) + + it('clears the active segment when the caller start hook rejects', async () => { + const { useVoiceInputSession } = await import('./voice-input-session') + const hookError = new Error('start hook failed') + + const session = useVoiceInputSession(shallowRef(createMediaStream()), { + volumeFallback: { enabled: false }, + onSegmentStart: vi.fn().mockRejectedValueOnce(hookError), + }) + + await expect(session.startSegment('manual')).resolves.toBe(false) + + expect(audioRecorderMock.startRecord).not.toHaveBeenCalled() + expect(session.activeRecordingTrigger.value).toBeUndefined() + expect(session.lastError.value).toBe(hookError) + }) + + it('stops and clears the recorder when the caller started hook rejects', async () => { + const { useVoiceInputSession } = await import('./voice-input-session') + const hookError = new Error('started hook failed') + + audioRecorderMock.startRecord.mockImplementation(async () => { + audioRecorderMock.isRecording.value = true + }) + audioRecorderMock.stopRecord.mockImplementation(async () => { + audioRecorderMock.isRecording.value = false + }) + + const session = useVoiceInputSession(shallowRef(createMediaStream()), { + volumeFallback: { enabled: false }, + onSegmentStarted: vi.fn().mockRejectedValueOnce(hookError), + }) + + await expect(session.startSegment('manual')).resolves.toBe(false) + + expect(audioRecorderMock.stopRecord).toHaveBeenCalledOnce() + expect(session.isRecording.value).toBe(false) + expect(session.activeRecordingTrigger.value).toBeUndefined() + expect(session.lastError.value).toBe(hookError) + }) + + it('finalizes the recorder when the caller stop hook rejects', async () => { + const { useVoiceInputSession } = await import('./voice-input-session') + const hookError = new Error('stop hook failed') + const onTranscriptionError = vi.fn() + + audioRecorderMock.startRecord.mockImplementation(async () => { + audioRecorderMock.isRecording.value = true + }) + audioRecorderMock.stopRecord.mockImplementation(async () => { + audioRecorderMock.isRecording.value = false + }) + + const session = useVoiceInputSession(shallowRef(createMediaStream()), { + volumeFallback: { enabled: false }, + onSegmentStop: vi.fn().mockRejectedValueOnce(hookError), + onTranscriptionError, + }) + + await expect(session.startSegment('manual')).resolves.toBe(true) + await expect(session.stopSegment('manual')).resolves.toBeUndefined() + + expect(audioRecorderMock.stopRecord).toHaveBeenCalledOnce() + expect(session.isRecording.value).toBe(false) + expect(session.activeRecordingTrigger.value).toBeUndefined() + expect(session.lastError.value).toBe(hookError) + expect(onTranscriptionError).toHaveBeenCalledWith(expect.objectContaining({ error: hookError })) + }) + + it('stops an active recorder segment after stream mode becomes enabled', async () => { + const { useVoiceInputSession } = await import('./voice-input-session') + const shouldUseStreamInput = ref(false) + + audioRecorderMock.startRecord.mockImplementation(async () => { + audioRecorderMock.isRecording.value = true + }) + audioRecorderMock.stopRecord.mockImplementation(async () => { + audioRecorderMock.isRecording.value = false + }) + + const session = useVoiceInputSession(shallowRef(createMediaStream()), { + shouldUseStreamInput, + volumeFallback: { enabled: false }, + }) + + await expect(session.startSegment('manual')).resolves.toBe(true) + shouldUseStreamInput.value = true + await session.stopSegment('manual') + + expect(audioRecorderMock.stopRecord).toHaveBeenCalledOnce() + expect(session.isRecording.value).toBe(false) + expect(session.activeRecordingTrigger.value).toBeUndefined() + }) + + it('lets volume fallback finalize a VAD-owned segment after silence', async () => { + vi.useFakeTimers() + vi.setSystemTime(1000) + + const animationFrames: FrameRequestCallback[] = [] + const stopRecord = audioRecorderMock.stopRecord.mockImplementation(async () => { + audioRecorderMock.isRecording.value = false + }) + audioRecorderMock.startRecord.mockImplementation(async () => { + audioRecorderMock.isRecording.value = true + }) + + class FakeAudioContext { + state: AudioContextState = 'running' + destination = {} + + createMediaStreamSource() { + return { + connect: vi.fn(), + disconnect: vi.fn(), + } + } + + createAnalyser() { + return { + fftSize: 512, + smoothingTimeConstant: 0, + connect: vi.fn(), + disconnect: vi.fn(), + getByteTimeDomainData: (data: Uint8Array) => data.fill(128), + } + } + + createGain() { + return { + gain: { value: 1 }, + connect: vi.fn(), + disconnect: vi.fn(), + } + } + + resume = vi.fn() + close = vi.fn() + } + + vi.stubGlobal('AudioContext', FakeAudioContext) + vi.stubGlobal('requestAnimationFrame', vi.fn((callback: FrameRequestCallback) => { + animationFrames.push(callback) + return animationFrames.length + })) + vi.stubGlobal('cancelAnimationFrame', vi.fn()) + + const { useVoiceInputSession } = await import('./voice-input-session') + const session = useVoiceInputSession(shallowRef(createMediaStream()), { + volumeFallback: { + enabled: true, + stopDelayMs: 10, + }, + }) + + await expect(session.startSegment('vad')).resolves.toBe(true) + await session.startAutoSegmentation() + + animationFrames.shift()?.(1000) + vi.setSystemTime(1011) + animationFrames.shift()?.(1011) + await Promise.resolve() + + expect(stopRecord).toHaveBeenCalledOnce() + expect(session.activeRecordingTrigger.value).toBeUndefined() + }) +}) diff --git a/packages/stage-ui/src/composables/audio/voice-input-session.ts b/packages/stage-ui/src/composables/audio/voice-input-session.ts new file mode 100644 index 000000000..db8ea2e18 --- /dev/null +++ b/packages/stage-ui/src/composables/audio/voice-input-session.ts @@ -0,0 +1,589 @@ +import type { MaybeRefOrGetter } from 'vue' + +import type { VoiceInputRecordingSegment, VoiceInputSessionTrigger } from './voice-input-segment' +import type { VoiceInputTranscriptionTicket } from './voice-input-transcription-chain' + +import { computed, ref, shallowRef, toRef } from 'vue' + +import workletUrl from '../../workers/vad/process.worklet?worker&url' + +import { useVAD } from '../../stores/ai/models/vad' +import { useHearingSpeechInputPipeline } from '../../stores/modules/hearing' +import { useAudioRecorder } from './audio-recorder' +import { + createVoiceInputRecordingSegment, + resolveActiveVoiceInputRecordingSegmentAfterStop, +} from './voice-input-segment' +import { createVoiceInputTranscriptionChain } from './voice-input-transcription-chain' +import { startVoiceInputVadDetectionSafely } from './voice-input-vad-startup' + +export type { VoiceInputSessionTrigger } from './voice-input-segment' + +export type VoiceInputSessionLogLevel = 'info' | 'warn' | 'error' + +export interface VoiceInputSessionGate { + skip?: boolean + reason?: string + details?: Record +} + +export interface VoiceInputSessionEvent { + trigger: VoiceInputSessionTrigger + recording?: Blob + text?: string + error?: unknown + metadata?: Record + gate?: VoiceInputSessionGate +} + +export interface VoiceInputSessionVadOptions { + threshold?: MaybeRefOrGetter + minSilenceDurationMs?: MaybeRefOrGetter + speechPadMs?: MaybeRefOrGetter + minSpeechDurationMs?: MaybeRefOrGetter +} + +export interface VoiceInputSessionVolumeFallbackOptions { + enabled?: MaybeRefOrGetter + startThreshold?: number + stopThreshold?: number + startFrames?: number + stopDelayMs?: number + logIntervalMs?: number +} + +export interface VoiceInputSessionOptions { + shouldUseStreamInput?: MaybeRefOrGetter + vad?: VoiceInputSessionVadOptions + volumeFallback?: VoiceInputSessionVolumeFallbackOptions + canStartSegment?: (event: VoiceInputSessionEvent) => boolean | Promise + inspectBeforeTranscription?: (event: VoiceInputSessionEvent) => VoiceInputSessionGate | Promise | undefined + inspectAfterTranscription?: (event: VoiceInputSessionEvent) => VoiceInputSessionGate | Promise | undefined + onLog?: (level: VoiceInputSessionLogLevel, event: string, message: string, details?: Record) => void + onSegmentStart?: (event: VoiceInputSessionEvent) => void | Promise + onSegmentStarted?: (event: VoiceInputSessionEvent) => void | Promise + onSegmentStop?: (event: VoiceInputSessionEvent) => void | Promise + onSegmentStopped?: (event: VoiceInputSessionEvent) => void | Promise + onRecordingReady?: (event: VoiceInputSessionEvent) => Record | void | Promise | void> + onRecordingSkipped?: (event: VoiceInputSessionEvent) => void | Promise + onTranscriptionStart?: (event: VoiceInputSessionEvent) => void | Promise + onTranscriptionResult?: (event: VoiceInputSessionEvent & { text: string }) => void | Promise + onTranscriptionEmpty?: (event: VoiceInputSessionEvent & { text: string }) => void | Promise + onTranscriptionError?: (event: VoiceInputSessionEvent & { error: unknown }) => void | Promise +} + +const DEFAULT_VOLUME_FALLBACK_START_THRESHOLD = 10 +const DEFAULT_VOLUME_FALLBACK_STOP_THRESHOLD = 6 +const DEFAULT_VOLUME_FALLBACK_START_FRAMES = 4 +const DEFAULT_VOLUME_FALLBACK_STOP_DELAY_MS = 900 +const DEFAULT_VOLUME_FALLBACK_LOG_INTERVAL_MS = 2000 + +function calculateTimeDomainVolumeLevel(dataArray: Uint8Array) { + let sum = 0 + for (let i = 0; i < dataArray.length; i++) { + const centered = (dataArray[i] - 128) / 128 + sum += centered * centered + } + + return Math.min(100, Math.sqrt(sum / dataArray.length) * 100 * 3) +} + +/** + * Shared voice-input session for both manual STT tests and always-on stage listening. + * + * Owns: + * - recorder-backed segment creation + * - VAD-triggered auto segmentation + * - volume-triggered fallback segmentation + * - record-then-transcribe ASR calls + * + * Leaves product-specific behavior, such as sending text to chat or updating UI state, to callbacks. + */ +export function useVoiceInputSession( + media: MaybeRefOrGetter, + options: VoiceInputSessionOptions = {}, +) { + const mediaRef = toRef(media) + const shouldUseStreamInput = toRef(options.shouldUseStreamInput ?? false) + const volumeFallbackEnabled = toRef(options.volumeFallback?.enabled ?? true) + const hearingPipeline = useHearingSpeechInputPipeline() + const { transcribeForRecording } = hearingPipeline + const recorder = useAudioRecorder(mediaRef) + + const activeRecordingSegment = shallowRef() + const activeRecordingTrigger = computed(() => activeRecordingSegment.value?.trigger) + const isTranscribing = ref(false) + const lastTranscriptionText = ref('') + const lastError = ref() + const transcriptionChain = createVoiceInputTranscriptionChain() + const stoppedRecordingSegments: VoiceInputRecordingSegment[] = [] + let nextRecordingSegmentId = 0 + let discardNextRecording = false + let activeTranscriptionCount = 0 + + const { + init: initVAD, + dispose: disposeVAD, + start: startVAD, + loaded: vadLoaded, + isSpeech: isSpeechVAD, + isSpeechProb, + isSpeechHistory, + inferenceError: vadError, + } = useVAD(workletUrl, { + threshold: options.vad?.threshold, + minSilenceDurationMs: options.vad?.minSilenceDurationMs, + speechPadMs: options.vad?.speechPadMs, + minSpeechDurationMs: options.vad?.minSpeechDurationMs, + onSpeechStart: () => { + void startSegment('vad') + }, + onSpeechEnd: () => { + void stopSegment('vad') + }, + }) + + let volumeFallbackAudioContext: AudioContext | undefined + let volumeFallbackSourceNode: MediaStreamAudioSourceNode | undefined + let volumeFallbackAnalyserNode: AnalyserNode | undefined + let volumeFallbackSilentGainNode: GainNode | undefined + let volumeFallbackDataArray: Uint8Array | undefined + let volumeFallbackAnimationFrame: number | undefined + let volumeFallbackSpeechFrames = 0 + let volumeFallbackLastSpeechAt = 0 + let volumeFallbackLastLogAt = 0 + + const isRecording = computed(() => recorder.isRecording.value) + + function log(level: VoiceInputSessionLogLevel, event: string, message: string, details?: Record) { + options.onLog?.(level, event, message, details) + } + + function markTranscriptionStarted() { + activeTranscriptionCount += 1 + isTranscribing.value = true + } + + function markTranscriptionFinished() { + activeTranscriptionCount = Math.max(0, activeTranscriptionCount - 1) + isTranscribing.value = activeTranscriptionCount > 0 + } + + function isStaleTranscriptionTicket(ticket: VoiceInputTranscriptionTicket, trigger: VoiceInputSessionTrigger, phase: string) { + if (ticket.isCurrent()) + return false + + log('info', 'recording-drop-stale-session', 'Dropping stale recorder-backed transcription work after the listening session changed.', { + trigger, + phase, + }) + return true + } + + async function discardActiveRecorderSegment(segment: VoiceInputRecordingSegment) { + discardNextRecording = true + try { + await recorder.stopRecord() + } + finally { + discardNextRecording = false + activeRecordingSegment.value = resolveActiveVoiceInputRecordingSegmentAfterStop(activeRecordingSegment.value, segment) + } + } + + async function startSegment(trigger: VoiceInputSessionTrigger = 'manual') { + const event: VoiceInputSessionEvent = { trigger } + if (shouldUseStreamInput.value) { + log('info', 'segment-start-skipped-streaming', 'Recorder segment start skipped because streaming transcription is active.', { trigger }) + return false + } + + if (isRecording.value || activeRecordingSegment.value) { + log('info', 'segment-start-skipped-active', 'Recorder segment start skipped because another segment is already active.', { + trigger, + activeRecordingTrigger: activeRecordingTrigger.value, + }) + return false + } + + const segment = createVoiceInputRecordingSegment(++nextRecordingSegmentId, trigger) + activeRecordingSegment.value = segment + + if (options.canStartSegment) { + try { + if (!await options.canStartSegment(event)) { + log('info', 'segment-start-skipped-gate', 'Recorder segment start skipped by caller gate.', { trigger }) + activeRecordingSegment.value = resolveActiveVoiceInputRecordingSegmentAfterStop(activeRecordingSegment.value, segment) + return false + } + } + catch (error) { + activeRecordingSegment.value = resolveActiveVoiceInputRecordingSegmentAfterStop(activeRecordingSegment.value, segment) + lastError.value = error + log('error', 'segment-start-gate-failed', 'Recorder segment start gate failed.', { trigger, error }) + await options.onTranscriptionError?.({ trigger, error }) + return false + } + } + + try { + await options.onSegmentStart?.(event) + await recorder.startRecord() + + try { + await options.onSegmentStarted?.(event) + } + catch (error) { + await discardActiveRecorderSegment(segment) + throw error + } + + return true + } + catch (error) { + activeRecordingSegment.value = resolveActiveVoiceInputRecordingSegmentAfterStop(activeRecordingSegment.value, segment) + lastError.value = error + log('error', 'segment-start-failed', 'Failed to start recorder-backed voice input segment.', { trigger, error }) + await options.onTranscriptionError?.({ trigger, error }) + return false + } + } + + async function stopSegment(trigger: VoiceInputSessionTrigger = 'manual') { + const event: VoiceInputSessionEvent = { trigger } + const segment = activeRecordingSegment.value + + if (shouldUseStreamInput.value && !isRecording.value && !segment) { + log('info', 'segment-stop-skipped-streaming', 'Recorder segment stop skipped because streaming transcription is active.', { trigger }) + return + } + + if (segment && segment.trigger !== trigger) { + log('info', 'segment-stop-skipped-trigger-mismatch', 'Recorder segment stop skipped because another detector owns the active segment.', { + trigger, + activeRecordingTrigger: activeRecordingTrigger.value, + }) + return + } + + if (!isRecording.value) { + log('warn', 'segment-stop-without-active-recorder', 'Recorder segment stop requested without an active recording.', { trigger }) + return + } + + const stoppedSegment = segment ?? createVoiceInputRecordingSegment(++nextRecordingSegmentId, trigger) + + try { + await options.onSegmentStop?.(event) + } + catch (error) { + lastError.value = error + log('error', 'segment-stop-hook-failed', 'Caller stop hook failed; finalizing recorder segment anyway.', { trigger, error }) + await options.onTranscriptionError?.({ trigger, error }) + } + + try { + stoppedRecordingSegments.push(stoppedSegment) + activeRecordingSegment.value = resolveActiveVoiceInputRecordingSegmentAfterStop(activeRecordingSegment.value, stoppedSegment) + await recorder.stopRecord() + await options.onSegmentStopped?.(event) + } + catch (error) { + const queuedIndex = stoppedRecordingSegments.findIndex(item => item.id === stoppedSegment.id) + if (queuedIndex !== -1) + stoppedRecordingSegments.splice(queuedIndex, 1) + lastError.value = error + log('error', 'segment-stop-failed', 'Failed to stop recorder-backed voice input segment.', { trigger, error }) + await options.onTranscriptionError?.({ trigger, error }) + } + finally { + activeRecordingSegment.value = resolveActiveVoiceInputRecordingSegmentAfterStop(activeRecordingSegment.value, stoppedSegment) + } + } + + async function processRecording(recording: Blob | undefined, trigger: VoiceInputSessionTrigger, ticket: VoiceInputTranscriptionTicket) { + const event: VoiceInputSessionEvent = { trigger, recording } + + if (isStaleTranscriptionTicket(ticket, trigger, 'recording-start')) + return + + if (!recording || recording.size <= 0) { + log('warn', 'recording-drop-empty', 'Dropping empty recorder-backed voice input segment.', { trigger, recording }) + await options.onRecordingSkipped?.(event) + return + } + + const metadata = await options.onRecordingReady?.(event) ?? undefined + const readyEvent = { ...event, metadata } + if (isStaleTranscriptionTicket(ticket, trigger, 'recording-ready')) + return + + const beforeGate = await options.inspectBeforeTranscription?.(readyEvent) + if (isStaleTranscriptionTicket(ticket, trigger, 'before-transcription-gate')) + return + + if (beforeGate?.skip) { + log('info', 'recording-drop-before-asr', 'Skipping recorder-backed segment before transcription request.', { + trigger, + gate: beforeGate, + }) + await options.onRecordingSkipped?.({ ...readyEvent, gate: beforeGate }) + return + } + + markTranscriptionStarted() + + let text = '' + try { + await options.onTranscriptionStart?.(readyEvent) + if (isStaleTranscriptionTicket(ticket, trigger, 'transcription-started')) + return + + text = await transcribeForRecording(recording) ?? '' + } + catch (error) { + if (isStaleTranscriptionTicket(ticket, trigger, 'transcription-error')) + return + + lastError.value = error + log('error', 'recording-transcription-error', 'Transcription provider threw while processing recorder-backed segment.', { trigger, error }) + await options.onTranscriptionError?.({ ...readyEvent, error }) + return + } + finally { + markTranscriptionFinished() + } + + if (isStaleTranscriptionTicket(ticket, trigger, 'transcription-result')) + return + + const resultEvent = { ...readyEvent, text } + const afterGate = await options.inspectAfterTranscription?.(resultEvent) + if (isStaleTranscriptionTicket(ticket, trigger, 'after-transcription-gate')) + return + + if (afterGate?.skip) { + log('info', 'recording-drop-after-asr', 'Dropping stale transcription result after transcription request.', { + trigger, + gate: afterGate, + text, + }) + await options.onRecordingSkipped?.({ ...resultEvent, gate: afterGate }) + return + } + + if (!text || !text.trim()) { + log('warn', 'recording-transcription-empty', 'Transcription provider returned empty text for recorder-backed segment.', { trigger, text }) + await options.onTranscriptionEmpty?.(resultEvent) + return + } + + lastTranscriptionText.value = text + await options.onTranscriptionResult?.(resultEvent) + } + + recorder.onStopRecord(async (recording) => { + if (discardNextRecording) { + discardNextRecording = false + return + } + + const segment = stoppedRecordingSegments.shift() + const trigger = segment?.trigger ?? activeRecordingTrigger.value ?? 'manual' + await transcriptionChain + .enqueue(ticket => processRecording(recording, trigger, ticket)) + .catch((error) => { + lastError.value = error + log('error', 'recording-processing-error', 'Voice input recording processing failed.', { trigger, error }) + }) + }) + + function stopVolumeFallback() { + if (volumeFallbackAnimationFrame !== undefined) { + cancelAnimationFrame(volumeFallbackAnimationFrame) + volumeFallbackAnimationFrame = undefined + } + + volumeFallbackSourceNode?.disconnect() + volumeFallbackAnalyserNode?.disconnect() + volumeFallbackSilentGainNode?.disconnect() + volumeFallbackSourceNode = undefined + volumeFallbackAnalyserNode = undefined + volumeFallbackSilentGainNode = undefined + volumeFallbackDataArray = undefined + volumeFallbackSpeechFrames = 0 + volumeFallbackLastSpeechAt = 0 + volumeFallbackLastLogAt = 0 + + if (volumeFallbackAudioContext && volumeFallbackAudioContext.state !== 'closed') + void volumeFallbackAudioContext.close() + volumeFallbackAudioContext = undefined + } + + async function startVolumeFallback(stream: MediaStream) { + if (!volumeFallbackEnabled.value || shouldUseStreamInput.value) + return + + stopVolumeFallback() + + const startThreshold = options.volumeFallback?.startThreshold ?? DEFAULT_VOLUME_FALLBACK_START_THRESHOLD + const stopThreshold = options.volumeFallback?.stopThreshold ?? DEFAULT_VOLUME_FALLBACK_STOP_THRESHOLD + const startFrames = options.volumeFallback?.startFrames ?? DEFAULT_VOLUME_FALLBACK_START_FRAMES + const stopDelayMs = options.volumeFallback?.stopDelayMs ?? DEFAULT_VOLUME_FALLBACK_STOP_DELAY_MS + const logIntervalMs = options.volumeFallback?.logIntervalMs ?? DEFAULT_VOLUME_FALLBACK_LOG_INTERVAL_MS + + try { + volumeFallbackAudioContext = new AudioContext({ latencyHint: 'interactive' }) + if (volumeFallbackAudioContext.state === 'suspended') + await volumeFallbackAudioContext.resume() + + volumeFallbackSourceNode = volumeFallbackAudioContext.createMediaStreamSource(stream) + volumeFallbackAnalyserNode = volumeFallbackAudioContext.createAnalyser() + volumeFallbackAnalyserNode.fftSize = 512 + volumeFallbackAnalyserNode.smoothingTimeConstant = 0.25 + volumeFallbackSilentGainNode = volumeFallbackAudioContext.createGain() + volumeFallbackSilentGainNode.gain.value = 0 + volumeFallbackDataArray = new Uint8Array(volumeFallbackAnalyserNode.fftSize) as Uint8Array + + volumeFallbackSourceNode.connect(volumeFallbackAnalyserNode) + volumeFallbackAnalyserNode.connect(volumeFallbackSilentGainNode) + volumeFallbackSilentGainNode.connect(volumeFallbackAudioContext.destination) + + log('info', 'volume-fallback-started', 'Volume-based recorder fallback started for record-then-transcribe voice input.', { + startThreshold, + stopThreshold, + stopDelayMs, + }) + + const analyze = () => { + if (!volumeFallbackAnalyserNode || !volumeFallbackDataArray) + return + + volumeFallbackAnalyserNode.getByteTimeDomainData(volumeFallbackDataArray) + const level = calculateTimeDomainVolumeLevel(volumeFallbackDataArray) + const now = Date.now() + + if (now - volumeFallbackLastLogAt >= logIntervalMs) { + volumeFallbackLastLogAt = now + log('info', 'volume-fallback-level', 'Volume fallback sampled microphone input.', { + level: Number(level.toFixed(1)), + isRecording: isRecording.value, + activeRecordingTrigger: activeRecordingTrigger.value, + startThreshold, + stopThreshold, + }) + } + + if (shouldUseStreamInput.value) { + volumeFallbackSpeechFrames = 0 + volumeFallbackAnimationFrame = requestAnimationFrame(analyze) + return + } + + if (!isRecording.value) { + if (level >= startThreshold) { + volumeFallbackSpeechFrames += 1 + if (volumeFallbackSpeechFrames >= startFrames) { + volumeFallbackLastSpeechAt = now + volumeFallbackSpeechFrames = 0 + log('info', 'volume-fallback-speech-start', 'Volume fallback detected speech; starting recorder segment.', { + level: Number(level.toFixed(1)), + }) + void startSegment('volume') + } + } + else { + volumeFallbackSpeechFrames = 0 + } + } + else if (activeRecordingTrigger.value === 'volume' || activeRecordingTrigger.value === 'vad') { + if (level > stopThreshold) { + volumeFallbackLastSpeechAt = now + } + else if (!volumeFallbackLastSpeechAt) { + volumeFallbackLastSpeechAt = now + } + else if (volumeFallbackLastSpeechAt && now - volumeFallbackLastSpeechAt >= stopDelayMs) { + const trigger = activeRecordingTrigger.value + volumeFallbackLastSpeechAt = 0 + log('info', 'volume-fallback-speech-end', 'Volume fallback detected silence; finalizing recorder segment.', { + level: Number(level.toFixed(1)), + silenceMs: stopDelayMs, + trigger, + }) + void stopSegment(trigger) + } + } + + volumeFallbackAnimationFrame = requestAnimationFrame(analyze) + } + + volumeFallbackAnimationFrame = requestAnimationFrame(analyze) + } + catch (error) { + stopVolumeFallback() + lastError.value = error + log('error', 'volume-fallback-start-failed', 'Failed to start volume-based recorder fallback.', { error }) + } + } + + async function startAutoSegmentation() { + const stream = mediaRef.value + if (!stream) + throw new Error('No microphone stream available for voice input') + + await startVoiceInputVadDetectionSafely({ + init: initVAD, + loaded: () => vadLoaded.value, + start: startVAD, + stream, + getError: () => vadError.value, + log, + }) + await startVolumeFallback(stream) + } + + async function stop(options: { flushActiveRecording?: boolean } = {}) { + stopVolumeFallback() + disposeVAD() + transcriptionChain.reset() + stoppedRecordingSegments.length = 0 + + if (options.flushActiveRecording && isRecording.value) { + await stopSegment(activeRecordingTrigger.value ?? 'manual') + await transcriptionChain.idle() + transcriptionChain.reset() + } + else if (isRecording.value) { + discardNextRecording = true + try { + await recorder.stopRecord() + } + finally { + discardNextRecording = false + activeRecordingSegment.value = undefined + } + } + else { + activeRecordingSegment.value = undefined + } + } + + return { + isRecording, + isTranscribing, + lastTranscriptionText, + lastError, + activeRecordingTrigger, + isSpeechVAD, + isSpeechProb, + isSpeechHistory, + vadLoaded, + vadError, + + startSegment, + stopSegment, + startAutoSegmentation, + stop, + } +} diff --git a/packages/stage-ui/src/composables/audio/voice-input-transcription-chain.test.ts b/packages/stage-ui/src/composables/audio/voice-input-transcription-chain.test.ts new file mode 100644 index 000000000..fd1592902 --- /dev/null +++ b/packages/stage-ui/src/composables/audio/voice-input-transcription-chain.test.ts @@ -0,0 +1,72 @@ +import { describe, expect, it } from 'vitest' + +import { createVoiceInputTranscriptionChain } from './voice-input-transcription-chain' + +describe('createVoiceInputTranscriptionChain', () => { + it('runs queued transcriptions in order while the session stays current', async () => { + const chain = createVoiceInputTranscriptionChain() + const resolved: string[] = [] + + let finishFirst!: () => void + const first = chain.enqueue(async () => { + await new Promise((resolve) => { + finishFirst = resolve + }) + resolved.push('first') + }) + const second = chain.enqueue(async () => { + resolved.push('second') + }) + + await Promise.resolve() + await Promise.resolve() + + expect(resolved).toEqual([]) + + finishFirst() + await first + await second + + expect(resolved).toEqual(['first', 'second']) + }) + + it('lets fresh transcriptions start after reset without waiting for stale provider work', async () => { + const chain = createVoiceInputTranscriptionChain() + const resolved: string[] = [] + + void chain.enqueue(async () => { + await new Promise(() => {}) + resolved.push('stale') + }) + + await Promise.resolve() + chain.reset() + + await chain.enqueue(async (ticket) => { + expect(ticket.isCurrent()).toBe(true) + resolved.push('fresh') + }) + + expect(resolved).toEqual(['fresh']) + }) + + it('marks running tickets stale after reset so late results cannot publish', async () => { + const chain = createVoiceInputTranscriptionChain() + let finishFirst!: () => void + let firstTicketStillCurrent = true + + const first = chain.enqueue(async (ticket) => { + await new Promise((resolve) => { + finishFirst = resolve + }) + firstTicketStillCurrent = ticket.isCurrent() + }) + + await Promise.resolve() + chain.reset() + finishFirst() + await first + + expect(firstTicketStillCurrent).toBe(false) + }) +}) diff --git a/packages/stage-ui/src/composables/audio/voice-input-transcription-chain.ts b/packages/stage-ui/src/composables/audio/voice-input-transcription-chain.ts new file mode 100644 index 000000000..ded56950b --- /dev/null +++ b/packages/stage-ui/src/composables/audio/voice-input-transcription-chain.ts @@ -0,0 +1,54 @@ +export interface VoiceInputTranscriptionTicket { + /** Returns whether this queued transcription still belongs to the active listening session. */ + isCurrent: () => boolean +} + +export interface VoiceInputTranscriptionChain { + /** Runs work after earlier current transcription tasks have settled. */ + enqueue: (task: (ticket: VoiceInputTranscriptionTicket) => Promise | T) => Promise + /** Invalidates pending/running tickets and lets future work start from a fresh tail. */ + reset: () => void + /** Resolves when all currently chained transcription work has settled. */ + idle: () => Promise +} + +export function createVoiceInputTranscriptionChain(): VoiceInputTranscriptionChain { + let tail = Promise.resolve() + let generation = 0 + + function enqueue(task: (ticket: VoiceInputTranscriptionTicket) => Promise | T) { + const taskGeneration = generation + const ticket: VoiceInputTranscriptionTicket = { + isCurrent: () => taskGeneration === generation, + } + + const run = tail.then(async () => { + if (!ticket.isCurrent()) + return undefined + + return task(ticket) + }) + + tail = run.then( + () => undefined, + () => undefined, + ) + + return run + } + + function reset() { + generation += 1 + tail = Promise.resolve() + } + + function idle() { + return tail + } + + return { + enqueue, + reset, + idle, + } +} diff --git a/packages/stage-ui/src/composables/audio/voice-input-vad-startup.test.ts b/packages/stage-ui/src/composables/audio/voice-input-vad-startup.test.ts new file mode 100644 index 000000000..0bf418b9a --- /dev/null +++ b/packages/stage-ui/src/composables/audio/voice-input-vad-startup.test.ts @@ -0,0 +1,29 @@ +import { describe, expect, it, vi } from 'vitest' + +import { startVoiceInputVadDetectionSafely } from './voice-input-vad-startup' + +describe('voice input VAD startup', () => { + it('returns false and logs when VAD initialization throws', async () => { + const init = vi.fn().mockRejectedValue(new Error('vad unavailable')) + const start = vi.fn() + const log = vi.fn() + + await expect(startVoiceInputVadDetectionSafely({ + init, + loaded: () => false, + start, + stream: {} as MediaStream, + log, + })).resolves.toBe(false) + + expect(start).not.toHaveBeenCalled() + expect(log).toHaveBeenCalledWith( + 'error', + 'vad-init-failed', + 'VAD initialization failed.', + expect.objectContaining({ + error: expect.any(Error), + }), + ) + }) +}) diff --git a/packages/stage-ui/src/composables/audio/voice-input-vad-startup.ts b/packages/stage-ui/src/composables/audio/voice-input-vad-startup.ts new file mode 100644 index 000000000..03f5e023c --- /dev/null +++ b/packages/stage-ui/src/composables/audio/voice-input-vad-startup.ts @@ -0,0 +1,38 @@ +import type { VoiceInputSessionLogLevel } from './voice-input-session' + +export interface VoiceInputVadStartupOptions { + init: () => Promise + loaded: () => boolean + start: (stream: MediaStream) => Promise + stream: MediaStream + getError?: () => unknown + log?: (level: VoiceInputSessionLogLevel, event: string, message: string, details?: Record) => void +} + +export async function startVoiceInputVadDetectionSafely(options: VoiceInputVadStartupOptions) { + try { + await options.init() + + if (options.loaded()) { + options.log?.('info', 'vad-start', 'VAD initialized successfully; starting against microphone stream.', { + stream: options.stream, + }) + await options.start(options.stream) + return true + } + + const error = options.getError?.() + if (error) { + options.log?.('error', 'vad-init-failed', 'VAD initialization failed.', { + error, + }) + } + } + catch (error) { + options.log?.('error', 'vad-init-failed', 'VAD initialization failed.', { + error, + }) + } + + return false +} diff --git a/packages/stage-ui/src/libs/audio/vad.test.ts b/packages/stage-ui/src/libs/audio/vad.test.ts new file mode 100644 index 000000000..d51141929 --- /dev/null +++ b/packages/stage-ui/src/libs/audio/vad.test.ts @@ -0,0 +1,111 @@ +import type { BaseVAD } from './vad' + +import { afterEach, describe, expect, it, vi } from 'vitest' + +import { createVADStates } from './vad' + +class FakeAudioNode { + connect = vi.fn() + disconnect = vi.fn() + port = { onmessage: null as ((event: MessageEvent) => void) | null } +} + +class FakeAudioContext { + state: AudioContextState = 'running' + destination = new FakeAudioNode() + audioWorklet = { + addModule: vi.fn(async () => {}), + } + + createMediaStreamSource = vi.fn(() => new FakeAudioNode()) + createGain = vi.fn(() => ({ + gain: { value: 1 }, + connect: vi.fn(), + disconnect: vi.fn(), + })) + + async resume() { + this.state = 'running' + } + + suspend = vi.fn(async () => { + this.state = 'suspended' + }) + + close = vi.fn(async () => { + this.state = 'closed' + }) +} + +class FakeAudioWorkletNode extends FakeAudioNode { + constructor() { + super() + } +} + +function createVADMock(): BaseVAD { + return { + initialize: vi.fn(async () => {}), + processAudio: vi.fn(async () => {}), + on: vi.fn(), + off: vi.fn(), + } +} + +describe('createVADStates', () => { + afterEach(() => { + vi.unstubAllGlobals() + }) + + it('does not stop the caller-owned microphone stream when disposing VAD nodes', async () => { + // NOTICE: + // Vitest node tests do not provide Web Audio constructors. + // The regression is about our ownership policy around a caller-owned MediaStream, not browser audio. + // Source/context: packages/stage-ui/src/libs/audio/vad.ts dispose previously called track.stop(). + // Removal condition: replace this with a browser-mode Web Audio lifecycle test. + vi.stubGlobal('AudioContext', FakeAudioContext) + vi.stubGlobal('AudioWorkletNode', FakeAudioWorkletNode) + const stop = vi.fn() + const stream = { + getTracks: () => [{ stop }], + } as unknown as MediaStream + + const manager = createVADStates(createVADMock(), '/vad-worklet.js') + await manager.initialize() + await manager.start(stream) + manager.dispose() + + expect(stop).not.toHaveBeenCalled() + }) + + it('disconnects the previous microphone source before starting a new graph', async () => { + // NOTICE: + // The page can call start from both the init continuation and the stream/loaded watcher. + // This fake Web Audio graph keeps the regression focused on duplicate source-node wiring. + // Source/context: apps/stage-tamagotchi/src/renderer/pages/index.vue can restart VAD around stream changes. + // Removal condition: replace this with browser-mode Web Audio graph lifecycle coverage. + const createdSources: FakeAudioNode[] = [] + class ReconnectAudioContext extends FakeAudioContext { + createMediaStreamSource = vi.fn(() => { + const source = new FakeAudioNode() + createdSources.push(source) + return source + }) + } + + vi.stubGlobal('AudioContext', ReconnectAudioContext) + vi.stubGlobal('AudioWorkletNode', FakeAudioWorkletNode) + const stream = { + getTracks: () => [], + } as unknown as MediaStream + + const manager = createVADStates(createVADMock(), '/vad-worklet.js') + await manager.initialize() + await manager.start(stream) + await manager.start(stream) + + expect(createdSources).toHaveLength(2) + expect(createdSources[0].disconnect).toHaveBeenCalledTimes(1) + expect(createdSources[1].disconnect).not.toHaveBeenCalled() + }) +}) diff --git a/packages/stage-ui/src/libs/audio/vad.ts b/packages/stage-ui/src/libs/audio/vad.ts index d581bb7a3..f9bbbc2f2 100644 --- a/packages/stage-ui/src/libs/audio/vad.ts +++ b/packages/stage-ui/src/libs/audio/vad.ts @@ -60,6 +60,7 @@ export function createVADStates(vad: BaseVAD, vadAudioWorkletUrl: string, option let audioWorkletNode: AudioWorkletNode | null let mediaStream: MediaStream | null let sourceNode: MediaStreamAudioSourceNode | null + let silentGainNode: GainNode | null let workletInitialized: boolean const { @@ -96,6 +97,20 @@ export function createVADStates(vad: BaseVAD, vadAudioWorkletUrl: string, option } } + /** + * Disconnects caller-owned microphone graph nodes before rebuilding the input graph. + */ + function disconnectInputGraph() { + if (sourceNode) { + sourceNode.disconnect() + sourceNode = null + } + if (silentGainNode) { + silentGainNode.disconnect() + silentGainNode = null + } + } + async function start(stream: MediaStream) { if (!audioContext || !audioWorkletNode) { throw new Error('Audio system not initialized. Call initialize() first.') @@ -107,6 +122,7 @@ export function createVADStates(vad: BaseVAD, vadAudioWorkletUrl: string, option } // Request microphone access + disconnectInputGraph() mediaStream = stream // Create source node and connect to worklet @@ -115,10 +131,10 @@ export function createVADStates(vad: BaseVAD, vadAudioWorkletUrl: string, option // Connect worklet to a silent destination (to keep the audio graph active) // Using a GainNode with gain=0 to ensure no sound is output - const silentGain = audioContext.createGain() - silentGain.gain.value = 0 - audioWorkletNode.connect(silentGain) - silentGain.connect(audioContext.destination) + silentGainNode = audioContext.createGain() + silentGainNode.gain.value = 0 + audioWorkletNode.connect(silentGainNode) + silentGainNode.connect(audioContext.destination) } catch (error) { console.error('Failed to start microphone:', error) @@ -133,18 +149,14 @@ export function createVADStates(vad: BaseVAD, vadAudioWorkletUrl: string, option } function dispose() { - if (sourceNode) { - sourceNode.disconnect() - sourceNode = null - } + disconnectInputGraph() if (audioWorkletNode) { audioWorkletNode.disconnect() audioWorkletNode = null } - if (mediaStream) { - mediaStream.getTracks().forEach(track => track.stop()) - mediaStream = null - } + // The MediaStream is owned by the caller (settings audio device store). VAD only borrows it + // to build an AudioNode graph, so disposing VAD must not stop the microphone device itself. + mediaStream = null if (audioContext && audioContext.state !== 'closed') { audioContext.close() } diff --git a/packages/stage-ui/src/libs/providers/providers/official/shared.ts b/packages/stage-ui/src/libs/providers/providers/official/shared.ts index ee2021a3b..2b8e7eb47 100644 --- a/packages/stage-ui/src/libs/providers/providers/official/shared.ts +++ b/packages/stage-ui/src/libs/providers/providers/official/shared.ts @@ -17,11 +17,17 @@ export function withCredentials() { const chatSession = getActivePinia() ? useChatSessionStore() : null if (chatSession?.activeSessionId) headers.set('x-airi-session-id', chatSession.activeSessionId) - return globalThis.fetch(input, { + + const requestInit = { ...init, headers, credentials: 'omit', - }) + } as RequestInit & { duplex?: 'half' } + + if (init?.body instanceof ReadableStream) + requestInit.duplex = 'half' + + return globalThis.fetch(input, requestInit) } } diff --git a/packages/stage-ui/src/libs/speech/playback-speaking-state.test.ts b/packages/stage-ui/src/libs/speech/playback-speaking-state.test.ts new file mode 100644 index 000000000..54f8aa5b7 --- /dev/null +++ b/packages/stage-ui/src/libs/speech/playback-speaking-state.test.ts @@ -0,0 +1,89 @@ +import type { + PlaybackEndEvent, + PlaybackInterruptEvent, + PlaybackItem, + PlaybackRejectEvent, + PlaybackStartEvent, +} from '@proj-airi/pipelines-audio' + +import { describe, expect, it } from 'vitest' + +import { bindSpeakingStateToPlaybackManager } from './playback-speaking-state' + +function createPlaybackItem(): PlaybackItem { + return { + id: 'playback-1', + streamId: 'stream-1', + intentId: 'intent-1', + segmentId: 'segment-1', + sequence: 1, + priority: 0, + text: 'hello', + special: null, + audio: {} as AudioBuffer, + createdAt: 1000, + } +} + +function createFakePlaybackManager() { + const listeners = { + start: [] as Array<(event: PlaybackStartEvent) => void>, + end: [] as Array<(event: PlaybackEndEvent) => void>, + interrupt: [] as Array<(event: PlaybackInterruptEvent) => void>, + reject: [] as Array<(event: PlaybackRejectEvent) => void>, + } + + return { + listeners, + manager: { + onStart: (listener: (event: PlaybackStartEvent) => void) => { + listeners.start.push(listener) + }, + onEnd: (listener: (event: PlaybackEndEvent) => void) => { + listeners.end.push(listener) + }, + onInterrupt: (listener: (event: PlaybackInterruptEvent) => void) => { + listeners.interrupt.push(listener) + }, + onReject: (listener: (event: PlaybackRejectEvent) => void) => { + listeners.reject.push(listener) + }, + }, + } +} + +describe('bindSpeakingStateToPlaybackManager', () => { + it('resets speaking state when playback is interrupted', () => { + const playback = createFakePlaybackManager() + let speaking = false + + bindSpeakingStateToPlaybackManager(playback.manager, { + setSpeaking: (value) => { + speaking = value + }, + }) + + const item = createPlaybackItem() + playback.listeners.start.forEach(listener => listener({ item, startedAt: 1000 })) + expect(speaking).toBe(true) + + playback.listeners.interrupt.forEach(listener => listener({ item, reason: 'playback-error', interruptedAt: 1100 })) + expect(speaking).toBe(false) + }) + + it('resets speaking state when playback is rejected before it can finish', () => { + const playback = createFakePlaybackManager() + let speaking = true + + bindSpeakingStateToPlaybackManager(playback.manager, { + setSpeaking: (value) => { + speaking = value + }, + }) + + const item = createPlaybackItem() + playback.listeners.reject.forEach(listener => listener({ item, reason: 'overflow' })) + + expect(speaking).toBe(false) + }) +}) diff --git a/packages/stage-ui/src/libs/speech/playback-speaking-state.ts b/packages/stage-ui/src/libs/speech/playback-speaking-state.ts new file mode 100644 index 000000000..88df4e1ce --- /dev/null +++ b/packages/stage-ui/src/libs/speech/playback-speaking-state.ts @@ -0,0 +1,53 @@ +import type { + PlaybackEndEvent, + PlaybackInterruptEvent, + PlaybackRejectEvent, + PlaybackStartEvent, +} from '@proj-airi/pipelines-audio' + +export interface PlaybackSpeakingStateManager { + onStart: (listener: (event: PlaybackStartEvent) => void) => void + onEnd: (listener: (event: PlaybackEndEvent) => void) => void + onInterrupt: (listener: (event: PlaybackInterruptEvent) => void) => void + onReject: (listener: (event: PlaybackRejectEvent) => void) => void +} + +export interface PlaybackSpeakingStateHandlers { + setSpeaking: (value: boolean) => void + onStart?: (event: PlaybackStartEvent) => void +} + +/** + * Binds assistant speaking state to every terminal playback outcome. + * + * Use when: + * - UI state must show whether assistant audio is currently audible. + * - Voice input should be suspended only while playback is actually active. + * + * Expects: + * - Playback managers emit exactly one terminal event for each accepted item. + * + * Returns: + * - Nothing; listeners are registered on the provided manager. + */ +export function bindSpeakingStateToPlaybackManager( + manager: PlaybackSpeakingStateManager, + handlers: PlaybackSpeakingStateHandlers, +) { + manager.onStart((event) => { + handlers.setSpeaking(true) + handlers.onStart?.(event) + }) + + manager.onEnd(() => { + handlers.setSpeaking(false) + }) + + manager.onInterrupt(() => { + handlers.setSpeaking(false) + }) + + manager.onReject(() => { + handlers.setSpeaking(false) + }) +} diff --git a/packages/stage-ui/src/stores/ai/models/vad.test.ts b/packages/stage-ui/src/stores/ai/models/vad.test.ts index 20fc3cea6..f75706483 100644 --- a/packages/stage-ui/src/stores/ai/models/vad.test.ts +++ b/packages/stage-ui/src/stores/ai/models/vad.test.ts @@ -5,17 +5,21 @@ import { resolveVADConfig } from './vad' describe('resolveVADConfig', () => { it('uses safer defaults for threshold and silence duration', () => { expect(resolveVADConfig()).toEqual({ - speechThreshold: 0.6, - exitThreshold: 0.18, - minSilenceDurationMs: 800, + speechThreshold: 0.52, + exitThreshold: 0.156, + minSilenceDurationMs: 1200, + speechPadMs: 360, + minSpeechDurationMs: 300, }) }) it('preserves explicit threshold and silence duration values', () => { - expect(resolveVADConfig(0.45, 650)).toEqual({ + expect(resolveVADConfig(0.45, 650, 420, 500)).toEqual({ speechThreshold: 0.45, exitThreshold: 0.135, minSilenceDurationMs: 650, + speechPadMs: 420, + minSpeechDurationMs: 500, }) }) }) diff --git a/packages/stage-ui/src/stores/ai/models/vad.ts b/packages/stage-ui/src/stores/ai/models/vad.ts index a6d13571a..73d944243 100644 --- a/packages/stage-ui/src/stores/ai/models/vad.ts +++ b/packages/stage-ui/src/stores/ai/models/vad.ts @@ -11,21 +11,33 @@ import { createVAD, createVADStates } from '../../../workers/vad' interface UseVADOptions { threshold?: MaybeRefOrGetter minSilenceDurationMs?: MaybeRefOrGetter + speechPadMs?: MaybeRefOrGetter + minSpeechDurationMs?: MaybeRefOrGetter onSpeechStart?: () => void onSpeechEnd?: () => void + onSpeechReady?: (event: { buffer: Float32Array, duration: number }) => void } -const DEFAULT_VAD_THRESHOLD = 0.6 -const DEFAULT_VAD_MIN_SILENCE_DURATION_MS = 800 +const DEFAULT_VAD_THRESHOLD = 0.52 +const DEFAULT_VAD_MIN_SILENCE_DURATION_MS = 1200 +const DEFAULT_VAD_SPEECH_PAD_MS = 360 +const DEFAULT_VAD_MIN_SPEECH_DURATION_MS = 300 -export function resolveVADConfig(threshold?: number, minSilenceDurationMs?: number): Pick { +export function resolveVADConfig( + threshold?: number, + minSilenceDurationMs?: number, + speechPadMs?: number, + minSpeechDurationMs?: number, +): Pick { const resolvedThreshold = threshold ?? DEFAULT_VAD_THRESHOLD return { speechThreshold: resolvedThreshold, exitThreshold: resolvedThreshold * 0.3, minSilenceDurationMs: minSilenceDurationMs ?? DEFAULT_VAD_MIN_SILENCE_DURATION_MS, + speechPadMs: speechPadMs ?? DEFAULT_VAD_SPEECH_PAD_MS, + minSpeechDurationMs: minSpeechDurationMs ?? DEFAULT_VAD_MIN_SPEECH_DURATION_MS, } } @@ -33,6 +45,8 @@ export function useVAD(workerUrl: string, options?: UseVADOptions) { const defaultOptions: UseVADOptions = { threshold: ref(DEFAULT_VAD_THRESHOLD), minSilenceDurationMs: ref(DEFAULT_VAD_MIN_SILENCE_DURATION_MS), + speechPadMs: ref(DEFAULT_VAD_SPEECH_PAD_MS), + minSpeechDurationMs: ref(DEFAULT_VAD_MIN_SPEECH_DURATION_MS), } options = merge(defaultOptions, options) @@ -51,6 +65,8 @@ export function useVAD(workerUrl: string, options?: UseVADOptions) { const threshold = toRef(options.threshold) const minSilenceDurationMs = toRef(options.minSilenceDurationMs) + const speechPadMs = toRef(options.speechPadMs) + const minSpeechDurationMs = toRef(options.minSpeechDurationMs) async function init() { if (loaded.value || loading.value || manager.value) @@ -60,7 +76,12 @@ export function useVAD(workerUrl: string, options?: UseVADOptions) { inferenceError.value = '' try { - const vadConfig = resolveVADConfig(threshold.value, minSilenceDurationMs.value) + const vadConfig = resolveVADConfig( + threshold.value, + minSilenceDurationMs.value, + speechPadMs.value, + minSpeechDurationMs.value, + ) vad.value = await createVAD({ sampleRate: 16000, @@ -78,6 +99,10 @@ export function useVAD(workerUrl: string, options?: UseVADOptions) { options?.onSpeechEnd?.() }) + vad.value.on('speech-ready', (event) => { + options?.onSpeechReady?.(event) + }) + vad.value.on('debug', ({ data }) => { if (data?.probability !== undefined) { isSpeechProb.value = data.probability @@ -149,6 +174,18 @@ export function useVAD(workerUrl: string, options?: UseVADOptions) { } }) + watch(speechPadMs, (newVal) => { + if (vad.value && newVal !== undefined) { + vad.value.updateConfig({ speechPadMs: newVal }) + } + }) + + watch(minSpeechDurationMs, (newVal) => { + if (vad.value && newVal !== undefined) { + vad.value.updateConfig({ minSpeechDurationMs: newVal }) + } + }) + return { isSpeech, isSpeechProb, @@ -158,6 +195,8 @@ export function useVAD(workerUrl: string, options?: UseVADOptions) { inferenceError, threshold, minSilenceDurationMs, + speechPadMs, + minSpeechDurationMs, init, start, diff --git a/packages/stage-ui/src/stores/modules/hearing.analytics.test.ts b/packages/stage-ui/src/stores/modules/hearing.analytics.test.ts new file mode 100644 index 000000000..da1847e20 --- /dev/null +++ b/packages/stage-ui/src/stores/modules/hearing.analytics.test.ts @@ -0,0 +1,71 @@ +import { createPinia, setActivePinia } from 'pinia' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const analyticsMock = vi.hoisted(() => ({ + allowComposableCall: true, + trackSttFailed: vi.fn(), + trackSttStarted: vi.fn(), + trackSttSucceeded: vi.fn(), +})) + +vi.mock('../../composables/use-analytics', () => ({ + useAnalytics: () => { + if (!analyticsMock.allowComposableCall) + throw new Error('Must be called at the top of a `setup` function') + + return { + trackSttFailed: analyticsMock.trackSttFailed, + trackSttStarted: analyticsMock.trackSttStarted, + trackSttSucceeded: analyticsMock.trackSttSucceeded, + } + }, +})) + +vi.mock('@xsai/generate-transcription', () => ({ + generateTranscription: vi.fn(async () => ({ text: 'hello' })), +})) + +vi.mock('vue-i18n', () => ({ + useI18n: () => ({ + locale: { value: 'en' }, + t: (_key: string, fallback?: string) => fallback ?? _key, + }), +})) + +describe('useHearingStore analytics lifecycle', () => { + beforeEach(() => { + setActivePinia(createPinia()) + analyticsMock.allowComposableCall = true + analyticsMock.trackSttFailed.mockReset() + analyticsMock.trackSttStarted.mockReset() + analyticsMock.trackSttSucceeded.mockReset() + }) + + /** + * @example + * await hearingStore.transcription(providerId, provider, model, file) + */ + it('does not call analytics composables when a recording is transcribed later', async () => { + const { useHearingStore } = await import('./hearing') + const hearingStore = useHearingStore() + analyticsMock.allowComposableCall = false + + const result = await hearingStore.transcription( + 'openai-compatible-audio-transcription', + { + transcription: () => ({}), + } as any, + 'FunAudioLLM/SenseVoiceSmall', + new File(['hello'], 'recording.wav', { type: 'audio/wav' }), + ) + + expect(result.text).toBe('hello') + expect(analyticsMock.trackSttStarted).toHaveBeenCalledWith('openai-compatible-audio-transcription') + expect(analyticsMock.trackSttSucceeded).toHaveBeenCalledWith({ + provider: 'openai-compatible-audio-transcription', + latency_ms: expect.any(Number), + char_count: 5, + stream: false, + }) + }, 10000) +}) diff --git a/packages/stage-ui/src/stores/modules/hearing.test.ts b/packages/stage-ui/src/stores/modules/hearing.test.ts index 27b5fb72d..a74d40cca 100644 --- a/packages/stage-ui/src/stores/modules/hearing.test.ts +++ b/packages/stage-ui/src/stores/modules/hearing.test.ts @@ -1,6 +1,15 @@ import { describe, expect, it } from 'vitest' -import { filterTranscriptionByConfidence, resolveStreamTranscriptionExecutor } from './hearing' +import { + describeEmptyTranscriptionResponse, + filterTranscriptionByConfidence, + normalizeGeneratedTranscriptionText, + resolveActiveTranscriptionModel, + resolveActiveTranscriptionProviderError, + resolveStreamTranscriptionExecutor, + resolveTranscriptionFileName, + resolveTranscriptionProviderOptions, +} from './hearing' describe('filterTranscriptionByConfidence', () => { const segments = [ @@ -41,3 +50,111 @@ describe('resolveStreamTranscriptionExecutor', () => { expect(executor).toBe(resolveStreamTranscriptionExecutor('aliyun-nls-transcription')) }) }) + +describe('resolveActiveTranscriptionProviderError', () => { + /** + * @example + * resolveActiveTranscriptionProviderError('') + */ + it('returns a clear setup error when no transcription provider is selected', () => { + expect(resolveActiveTranscriptionProviderError('')).toBe('No active transcription provider selected. Select a provider in Settings > Hearing.') + }) + + /** + * @example + * resolveActiveTranscriptionProviderError('openai-compatible-audio-transcription') + */ + it('allows a selected transcription provider', () => { + expect(resolveActiveTranscriptionProviderError('openai-compatible-audio-transcription')).toBeUndefined() + }) +}) + +describe('resolveActiveTranscriptionModel', () => { + /** + * @example + * resolveActiveTranscriptionModel('', { model: 'FunAudioLLM/SenseVoiceSmall' }) + */ + it('uses the provider config model when the hearing model has not been synced', () => { + expect(resolveActiveTranscriptionModel('', { model: 'FunAudioLLM/SenseVoiceSmall' })).toBe('FunAudioLLM/SenseVoiceSmall') + }) + + /** + * @example + * resolveActiveTranscriptionModel('whisper-1', { model: 'FunAudioLLM/SenseVoiceSmall' }) + */ + it('prefers the explicit hearing model over the provider config model', () => { + expect(resolveActiveTranscriptionModel('whisper-1', { model: 'FunAudioLLM/SenseVoiceSmall' })).toBe('whisper-1') + }) +}) + +describe('resolveTranscriptionProviderOptions', () => { + /** + * @example + * resolveTranscriptionProviderOptions({}, 'zh-Hans') + */ + it('derives a two-letter transcription language from the active UI locale', () => { + expect(resolveTranscriptionProviderOptions({}, 'zh-Hans')).toEqual({ language: 'zh' }) + }) + + /** + * @example + * resolveTranscriptionProviderOptions({ language: 'ja' }, 'zh-Hans') + */ + it('prefers the provider language when one is configured explicitly', () => { + expect(resolveTranscriptionProviderOptions({ language: 'ja' }, 'zh-Hans')).toEqual({ language: 'ja' }) + }) +}) + +describe('normalizeGeneratedTranscriptionText', () => { + /** + * @example + * normalizeGeneratedTranscriptionText({ result: { text: '你好' } }) + */ + it('reads nested text from OpenAI-compatible provider variants', () => { + expect(normalizeGeneratedTranscriptionText({ result: { text: '你好' } })).toBe('你好') + }) + + /** + * @example + * normalizeGeneratedTranscriptionText({ segments: [{ text: '你' }, { text: '好' }] }) + */ + it('joins segment text when no top-level text is returned', () => { + expect(normalizeGeneratedTranscriptionText({ segments: [{ text: '你' }, { text: '好' }] })).toBe('你好') + }) + + /** + * @example + * normalizeGeneratedTranscriptionText({ segments: [{ text: ' Hello' }, { text: ' world' }] }) + */ + it('preserves segment whitespace before trimming the final fallback text', () => { + expect(normalizeGeneratedTranscriptionText({ segments: [{ text: ' Hello' }, { text: ' world' }] })).toBe('Hello world') + }) + + /** + * @example + * normalizeGeneratedTranscriptionText({ data: { text: '你好' } }) + */ + it('reads data text from provider envelope responses', () => { + expect(normalizeGeneratedTranscriptionText({ data: { text: '你好' } })).toBe('你好') + }) +}) + +describe('describeEmptyTranscriptionResponse', () => { + /** + * @example + * describeEmptyTranscriptionResponse({ result: { duration: 1 } }) + */ + it('describes response keys when no usable text was returned', () => { + expect(describeEmptyTranscriptionResponse({ result: { duration: 1 } })).toContain('keys=result') + }) +}) + +describe('resolveTranscriptionFileName', () => { + /** + * @example + * resolveTranscriptionFileName(new File([], 'recording.wav')) + */ + it('uses the File name so OpenAI-compatible providers can infer the audio format', () => { + expect(resolveTranscriptionFileName(new File([], 'recording.wav'))).toBe('recording.wav') + }) +}) diff --git a/packages/stage-ui/src/stores/modules/hearing.ts b/packages/stage-ui/src/stores/modules/hearing.ts index 42c4ebf54..df80e4b5a 100644 --- a/packages/stage-ui/src/stores/modules/hearing.ts +++ b/packages/stage-ui/src/stores/modules/hearing.ts @@ -74,6 +74,7 @@ export type HearingTranscriptionResult = HearingTranscriptionGenerateResult | He type HearingTranscriptionInput = File | { file?: File + fileName?: string inputAudioStream?: ReadableStream } @@ -94,6 +95,111 @@ export function filterTranscriptionByConfidence( return segments.filter(s => (s?.avg_logprob ?? -Infinity) >= threshold).map(s => s?.text ?? '').join('').trim() } +/** + * Reads a string field from an unknown response object. + */ +function stringField(value: unknown, key: string, options: { trim?: boolean } = {}) { + if (!value || typeof value !== 'object') + return '' + + const field = (value as Record)[key] + if (typeof field !== 'string') + return '' + + return options.trim === false ? field : field.trim() +} + +/** + * Reads a nested object field from an unknown response object. + */ +function objectField(value: unknown, key: string) { + if (!value || typeof value !== 'object') + return undefined + + const field = (value as Record)[key] + return field && typeof field === 'object' ? field : undefined +} + +/** + * Normalizes generated transcription text from OpenAI-compatible response variants. + * + * Before: + * - `{ result: { text: "你好" } }` + * - `{ segments: [{ text: "你" }, { text: "好" }] }` + * + * After: + * - `"你好"` + */ +export function normalizeGeneratedTranscriptionText(response: unknown) { + const directText = stringField(response, 'text') + if (directText) + return directText + + for (const envelopeKey of ['result', 'data', 'output']) { + const nested = objectField(response, envelopeKey) + const nestedText = stringField(nested, 'text') + if (nestedText) + return nestedText + } + + const segments = objectField(response, 'segments') ?? (response && typeof response === 'object' ? (response as Record).segments : undefined) + if (Array.isArray(segments)) { + const text = segments + .map(segment => stringField(segment, 'text', { trim: false })) + .join('') + .trim() + if (text) + return text + } + + return '' +} + +/** + * Builds a compact diagnostic summary for an empty transcription response. + */ +export function describeEmptyTranscriptionResponse(response: unknown) { + if (!response || typeof response !== 'object') + return `response=${String(response)}` + + const keys = Object.keys(response as Record) + const nestedKeys = keys + .map((key) => { + const nested = objectField(response, key) + return nested ? `${key}.{${Object.keys(nested as Record).join(',')}}` : '' + }) + .filter(Boolean) + + return [ + `keys=${keys.join(',') || '(none)'}`, + ...(nestedKeys.length ? [`nested=${nestedKeys.join(';')}`] : []), + ].join(' ') +} + +/** + * Resolves the upload filename for transcription requests. + * + * Use when: + * - OpenAI-compatible providers infer audio format from multipart filenames. + * + * Expects: + * - `file.name` may carry the recorder-generated extension. + * + * Returns: + * - A stable filename with an audio extension. + */ +export function resolveTranscriptionFileName(file: File, explicitFileName?: string) { + const explicit = explicitFileName?.trim() + if (explicit) + return explicit + + const fileName = file.name.trim() + if (fileName) + return fileName + + return 'recording.wav' +} + const STREAM_TRANSCRIPTION_EXECUTORS: Record = { 'aliyun-nls-transcription': streamAliyunTranscription, [OFFICIAL_TRANSCRIPTION_PROVIDER_ID]: streamAliyunTranscription, @@ -104,9 +210,78 @@ export function resolveStreamTranscriptionExecutor(providerId: string): StreamTr return STREAM_TRANSCRIPTION_EXECUTORS[providerId] } +/** + * Resolves the setup error for the selected transcription provider. + * + * Use when: + * - A speech pipeline entry point needs to fail before provider instantiation. + * - User-facing diagnostics should explain the missing Hearing selection. + * + * Expects: + * - `providerId` is the current `settings/hearing/active-provider` value. + * + * Returns: + * - A setup error when no provider is selected, otherwise `undefined`. + */ +export function resolveActiveTranscriptionProviderError(providerId: string): string | undefined { + if (providerId) + return undefined + + return 'No active transcription provider selected. Select a provider in Settings > Hearing.' +} + +/** + * Resolves the transcription model from Hearing state with provider config fallback. + * + * Use when: + * - OpenAI-compatible transcription stores the model in provider settings. + * - The Hearing module has not yet synchronized that model into its active model state. + * + * Expects: + * - `activeModel` is the current Hearing model value. + * - `providerConfig.model` may contain a provider-scoped model name. + * + * Returns: + * - The explicit Hearing model first, then the provider config model, otherwise an empty string. + */ +export function resolveActiveTranscriptionModel(activeModel: string, providerConfig?: Record) { + const modelFromHearing = activeModel.trim() + if (modelFromHearing) + return modelFromHearing + + const modelFromProviderConfig = typeof providerConfig?.model === 'string' ? providerConfig.model.trim() : '' + return modelFromProviderConfig +} + +/** + * Resolves extra transcription request options from provider config and UI locale. + * + * Use when: + * - Short ASR recordings need a language hint to avoid multilingual auto-detection drift. + * - Provider-specific transcription prompts are configured outside the Hearing active model field. + * + * Expects: + * - `uiLocale` uses a BCP-47-like language tag such as `zh-Hans` or `en-US`. + * + * Returns: + * - OpenAI-compatible transcription options that can be merged into the provider request. + */ +export function resolveTranscriptionProviderOptions(providerConfig?: Record, uiLocale = globalThis.navigator?.language ?? '') { + const configuredLanguage = typeof providerConfig?.language === 'string' ? providerConfig.language.trim() : '' + const localeLanguage = uiLocale.split(/[-_]/)[0]?.trim().toLowerCase() ?? '' + const language = configuredLanguage || localeLanguage + const prompt = typeof providerConfig?.prompt === 'string' ? providerConfig.prompt.trim() : '' + + return { + ...(language ? { language } : {}), + ...(prompt ? { prompt } : {}), + } +} + export const useHearingStore = defineStore('hearing-store', () => { const providersStore = useProvidersStore() const { allAudioTranscriptionProvidersMetadata } = storeToRefs(providersStore) + const { trackSttStarted, trackSttSucceeded, trackSttFailed } = useAnalytics() // State const activeTranscriptionProvider = useLocalStorageManualReset('settings/hearing/active-provider', '') @@ -196,12 +371,12 @@ export const useHearingStore = defineStore('hearing-store', () => { ): Promise { const normalizedInput = (input instanceof File ? { file: input } : input ?? {}) as { file?: File + fileName?: string inputAudioStream?: ReadableStream } const features = providersStore.getTranscriptionFeatures(providerId) const streamExecutor = resolveStreamTranscriptionExecutor(providerId) - const { trackSttStarted, trackSttSucceeded, trackSttFailed } = useAnalytics() const sttStartedAt = performance.now() trackSttStarted(providerId) @@ -275,6 +450,7 @@ export const useHearingStore = defineStore('hearing-store', () => { const response = await generateTranscription({ ...provider.transcription(model, options?.providerOptions), file: normalizedInput.file, + fileName: resolveTranscriptionFileName(normalizedInput.file, normalizedInput.fileName), responseFormat: useVerboseJson ? 'verbose_json' : format, }) @@ -295,11 +471,12 @@ export const useHearingStore = defineStore('hearing-store', () => { } } - const fallbackText = typeof response.text === 'string' ? response.text : '' + const fallbackText = normalizeGeneratedTranscriptionText(response) emitSucceeded(fallbackText.length, false) return { mode: 'generate', ...response, + text: fallbackText, } } catch (err) { @@ -355,6 +532,24 @@ export const useHearingSpeechInputPipeline = defineStore('modules:hearing:speech let asrSpan: Span | undefined + function startStreamingAsrSpan(providerId: string) { + activeTurnSpan.value?.end() + const turnSpan = startSpan(IOSpanNames.InteractionTurn) + activeTurnSpan.value = turnSpan + asrSpan = startSpan(IOSpanNames.SpeechRecognition, turnSpan, { + [IOAttributes.Subsystem]: IOSubsystems.ASR, + [IOAttributes.GenAIRequestModel]: providerId, + }) + } + + function endStreamingAsrSpan() { + if (!asrSpan) + return + + asrSpan.end() + asrSpan = undefined + } + const supportsStreamInput = computed(() => { const providerId = activeTranscriptionProvider.value if (!providerId) @@ -543,14 +738,6 @@ export const useHearingSpeechInputPipeline = defineStore('modules:hearing:speech onSentenceEnd?: (delta: string) => void onSpeechEnd?: (text: string) => void }) { - activeTurnSpan.value?.end() - const turnSpan = startSpan(IOSpanNames.InteractionTurn) - activeTurnSpan.value = turnSpan - asrSpan = startSpan(IOSpanNames.SpeechRecognition, turnSpan, { - [IOAttributes.Subsystem]: IOSubsystems.ASR, - [IOAttributes.GenAIRequestModel]: activeTranscriptionProvider.value ?? '', - }) - console.info('[Hearing Pipeline] transcribeForMediaStream called', { supportsStreamInput: supportsStreamInput.value, hasStream: !!stream, @@ -567,9 +754,10 @@ export const useHearingSpeechInputPipeline = defineStore('modules:hearing:speech try { const providerId = activeTranscriptionProvider.value - if (!providerId) { - error.value = 'No transcription provider selected' - console.error('[Hearing Pipeline] No transcription provider selected') + const providerError = resolveActiveTranscriptionProviderError(providerId) + if (providerError) { + error.value = providerError + console.error('[Hearing Pipeline]', providerError) return } @@ -619,6 +807,8 @@ export const useHearingSpeechInputPipeline = defineStore('modules:hearing:speech } } + startStreamingAsrSpan(providerId) + // Auto-select default model if not selected if (!activeTranscriptionModel.value) { // Try to get models for the provider and select the first one @@ -760,6 +950,8 @@ export const useHearingSpeechInputPipeline = defineStore('modules:hearing:speech } } + startStreamingAsrSpan(providerId) + const abortController = new AbortController() let idleTimer: ReturnType | undefined const bumpIdle = () => { @@ -856,6 +1048,8 @@ export const useHearingSpeechInputPipeline = defineStore('modules:hearing:speech } } catch (err) { + endStreamingAsrSpan() + if (isExpectedStreamStopError(err)) return @@ -867,33 +1061,58 @@ export const useHearingSpeechInputPipeline = defineStore('modules:hearing:speech async function transcribeForRecording(recording: Blob | null | undefined) { error.value = undefined - if (!recording) + if (!recording) { + error.value = 'No recording captured from microphone' return + } + + if (recording.size <= 0) { + error.value = 'Recording captured from microphone is empty' + return + } try { - if (recording && recording.size > 0) { - const providerId = activeTranscriptionProvider.value - const provider = await providersStore.getProviderInstance>(providerId) - if (!provider) { - throw new Error('Failed to initialize speech provider') - } - - // Get model from configuration or use default - const model = activeTranscriptionModel.value - const result = await hearingStore.transcription( - providerId, - provider, - model, - new File([recording], 'recording.wav'), - ) - const text = result.mode === 'stream' ? await result.text : result.text - if (!text || !text.trim()) { - error.value = 'No transcription result returned from provider' - return - } - - return text + const providerId = activeTranscriptionProvider.value + const providerError = resolveActiveTranscriptionProviderError(providerId) + if (providerError) { + error.value = providerError + console.error('[Hearing Pipeline]', providerError) + return } + + const provider = await providersStore.getProviderInstance>(providerId) + if (!provider) { + throw new Error('Failed to initialize speech provider') + } + + const providerConfig = providersStore.getProviderConfig(providerId) + const model = resolveActiveTranscriptionModel(activeTranscriptionModel.value, providerConfig) + const providerOptions = resolveTranscriptionProviderOptions(providerConfig) + console.info('[Hearing Pipeline] Transcribing recording', { + providerId, + language: providerOptions.language, + model, + recordingSize: recording.size, + recordingType: recording.type, + }) + const result = await hearingStore.transcription( + providerId, + provider, + model, + new File([recording], 'recording.wav', { type: recording.type || 'audio/wav' }), + undefined, + { providerOptions }, + ) + const text = result.mode === 'stream' ? await result.text : result.text + if (!text || !text.trim()) { + const responseSummary = result.mode === 'generate' + ? describeEmptyTranscriptionResponse(result) + : 'stream result returned empty text' + error.value = `No transcription result returned from provider (${responseSummary})` + return + } + + return text } catch (err) { error.value = errorMessage(err) diff --git a/packages/stage-ui/src/stores/providers/openai-compatible-builder.test.ts b/packages/stage-ui/src/stores/providers/openai-compatible-builder.test.ts new file mode 100644 index 000000000..14a5cbbca --- /dev/null +++ b/packages/stage-ui/src/stores/providers/openai-compatible-builder.test.ts @@ -0,0 +1,36 @@ +import { describe, expect, it } from 'vitest' + +import { buildOpenAICompatibleProvider } from './openai-compatible-builder' + +describe('buildOpenAICompatibleProvider', () => { + /** + * @example + * provider.transcription('FunAudioLLM/SenseVoiceSmall', { language: 'zh' }) + */ + it('preserves transcription extra options for OpenAI-compatible ASR providers', async () => { + const metadata = buildOpenAICompatibleProvider({ + id: 'test-openai-compatible-transcription', + name: 'Test Transcription', + nameKey: 'test.transcription.title', + description: 'Test transcription provider', + descriptionKey: 'test.transcription.description', + icon: 'i-lobe-icons:openai', + category: 'transcription', + creator: () => ({ + transcription: (model: string) => ({ + baseURL: 'https://example.com/v1/', + model, + }), + }), + }) + + const provider = await metadata.createProvider({}) + + expect('transcription' in provider).toBe(true) + expect((provider as any).transcription('FunAudioLLM/SenseVoiceSmall', { language: 'zh' })).toEqual({ + baseURL: 'https://example.com/v1/', + language: 'zh', + model: 'FunAudioLLM/SenseVoiceSmall', + }) + }) +}) diff --git a/packages/stage-ui/src/stores/providers/openai-compatible-builder.ts b/packages/stage-ui/src/stores/providers/openai-compatible-builder.ts index 838426a85..a75ea690d 100644 --- a/packages/stage-ui/src/stores/providers/openai-compatible-builder.ts +++ b/packages/stage-ui/src/stores/providers/openai-compatible-builder.ts @@ -35,6 +35,26 @@ function logWarn(...args: unknown[]) { console.warn(...args) } +/** + * Wraps transcription providers so OpenAI audio options like `language` and `prompt` are preserved. + */ +function withTranscriptionExtraOptions(provider: unknown) { + if (!provider || typeof provider !== 'object' || !('transcription' in provider)) + return provider + + const transcription = (provider as { transcription?: unknown }).transcription + if (typeof transcription !== 'function') + return provider + + return { + ...provider, + transcription: (model: string, extraOptions?: Record) => ({ + ...transcription(model), + ...extraOptions, + }), + } +} + export function buildOpenAICompatibleProvider( options: Partial & { id: string @@ -270,7 +290,11 @@ export function buildOpenAICompatibleProvider( createProvider: async (config: { apiKey: string, baseUrl: string }) => { const apiKey = normalizeString(config.apiKey) const baseUrl = normalizeBaseUrl(config.baseUrl) - return creator(apiKey, baseUrl) + const provider = await creator(apiKey, baseUrl) + if (resolvedCategory === 'transcription') + return withTranscriptionExtraOptions(provider) + + return provider }, capabilities: finalCapabilities, validators: finalValidators, diff --git a/packages/stage-ui/src/stores/settings/audio-device.test.ts b/packages/stage-ui/src/stores/settings/audio-device.test.ts new file mode 100644 index 000000000..fe11ef4cd --- /dev/null +++ b/packages/stage-ui/src/stores/settings/audio-device.test.ts @@ -0,0 +1,136 @@ +import { createTestingPinia } from '@pinia/testing' +import { setActivePinia } from 'pinia' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { nextTick } from 'vue' + +const storageMock = vi.hoisted(() => ({ + values: new Map(), +})) + +const audioDeviceMock = vi.hoisted(() => ({ + audioInputs: { value: [] as MediaDeviceInfo[] }, + selectedAudioInput: { value: '' }, + startStream: vi.fn(), + stopStream: vi.fn(), + askPermission: vi.fn(), +})) + +vi.mock('@proj-airi/stage-shared/composables', async () => { + const vue = await vi.importActual('vue') + + return { + useLocalStorageManualReset: (key: string, initialValue: T) => { + const value = vue.ref((storageMock.values.has(key) ? storageMock.values.get(key) : initialValue) as T) + + storageMock.values.set(key, value.value) + vue.watch(value, (newValue) => { + storageMock.values.set(key, newValue) + }, { flush: 'sync' }) + + return Object.assign(value, { + reset: () => { + value.value = initialValue + }, + }) + }, + } +}) + +vi.mock('../../composables/audio', async () => { + const vue = await vi.importActual('vue') + + return { + useAudioDevice: () => ({ + audioInputs: audioDeviceMock.audioInputs, + deviceConstraints: vue.computed(() => ({ audio: true })), + selectedAudioInput: audioDeviceMock.selectedAudioInput, + startStream: audioDeviceMock.startStream, + stopStream: audioDeviceMock.stopStream, + stream: vue.shallowRef(), + askPermission: audioDeviceMock.askPermission, + }), + } +}) + +function createAudioInput(deviceId: string): MediaDeviceInfo { + return { + deviceId, + groupId: '', + kind: 'audioinput', + label: deviceId, + toJSON: () => ({}), + } +} + +describe('store settings-audio-devices', () => { + beforeEach(() => { + setActivePinia(createTestingPinia({ createSpy: vi.fn, stubActions: false })) + storageMock.values.clear() + audioDeviceMock.audioInputs.value = [] + audioDeviceMock.selectedAudioInput.value = '' + vi.clearAllMocks() + }) + + afterEach(() => { + vi.resetModules() + }) + + it('starts with the persisted microphone instead of overwriting it with the runtime default', async () => { + storageMock.values.set('settings/audio/input', 'microphone-1') + storageMock.values.set('settings/audio/input/enabled', true) + audioDeviceMock.audioInputs.value = [ + createAudioInput('default'), + createAudioInput('microphone-1'), + ] + audioDeviceMock.selectedAudioInput.value = 'default' + + const startedWith: string[] = [] + audioDeviceMock.startStream.mockImplementation(async () => { + startedWith.push(audioDeviceMock.selectedAudioInput.value) + }) + + const { useSettingsAudioDevice } = await import('./audio-device') + const store = useSettingsAudioDevice() + + store.initialize() + await Promise.resolve() + + expect(startedWith).toEqual(['microphone-1']) + expect(store.selectedAudioInput).toBe('microphone-1') + expect(storageMock.values.get('settings/audio/input')).toBe('microphone-1') + }) + + it('ignores stale microphone startup failures after a newer start succeeds', async () => { + const { useSettingsAudioDevice } = await import('./audio-device') + const store = useSettingsAudioDevice() + + let rejectFirstStart!: (error: unknown) => void + let resolveSecondStart!: () => void + audioDeviceMock.startStream + .mockImplementationOnce(() => new Promise((_resolve, reject) => { + rejectFirstStart = reject + })) + .mockImplementationOnce(() => new Promise((resolve) => { + resolveSecondStart = resolve + })) + + store.enabled = true + await nextTick() + + store.enabled = false + await nextTick() + + store.enabled = true + await nextTick() + + resolveSecondStart() + await Promise.resolve() + + rejectFirstStart(new Error('old startup failed')) + await Promise.resolve() + await nextTick() + + expect(store.enabled).toBe(true) + expect(audioDeviceMock.stopStream).toHaveBeenCalledTimes(1) + }) +}) diff --git a/packages/stage-ui/src/stores/settings/audio-device.ts b/packages/stage-ui/src/stores/settings/audio-device.ts index 8ee31f1b9..0d7e0ffac 100644 --- a/packages/stage-ui/src/stores/settings/audio-device.ts +++ b/packages/stage-ui/src/stores/settings/audio-device.ts @@ -7,10 +7,68 @@ import { useAudioDevice } from '../../composables/audio' let microphonePermissionStatus: PermissionStatus export const useSettingsAudioDevice = defineStore('settings-audio-devices', () => { - const { audioInputs, deviceConstraints, selectedAudioInput: selectedAudioInputNonPersist, startStream, stopStream, stream, askPermission } = useAudioDevice() + const { + audioInputs, + deviceConstraints, + selectedAudioInput: selectedAudioInputNonPersist, + startStream: startAudioInputStream, + stopStream: stopAudioInputStream, + stream, + askPermission: askAudioInputPermission, + } = useAudioDevice() const selectedAudioInputPersist = useLocalStorageManualReset('settings/audio/input', selectedAudioInputNonPersist.value) const audioInputEnabled = useLocalStorageManualReset('settings/audio/input/enabled', false) + let audioInputStartGeneration = 0 + + function syncSelectedAudioInputFromRuntime() { + if (selectedAudioInputPersist.value !== selectedAudioInputNonPersist.value) + selectedAudioInputPersist.value = selectedAudioInputNonPersist.value + } + + function syncSelectedAudioInputToRuntime() { + if (selectedAudioInputPersist.value && selectedAudioInputPersist.value !== selectedAudioInputNonPersist.value) + selectedAudioInputNonPersist.value = selectedAudioInputPersist.value + } + + async function askPermission() { + syncSelectedAudioInputToRuntime() + await askAudioInputPermission() + syncSelectedAudioInputFromRuntime() + } + + function createAudioInputStartGeneration() { + audioInputStartGeneration += 1 + return audioInputStartGeneration + } + + function invalidateAudioInputStarts() { + audioInputStartGeneration += 1 + } + + async function startStreamForGeneration(generation: number) { + syncSelectedAudioInputToRuntime() + await startAudioInputStream() + + if (generation === audioInputStartGeneration) + syncSelectedAudioInputFromRuntime() + } + + async function startStream() { + await startStreamForGeneration(createAudioInputStartGeneration()) + } + + function stopStream() { + invalidateAudioInputStarts() + stopAudioInputStream() + } + + function handleStartStreamError(generation: number, error: unknown, message: string) { + console.error(message, error) + + if (generation === audioInputStartGeneration) + audioInputEnabled.value = false + } watch(selectedAudioInputPersist, (newValue) => { selectedAudioInputNonPersist.value = newValue @@ -18,7 +76,10 @@ export const useSettingsAudioDevice = defineStore('settings-audio-devices', () = watch(audioInputEnabled, (val) => { if (val) { - startStream() + const generation = createAudioInputStartGeneration() + startStreamForGeneration(generation).catch((error) => { + handleStartStreamError(generation, error, 'Unable to start audio input stream:') + }) } else { stopStream() @@ -42,8 +103,17 @@ export const useSettingsAudioDevice = defineStore('settings-audio-devices', () = const hasSelectedInput = selectedAudioInputPersist.value && audioInputs.value.some(device => device.deviceId === selectedAudioInputPersist.value) + if (hasSelectedInput) + syncSelectedAudioInputToRuntime() + if (audioInputEnabled.value && hasSelectedInput) { - startStream() + const generation = createAudioInputStartGeneration() + startStreamForGeneration(generation).catch((error) => { + handleStartStreamError(generation, error, 'Unable to initialize audio input stream:') + }) + } + else if (selectedAudioInputPersist.value && audioInputs.value.length > 0 && !hasSelectedInput) { + selectedAudioInputPersist.value = selectedAudioInputNonPersist.value } if (selectedAudioInputNonPersist.value && !audioInputEnabled.value) { selectedAudioInputPersist.value = selectedAudioInputNonPersist.value