From bcf7742862c1dd7b3366a0f9b5426cad5707a014 Mon Sep 17 00:00:00 2001 From: Ryanba <92616678+Gujiassh@users.noreply.github.com> Date: Sun, 26 Jul 2026 21:08:57 +0800 Subject: [PATCH] fix(stage-ui): preserve VAD speech padding in chat input (#2095) --- .../audio/voice-input-session.test.ts | 128 ++++++++++++++++-- .../composables/audio/voice-input-session.ts | 33 ++++- 2 files changed, 145 insertions(+), 16 deletions(-) diff --git a/packages/stage-ui/src/composables/audio/voice-input-session.test.ts b/packages/stage-ui/src/composables/audio/voice-input-session.test.ts index a79c59bab..b999e9037 100644 --- a/packages/stage-ui/src/composables/audio/voice-input-session.test.ts +++ b/packages/stage-ui/src/composables/audio/voice-input-session.test.ts @@ -5,6 +5,20 @@ const audioRecorderMock = vi.hoisted(() => ({ isRecording: undefined as unknown as { value: boolean }, startRecord: vi.fn(), stopRecord: vi.fn(), + onStopRecordHook: undefined as ((recording: Blob | undefined) => Promise) | undefined, +})) + +const vadMock = vi.hoisted(() => ({ + options: undefined as { + onSpeechStart?: () => void + onSpeechEnd?: () => void + onSpeechReady?: (event: { buffer: Float32Array, duration: number }) => void + minSilenceDurationMs?: number + } | undefined, +})) + +const hearingPipelineMock = vi.hoisted(() => ({ + transcribeForRecording: vi.fn(async (_recording: Blob | null | undefined) => ''), })) vi.mock('../../workers/vad/process.worklet?worker&url', () => ({ @@ -15,22 +29,26 @@ vi.mock('../../stores/ai/models/vad', async () => { const vue = await vi.importActual('vue') return { - useVAD: () => ({ - init: vi.fn(), - dispose: vi.fn(), - start: vi.fn(), - loaded: vue.ref(true), - isSpeech: vue.ref(false), - isSpeechProb: vue.ref(0), - isSpeechHistory: vue.ref([]), - inferenceError: vue.ref(), - }), + useVAD: (_workerUrl: string, options: typeof vadMock.options) => { + vadMock.options = options + return { + init: vi.fn(), + dispose: vi.fn(), + start: vi.fn(), + loaded: vue.ref(true), + isSpeech: vue.ref(false), + isSpeechProb: vue.ref(0), + isSpeechHistory: vue.ref([]), + inferenceError: vue.ref(), + minSilenceDurationMs: vue.toRef(options?.minSilenceDurationMs ?? 1200), + } + }, } }) vi.mock('../../stores/modules/hearing', () => ({ useHearingSpeechInputPipeline: () => ({ - transcribeForRecording: vi.fn(async () => ''), + transcribeForRecording: hearingPipelineMock.transcribeForRecording, }), })) @@ -43,7 +61,10 @@ vi.mock('./audio-recorder', async () => { isRecording: audioRecorderMock.isRecording, startRecord: audioRecorderMock.startRecord, stopRecord: audioRecorderMock.stopRecord, - onStopRecord: vi.fn(), + onStopRecord: vi.fn((hook: (recording: Blob | undefined) => Promise) => { + audioRecorderMock.onStopRecordHook = hook + return vi.fn() + }), }), } }) @@ -57,9 +78,85 @@ function createMediaStream() { describe('useVoiceInputSession', () => { afterEach(() => { audioRecorderMock.isRecording.value = false + audioRecorderMock.onStopRecordHook = undefined + vadMock.options = undefined vi.useRealTimers() vi.unstubAllGlobals() vi.clearAllMocks() + audioRecorderMock.startRecord.mockReset() + audioRecorderMock.stopRecord.mockReset() + hearingPipelineMock.transcribeForRecording.mockReset().mockResolvedValue('') + }) + + // https://github.com/moeru-ai/airi/issues/2092 + it('issue #2092 transcribes the padded VAD buffer instead of the recorder-only segment', async () => { + const { useVoiceInputSession } = await import('./voice-input-session') + const recorderRecording = new Blob(['recorder-only'], { type: 'audio/wav' }) + const onTranscriptionResult = vi.fn() + + hearingPipelineMock.transcribeForRecording.mockResolvedValueOnce('transcribed') + audioRecorderMock.startRecord.mockImplementation(async () => { + audioRecorderMock.isRecording.value = true + }) + audioRecorderMock.stopRecord.mockImplementation(async () => { + audioRecorderMock.isRecording.value = false + await audioRecorderMock.onStopRecordHook?.(recorderRecording) + }) + + const session = useVoiceInputSession(shallowRef(createMediaStream()), { + volumeFallback: { enabled: false }, + onTranscriptionResult, + }) + + await expect(session.startSegment('vad')).resolves.toBe(true) + + vadMock.options?.onSpeechEnd?.() + vadMock.options?.onSpeechReady?.({ + buffer: new Float32Array([0.5, -0.5]), + duration: 0.125, + }) + + await vi.waitFor(() => expect(hearingPipelineMock.transcribeForRecording).toHaveBeenCalledOnce()) + + const recording = hearingPipelineMock.transcribeForRecording.mock.calls[0]?.[0] + expect(recording).toBeInstanceOf(Blob) + expect(recording).not.toBe(recorderRecording) + expect(recording).toMatchObject({ size: 48, type: 'audio/wav' }) + + const wav = new DataView(await recording!.arrayBuffer()) + expect(String.fromCharCode(...new Uint8Array(wav.buffer, 0, 4))).toBe('RIFF') + expect(wav.getUint32(24, true)).toBe(16000) + expect(wav.getInt16(44, true)).toBe(16383) + expect(wav.getInt16(46, true)).toBe(-16384) + expect(onTranscriptionResult).toHaveBeenCalledWith(expect.objectContaining({ + trigger: 'vad', + recording, + text: 'transcribed', + })) + }) + + // https://github.com/moeru-ai/airi/issues/2092 + it.each(['manual', 'volume'] as const)('issue #2092 keeps %s segments on the recorder-provided audio', async (trigger) => { + const { useVoiceInputSession } = await import('./voice-input-session') + const recorderRecording = new Blob(['manual'], { type: 'audio/wav' }) + + audioRecorderMock.startRecord.mockImplementation(async () => { + audioRecorderMock.isRecording.value = true + }) + audioRecorderMock.stopRecord.mockImplementation(async () => { + audioRecorderMock.isRecording.value = false + await audioRecorderMock.onStopRecordHook?.(recorderRecording) + }) + + const session = useVoiceInputSession(shallowRef(createMediaStream()), { + volumeFallback: { enabled: false }, + }) + + await expect(session.startSegment(trigger)).resolves.toBe(true) + await expect(session.stopSegment(trigger)).resolves.toBeUndefined() + await vi.waitFor(() => expect(hearingPipelineMock.transcribeForRecording).toHaveBeenCalledOnce()) + + expect(hearingPipelineMock.transcribeForRecording).toHaveBeenCalledWith(recorderRecording) }) it('clears the active recorder segment when discarding fails during stop', async () => { @@ -272,6 +369,7 @@ describe('useVoiceInputSession', () => { const { useVoiceInputSession } = await import('./voice-input-session') const session = useVoiceInputSession(shallowRef(createMediaStream()), { + vad: { minSilenceDurationMs: 20 }, volumeFallback: { enabled: true, stopDelayMs: 10, @@ -286,6 +384,12 @@ describe('useVoiceInputSession', () => { animationFrames.shift()?.(1011) await Promise.resolve() + expect(stopRecord).not.toHaveBeenCalled() + + vi.setSystemTime(1031) + animationFrames.shift()?.(1031) + await Promise.resolve() + expect(stopRecord).toHaveBeenCalledOnce() expect(session.activeRecordingTrigger.value).toBeUndefined() }) diff --git a/packages/stage-ui/src/composables/audio/voice-input-session.ts b/packages/stage-ui/src/composables/audio/voice-input-session.ts index db8ea2e18..4538da6cd 100644 --- a/packages/stage-ui/src/composables/audio/voice-input-session.ts +++ b/packages/stage-ui/src/composables/audio/voice-input-session.ts @@ -3,6 +3,7 @@ import type { MaybeRefOrGetter } from 'vue' import type { VoiceInputRecordingSegment, VoiceInputSessionTrigger } from './voice-input-segment' import type { VoiceInputTranscriptionTicket } from './voice-input-transcription-chain' +import { toWav } from '@proj-airi/audio/encoding' import { computed, ref, shallowRef, toRef } from 'vue' import workletUrl from '../../workers/vad/process.worklet?worker&url' @@ -77,6 +78,7 @@ const DEFAULT_VOLUME_FALLBACK_STOP_THRESHOLD = 6 const DEFAULT_VOLUME_FALLBACK_START_FRAMES = 4 const DEFAULT_VOLUME_FALLBACK_STOP_DELAY_MS = 900 const DEFAULT_VOLUME_FALLBACK_LOG_INTERVAL_MS = 2000 +const VAD_SAMPLE_RATE = 16000 function calculateTimeDomainVolumeLevel(dataArray: Uint8Array) { let sum = 0 @@ -93,7 +95,7 @@ function calculateTimeDomainVolumeLevel(dataArray: Uint8Array) { * * Owns: * - recorder-backed segment creation - * - VAD-triggered auto segmentation + * - VAD-buffer-backed auto segmentation with recorder fallback * - volume-triggered fallback segmentation * - record-then-transcribe ASR calls * @@ -117,6 +119,7 @@ export function useVoiceInputSession( const lastError = ref() const transcriptionChain = createVoiceInputTranscriptionChain() const stoppedRecordingSegments: VoiceInputRecordingSegment[] = [] + const vadRecordings = new Map() let nextRecordingSegmentId = 0 let discardNextRecording = false let activeTranscriptionCount = 0 @@ -130,6 +133,7 @@ export function useVoiceInputSession( isSpeechProb, isSpeechHistory, inferenceError: vadError, + minSilenceDurationMs: vadMinSilenceDurationMs, } = useVAD(workletUrl, { threshold: options.vad?.threshold, minSilenceDurationMs: options.vad?.minSilenceDurationMs, @@ -141,6 +145,15 @@ export function useVoiceInputSession( onSpeechEnd: () => { void stopSegment('vad') }, + onSpeechReady: ({ buffer }) => { + const segment = activeRecordingSegment.value + if (!segment || segment.trigger !== 'vad') + return + + vadRecordings.set(segment.id, new Blob([ + toWav(buffer.slice().buffer, VAD_SAMPLE_RATE), + ], { type: 'audio/wav' })) + }, }) let volumeFallbackAudioContext: AudioContext | undefined @@ -187,6 +200,7 @@ export function useVoiceInputSession( } finally { discardNextRecording = false + vadRecordings.delete(segment.id) activeRecordingSegment.value = resolveActiveVoiceInputRecordingSegmentAfterStop(activeRecordingSegment.value, segment) } } @@ -292,6 +306,7 @@ export function useVoiceInputSession( const queuedIndex = stoppedRecordingSegments.findIndex(item => item.id === stoppedSegment.id) if (queuedIndex !== -1) stoppedRecordingSegments.splice(queuedIndex, 1) + vadRecordings.delete(stoppedSegment.id) lastError.value = error log('error', 'segment-stop-failed', 'Failed to stop recorder-backed voice input segment.', { trigger, error }) await options.onTranscriptionError?.({ trigger, error }) @@ -390,8 +405,13 @@ export function useVoiceInputSession( const segment = stoppedRecordingSegments.shift() const trigger = segment?.trigger ?? activeRecordingTrigger.value ?? 'manual' + const recordingForTranscription = segment + ? vadRecordings.get(segment.id) ?? recording + : recording + if (segment) + vadRecordings.delete(segment.id) await transcriptionChain - .enqueue(ticket => processRecording(recording, trigger, ticket)) + .enqueue(ticket => processRecording(recordingForTranscription, trigger, ticket)) .catch((error) => { lastError.value = error log('error', 'recording-processing-error', 'Voice input recording processing failed.', { trigger, error }) @@ -497,18 +517,22 @@ export function useVoiceInputSession( } } else if (activeRecordingTrigger.value === 'volume' || activeRecordingTrigger.value === 'vad') { + const requiredSilenceMs = activeRecordingTrigger.value === 'vad' + ? (vadMinSilenceDurationMs.value ?? 0) + stopDelayMs + : stopDelayMs + if (level > stopThreshold) { volumeFallbackLastSpeechAt = now } else if (!volumeFallbackLastSpeechAt) { volumeFallbackLastSpeechAt = now } - else if (volumeFallbackLastSpeechAt && now - volumeFallbackLastSpeechAt >= stopDelayMs) { + else if (volumeFallbackLastSpeechAt && now - volumeFallbackLastSpeechAt >= requiredSilenceMs) { const trigger = activeRecordingTrigger.value volumeFallbackLastSpeechAt = 0 log('info', 'volume-fallback-speech-end', 'Volume fallback detected silence; finalizing recorder segment.', { level: Number(level.toFixed(1)), - silenceMs: stopDelayMs, + silenceMs: requiredSilenceMs, trigger, }) void stopSegment(trigger) @@ -548,6 +572,7 @@ export function useVoiceInputSession( disposeVAD() transcriptionChain.reset() stoppedRecordingSegments.length = 0 + vadRecordings.clear() if (options.flushActiveRecording && isRecording.value) { await stopSegment(activeRecordingTrigger.value ?? 'manual')