nanakura/moeka-project
1
0
Fork 0
Code Issues Pull Requests 1 Actions 1 Packages Projects Releases Wiki Activity

fix(stage-ui): preserve VAD speech padding in chat input (#2095)

Browse Source
This commit is contained in:
Ryanba
2026-07-26 21:08:57 +08:00
committed by GitHub
parent f414d165de
commit bcf7742862
2 changed files with 145 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/stage-ui/src/composables/audio/voice-input-session.test.ts
+116 -12
View File
@@ -5,6 +5,20 @@ const audioRecorderMock = vi.hoisted(() => ({
isRecording: undefined as unknown as { value: boolean },
startRecord: vi.fn(),
stopRecord: vi.fn(),
onStopRecordHook: undefined as ((recording: Blob | undefined) => Promise<void>) | undefined,
}))
const vadMock = vi.hoisted(() => ({
options: undefined as {
onSpeechStart?: () => void
onSpeechEnd?: () => void
onSpeechReady?: (event: { buffer: Float32Array, duration: number }) => void
minSilenceDurationMs?: number
} | undefined,
}))
const hearingPipelineMock = vi.hoisted(() => ({
transcribeForRecording: vi.fn(async (_recording: Blob | null | undefined) => ''),
}))
vi.mock('../../workers/vad/process.worklet?worker&url', () => ({
@@ -15,22 +29,26 @@ vi.mock('../../stores/ai/models/vad', async () => {
const vue = await vi.importActual<typeof import('vue')>('vue')
return {
useVAD: () => ({
init: vi.fn(),
dispose: vi.fn(),
start: vi.fn(),
loaded: vue.ref(true),
isSpeech: vue.ref(false),
isSpeechProb: vue.ref(0),
isSpeechHistory: vue.ref([]),
inferenceError: vue.ref(),
}),
useVAD: (_workerUrl: string, options: typeof vadMock.options) => {
vadMock.options = options
return {
init: vi.fn(),
dispose: vi.fn(),
start: vi.fn(),
loaded: vue.ref(true),
isSpeech: vue.ref(false),
isSpeechProb: vue.ref(0),
isSpeechHistory: vue.ref([]),
inferenceError: vue.ref(),
minSilenceDurationMs: vue.toRef(options?.minSilenceDurationMs ?? 1200),
}
},
}
})
vi.mock('../../stores/modules/hearing', () => ({
useHearingSpeechInputPipeline: () => ({
transcribeForRecording: vi.fn(async () => ''),
transcribeForRecording: hearingPipelineMock.transcribeForRecording,
}),
}))
@@ -43,7 +61,10 @@ vi.mock('./audio-recorder', async () => {
isRecording: audioRecorderMock.isRecording,
startRecord: audioRecorderMock.startRecord,
stopRecord: audioRecorderMock.stopRecord,
onStopRecord: vi.fn(),
onStopRecord: vi.fn((hook: (recording: Blob | undefined) => Promise<void>) => {
audioRecorderMock.onStopRecordHook = hook
return vi.fn()
}),
}),
}
})
@@ -57,9 +78,85 @@ function createMediaStream() {
describe('useVoiceInputSession', () => {
afterEach(() => {
audioRecorderMock.isRecording.value = false
audioRecorderMock.onStopRecordHook = undefined
vadMock.options = undefined
vi.useRealTimers()
vi.unstubAllGlobals()
vi.clearAllMocks()
audioRecorderMock.startRecord.mockReset()
audioRecorderMock.stopRecord.mockReset()
hearingPipelineMock.transcribeForRecording.mockReset().mockResolvedValue('')
})
// https://github.com/moeru-ai/airi/issues/2092
it('issue #2092 transcribes the padded VAD buffer instead of the recorder-only segment', async () => {
const { useVoiceInputSession } = await import('./voice-input-session')
const recorderRecording = new Blob(['recorder-only'], { type: 'audio/wav' })
const onTranscriptionResult = vi.fn()
hearingPipelineMock.transcribeForRecording.mockResolvedValueOnce('transcribed')
audioRecorderMock.startRecord.mockImplementation(async () => {
audioRecorderMock.isRecording.value = true
})
audioRecorderMock.stopRecord.mockImplementation(async () => {
audioRecorderMock.isRecording.value = false
await audioRecorderMock.onStopRecordHook?.(recorderRecording)
})
const session = useVoiceInputSession(shallowRef(createMediaStream()), {
volumeFallback: { enabled: false },
onTranscriptionResult,
})
await expect(session.startSegment('vad')).resolves.toBe(true)
vadMock.options?.onSpeechEnd?.()
vadMock.options?.onSpeechReady?.({
buffer: new Float32Array([0.5, -0.5]),
duration: 0.125,
})
await vi.waitFor(() => expect(hearingPipelineMock.transcribeForRecording).toHaveBeenCalledOnce())
const recording = hearingPipelineMock.transcribeForRecording.mock.calls[0]?.[0]
expect(recording).toBeInstanceOf(Blob)
expect(recording).not.toBe(recorderRecording)
expect(recording).toMatchObject({ size: 48, type: 'audio/wav' })
const wav = new DataView(await recording!.arrayBuffer())
expect(String.fromCharCode(...new Uint8Array(wav.buffer, 0, 4))).toBe('RIFF')
expect(wav.getUint32(24, true)).toBe(16000)
expect(wav.getInt16(44, true)).toBe(16383)
expect(wav.getInt16(46, true)).toBe(-16384)
expect(onTranscriptionResult).toHaveBeenCalledWith(expect.objectContaining({
trigger: 'vad',
recording,
text: 'transcribed',
}))
})
// https://github.com/moeru-ai/airi/issues/2092
it.each(['manual', 'volume'] as const)('issue #2092 keeps %s segments on the recorder-provided audio', async (trigger) => {
const { useVoiceInputSession } = await import('./voice-input-session')
const recorderRecording = new Blob(['manual'], { type: 'audio/wav' })
audioRecorderMock.startRecord.mockImplementation(async () => {
audioRecorderMock.isRecording.value = true
})
audioRecorderMock.stopRecord.mockImplementation(async () => {
audioRecorderMock.isRecording.value = false
await audioRecorderMock.onStopRecordHook?.(recorderRecording)
})
const session = useVoiceInputSession(shallowRef(createMediaStream()), {
volumeFallback: { enabled: false },
})
await expect(session.startSegment(trigger)).resolves.toBe(true)
await expect(session.stopSegment(trigger)).resolves.toBeUndefined()
await vi.waitFor(() => expect(hearingPipelineMock.transcribeForRecording).toHaveBeenCalledOnce())
expect(hearingPipelineMock.transcribeForRecording).toHaveBeenCalledWith(recorderRecording)
})
it('clears the active recorder segment when discarding fails during stop', async () => {
@@ -272,6 +369,7 @@ describe('useVoiceInputSession', () => {
const { useVoiceInputSession } = await import('./voice-input-session')
const session = useVoiceInputSession(shallowRef(createMediaStream()), {
vad: { minSilenceDurationMs: 20 },
volumeFallback: {
enabled: true,
stopDelayMs: 10,
@@ -286,6 +384,12 @@ describe('useVoiceInputSession', () => {
animationFrames.shift()?.(1011)
await Promise.resolve()
expect(stopRecord).not.toHaveBeenCalled()
vi.setSystemTime(1031)
animationFrames.shift()?.(1031)
await Promise.resolve()
expect(stopRecord).toHaveBeenCalledOnce()
expect(session.activeRecordingTrigger.value).toBeUndefined()
})
packages/stage-ui/src/composables/audio/voice-input-session.ts
+29 -4
View File
@@ -3,6 +3,7 @@ import type { MaybeRefOrGetter } from 'vue'
import type { VoiceInputRecordingSegment, VoiceInputSessionTrigger } from './voice-input-segment'
import type { VoiceInputTranscriptionTicket } from './voice-input-transcription-chain'
import { toWav } from '@proj-airi/audio/encoding'
import { computed, ref, shallowRef, toRef } from 'vue'
import workletUrl from '../../workers/vad/process.worklet?worker&url'
@@ -77,6 +78,7 @@ const DEFAULT_VOLUME_FALLBACK_STOP_THRESHOLD = 6
const DEFAULT_VOLUME_FALLBACK_START_FRAMES = 4
const DEFAULT_VOLUME_FALLBACK_STOP_DELAY_MS = 900
const DEFAULT_VOLUME_FALLBACK_LOG_INTERVAL_MS = 2000
const VAD_SAMPLE_RATE = 16000
function calculateTimeDomainVolumeLevel(dataArray: Uint8Array<ArrayBuffer>) {
let sum = 0
@@ -93,7 +95,7 @@ function calculateTimeDomainVolumeLevel(dataArray: Uint8Array<ArrayBuffer>) {
*
* Owns:
* - recorder-backed segment creation
* - VAD-triggered auto segmentation
* - VAD-buffer-backed auto segmentation with recorder fallback
* - volume-triggered fallback segmentation
* - record-then-transcribe ASR calls
*
@@ -117,6 +119,7 @@ export function useVoiceInputSession(
const lastError = ref<unknown>()
const transcriptionChain = createVoiceInputTranscriptionChain()
const stoppedRecordingSegments: VoiceInputRecordingSegment[] = []
const vadRecordings = new Map<number, Blob>()
let nextRecordingSegmentId = 0
let discardNextRecording = false
let activeTranscriptionCount = 0
@@ -130,6 +133,7 @@ export function useVoiceInputSession(
isSpeechProb,
isSpeechHistory,
inferenceError: vadError,
minSilenceDurationMs: vadMinSilenceDurationMs,
} = useVAD(workletUrl, {
threshold: options.vad?.threshold,
minSilenceDurationMs: options.vad?.minSilenceDurationMs,
@@ -141,6 +145,15 @@ export function useVoiceInputSession(
onSpeechEnd: () => {
void stopSegment('vad')
},
onSpeechReady: ({ buffer }) => {
const segment = activeRecordingSegment.value
if (!segment || segment.trigger !== 'vad')
return
vadRecordings.set(segment.id, new Blob([
toWav(buffer.slice().buffer, VAD_SAMPLE_RATE),
], { type: 'audio/wav' }))
},
})
let volumeFallbackAudioContext: AudioContext | undefined
@@ -187,6 +200,7 @@ export function useVoiceInputSession(
}
finally {
discardNextRecording = false
vadRecordings.delete(segment.id)
activeRecordingSegment.value = resolveActiveVoiceInputRecordingSegmentAfterStop(activeRecordingSegment.value, segment)
}
}
@@ -292,6 +306,7 @@ export function useVoiceInputSession(
const queuedIndex = stoppedRecordingSegments.findIndex(item => item.id === stoppedSegment.id)
if (queuedIndex !== -1)
stoppedRecordingSegments.splice(queuedIndex, 1)
vadRecordings.delete(stoppedSegment.id)
lastError.value = error
log('error', 'segment-stop-failed', 'Failed to stop recorder-backed voice input segment.', { trigger, error })
await options.onTranscriptionError?.({ trigger, error })
@@ -390,8 +405,13 @@ export function useVoiceInputSession(
const segment = stoppedRecordingSegments.shift()
const trigger = segment?.trigger ?? activeRecordingTrigger.value ?? 'manual'
const recordingForTranscription = segment
? vadRecordings.get(segment.id) ?? recording
: recording
if (segment)
vadRecordings.delete(segment.id)
await transcriptionChain
.enqueue(ticket => processRecording(recording, trigger, ticket))
.enqueue(ticket => processRecording(recordingForTranscription, trigger, ticket))
.catch((error) => {
lastError.value = error
log('error', 'recording-processing-error', 'Voice input recording processing failed.', { trigger, error })
@@ -497,18 +517,22 @@ export function useVoiceInputSession(
}
}
else if (activeRecordingTrigger.value === 'volume' || activeRecordingTrigger.value === 'vad') {
const requiredSilenceMs = activeRecordingTrigger.value === 'vad'
? (vadMinSilenceDurationMs.value ?? 0) + stopDelayMs
: stopDelayMs
if (level > stopThreshold) {
volumeFallbackLastSpeechAt = now
}
else if (!volumeFallbackLastSpeechAt) {
volumeFallbackLastSpeechAt = now
}
else if (volumeFallbackLastSpeechAt && now - volumeFallbackLastSpeechAt >= stopDelayMs) {
else if (volumeFallbackLastSpeechAt && now - volumeFallbackLastSpeechAt >= requiredSilenceMs) {
const trigger = activeRecordingTrigger.value
volumeFallbackLastSpeechAt = 0
log('info', 'volume-fallback-speech-end', 'Volume fallback detected silence; finalizing recorder segment.', {
level: Number(level.toFixed(1)),
silenceMs: stopDelayMs,
silenceMs: requiredSilenceMs,
trigger,
})
void stopSegment(trigger)
@@ -548,6 +572,7 @@ export function useVoiceInputSession(
disposeVAD()
transcriptionChain.reset()
stoppedRecordingSegments.length = 0
vadRecordings.clear()
if (options.flushActiveRecording && isRecording.value) {
await stopSegment(activeRecordingTrigger.value ?? 'manual')