refactor(stage-tamagotchi,stage-ui): improve input transcription pipeline (#2001)
--------- Co-authored-by-agent: Codex <codex@openai.com>
This commit is contained in:
Lovehsigure_520
@@ -3,7 +3,7 @@ import workletUrl from '@proj-airi/stage-ui/workers/vad/process.worklet?worker&u
|
||||
|
||||
import { errorMessageFromValue } from '@proj-airi/stage-shared'
|
||||
import { Alert, ErrorContainer, LevelMeter, RadioCardManySelect, RadioCardSimple, TestDummyMarker, ThresholdMeter, TimeSeriesChart } from '@proj-airi/stage-ui/components'
|
||||
import { useAnalytics, useAudioAnalyzer, useAudioRecorder } from '@proj-airi/stage-ui/composables'
|
||||
import { useAnalytics, useAudioAnalyzer, useAudioRecorder, useVoiceInputSession } from '@proj-airi/stage-ui/composables'
|
||||
import { useVAD } from '@proj-airi/stage-ui/stores/ai/models/vad'
|
||||
import { useAudioContext } from '@proj-airi/stage-ui/stores/audio'
|
||||
import { CONFIDENCE_THRESHOLD_DISABLED, useHearingSpeechInputPipeline, useHearingStore } from '@proj-airi/stage-ui/stores/modules/hearing'
|
||||
@@ -81,12 +81,67 @@ const useVADThreshold = ref(0.6) // 0.1 - 0.9
|
||||
const useVADMinSilenceDurationMs = ref(800)
|
||||
const useVADModel = ref(true) // Toggle between VAD and volume-based detection
|
||||
const shouldUseStreamInput = computed(() => supportsStreamInput.value && !!stream.value)
|
||||
let sttTestStopTimer: ReturnType<typeof setTimeout> | undefined
|
||||
|
||||
const sttTestVoiceInputSession = useVoiceInputSession(stream, {
|
||||
shouldUseStreamInput,
|
||||
// Manual Settings tests own their 3s recording window; automatic volume segmentation
|
||||
// would race that timer and make provider diagnostics harder to interpret.
|
||||
volumeFallback: {
|
||||
enabled: false,
|
||||
},
|
||||
onSegmentStart: () => {
|
||||
testStatusMessage.value = 'Recording audio for transcription... (3 seconds)'
|
||||
},
|
||||
onTranscriptionStart: () => {
|
||||
testStatusMessage.value = 'Transcribing recording...'
|
||||
isTranscribing.value = true
|
||||
},
|
||||
onTranscriptionResult: ({ text }) => {
|
||||
testTranscriptionText.value = text
|
||||
testStatusMessage.value = 'Transcription complete!'
|
||||
isTranscribing.value = false
|
||||
isTestingSTT.value = false
|
||||
console.info('STT test transcription result:', text)
|
||||
},
|
||||
onTranscriptionEmpty: () => {
|
||||
testTranscriptionError.value = transcriptionPipelineError.value || 'No transcription result returned from provider'
|
||||
testStatusMessage.value = 'Transcription failed'
|
||||
isTranscribing.value = false
|
||||
isTestingSTT.value = false
|
||||
},
|
||||
onRecordingSkipped: ({ gate }) => {
|
||||
testTranscriptionError.value = gate?.reason || transcriptionPipelineError.value || 'No recording captured from microphone'
|
||||
testStatusMessage.value = 'Transcription failed'
|
||||
isTranscribing.value = false
|
||||
isTestingSTT.value = false
|
||||
},
|
||||
onTranscriptionError: ({ error }) => {
|
||||
testTranscriptionError.value = errorMessageFromValue(error)
|
||||
testStatusMessage.value = `Error: ${testTranscriptionError.value}`
|
||||
isTranscribing.value = false
|
||||
isTestingSTT.value = false
|
||||
console.error('STT test transcription error:', error)
|
||||
},
|
||||
})
|
||||
|
||||
async function resetSttTestVoiceInputSession() {
|
||||
if (sttTestStopTimer) {
|
||||
clearTimeout(sttTestStopTimer)
|
||||
sttTestStopTimer = undefined
|
||||
}
|
||||
|
||||
await sttTestVoiceInputSession.stop({ flushActiveRecording: false })
|
||||
}
|
||||
|
||||
function formatVADThreshold(value: number) {
|
||||
return value.toFixed(2)
|
||||
}
|
||||
|
||||
async function handleSpeechStart() {
|
||||
if (isTestingSTT.value)
|
||||
return
|
||||
|
||||
if (shouldUseStreamInput.value && stream.value) {
|
||||
// Use both callbacks to support incremental updates and final transcript replacement.
|
||||
// ChatArea uses only onSentenceEnd to avoid re-adding deleted text.
|
||||
@@ -105,6 +160,9 @@ async function handleSpeechStart() {
|
||||
}
|
||||
|
||||
async function handleSpeechEnd() {
|
||||
if (isTestingSTT.value)
|
||||
return
|
||||
|
||||
if (shouldUseStreamInput.value) {
|
||||
// For streaming providers, keep the session alive; idle timer will handle teardown.
|
||||
return
|
||||
@@ -264,38 +322,12 @@ onStopRecord(async (recording) => {
|
||||
if (shouldUseStreamInput.value)
|
||||
return
|
||||
|
||||
if (isTestingSTT.value)
|
||||
return
|
||||
|
||||
if (!recording || recording.size === 0)
|
||||
return
|
||||
|
||||
// Handle STT test transcription directly here
|
||||
if (isTestingSTT.value) {
|
||||
testStatusMessage.value = 'Transcribing recording...'
|
||||
isTranscribing.value = true
|
||||
|
||||
try {
|
||||
const result = await transcribeForRecording(recording)
|
||||
if (result) {
|
||||
testTranscriptionText.value = result
|
||||
testStatusMessage.value = 'Transcription complete!'
|
||||
console.info('STT test transcription result:', result)
|
||||
}
|
||||
else {
|
||||
testTranscriptionError.value = transcriptionPipelineError.value || 'No transcription result returned from provider'
|
||||
testStatusMessage.value = 'Transcription failed'
|
||||
}
|
||||
}
|
||||
catch (err) {
|
||||
testTranscriptionError.value = errorMessageFromValue(err)
|
||||
testStatusMessage.value = `Error: ${testTranscriptionError.value}`
|
||||
console.error('STT test transcription error:', err)
|
||||
}
|
||||
finally {
|
||||
isTranscribing.value = false
|
||||
isTestingSTT.value = false
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// Normal monitoring mode - add to audios and transcribe
|
||||
audios.value.push(recording)
|
||||
|
||||
@@ -393,12 +425,29 @@ async function startSTTTest() {
|
||||
testStatusMessage.value = 'Recording audio for transcription... (3 seconds)'
|
||||
console.info('Starting STT test with recording-based transcription for provider:', activeTranscriptionProvider.value)
|
||||
|
||||
startRecord()
|
||||
const recordingStarted = await sttTestVoiceInputSession.startSegment('manual')
|
||||
if (!recordingStarted) {
|
||||
if (!testTranscriptionError.value)
|
||||
testStatusMessage.value = 'Recording did not start'
|
||||
isTranscribing.value = false
|
||||
isTestingSTT.value = false
|
||||
return
|
||||
}
|
||||
|
||||
// Wait a bit for recording to start, then stop it after a delay
|
||||
setTimeout(async () => {
|
||||
stopRecord()
|
||||
sttTestStopTimer = setTimeout(async () => {
|
||||
sttTestStopTimer = undefined
|
||||
testStatusMessage.value = 'Processing transcription...'
|
||||
try {
|
||||
await sttTestVoiceInputSession.stopSegment('manual')
|
||||
}
|
||||
catch (err) {
|
||||
testTranscriptionError.value = errorMessageFromValue(err)
|
||||
testStatusMessage.value = `Error: ${testTranscriptionError.value}`
|
||||
isTranscribing.value = false
|
||||
isTestingSTT.value = false
|
||||
console.error('STT test stop timer error:', err)
|
||||
}
|
||||
}, 3000) // Record for 3 seconds
|
||||
}
|
||||
}
|
||||
@@ -422,7 +471,7 @@ async function stopSTTTest() {
|
||||
await stopStreamingTranscription(false, activeTranscriptionProvider.value)
|
||||
}
|
||||
else {
|
||||
stopRecord()
|
||||
await resetSttTestVoiceInputSession()
|
||||
}
|
||||
}
|
||||
catch (err) {
|
||||
@@ -446,9 +495,6 @@ async function stopSTTTest() {
|
||||
}
|
||||
}
|
||||
|
||||
// Note: STT test transcription is now handled directly in onStopRecord handler above
|
||||
// This watch is kept for potential future use but is no longer needed for STT tests
|
||||
|
||||
watch(selectedAudioInput, async () => isMonitoring.value && await setupAudioMonitoring())
|
||||
|
||||
function handleStreamStartError() {
|
||||
|
||||
@@ -36,6 +36,7 @@ import { useSpeechPipelineAnalytics } from '../../composables/use-speech-pipelin
|
||||
import { Emotion, EMOTION_EmotionMotionName_value, EMOTION_VRMExpressionName_value, EmotionThinkMotionName } from '../../constants/emotions'
|
||||
import { getDefaultStreamingModel, getDefinedProvider } from '../../libs/providers/providers'
|
||||
import { OFFICIAL_SPEECH_PROVIDER_ID } from '../../libs/providers/providers/official'
|
||||
import { bindSpeakingStateToPlaybackManager } from '../../libs/speech/playback-speaking-state'
|
||||
import { createStageTtsSession } from '../../libs/speech/tts-session'
|
||||
import { useAudioContext, useSpeakingStore } from '../../stores/audio'
|
||||
import { useBackgroundStore } from '../../stores/background'
|
||||
@@ -519,29 +520,36 @@ speechPipeline.on('onTurnCancel', ({ turnId }) => {
|
||||
streamingControl.cancelTurn(turnId)
|
||||
})
|
||||
|
||||
playbackManager.onEnd(() => {
|
||||
function resetSpeakingState() {
|
||||
nowSpeaking.value = false
|
||||
mouthOpenSize.value = 0
|
||||
})
|
||||
}
|
||||
|
||||
playbackManager.onStart(({ item }) => {
|
||||
nowSpeaking.value = true
|
||||
// NOTICE: postCaption and postPresent may throw errors if the BroadcastChannel is closed
|
||||
// (e.g., when navigating away from the page). We wrap these in try-catch to prevent
|
||||
// breaking playback when the channel is unavailable.
|
||||
assistantCaption.value += ` ${item.text}`
|
||||
try {
|
||||
postCaption({ type: 'caption-assistant', text: item.text })
|
||||
}
|
||||
catch {
|
||||
// BroadcastChannel may be closed - don't break playback
|
||||
}
|
||||
try {
|
||||
postPresent({ type: 'assistant-append', text: item.text })
|
||||
}
|
||||
catch {
|
||||
// BroadcastChannel may be closed - don't break playback
|
||||
}
|
||||
bindSpeakingStateToPlaybackManager(playbackManager, {
|
||||
setSpeaking: (speaking) => {
|
||||
if (!speaking)
|
||||
resetSpeakingState()
|
||||
else
|
||||
nowSpeaking.value = true
|
||||
},
|
||||
onStart: ({ item }) => {
|
||||
// NOTICE: postCaption and postPresent may throw errors if the BroadcastChannel is closed
|
||||
// (e.g., when navigating away from the page). We wrap these in try-catch to prevent
|
||||
// breaking playback when the channel is unavailable.
|
||||
assistantCaption.value += ` ${item.text}`
|
||||
try {
|
||||
postCaption({ type: 'caption-assistant', text: item.text })
|
||||
}
|
||||
catch {
|
||||
// BroadcastChannel may be closed - don't break playback
|
||||
}
|
||||
try {
|
||||
postPresent({ type: 'assistant-append', text: item.text })
|
||||
}
|
||||
catch {
|
||||
// BroadcastChannel may be closed - don't break playback
|
||||
}
|
||||
},
|
||||
})
|
||||
|
||||
function startLipSyncLoop() {
|
||||
|
||||
@@ -0,0 +1,110 @@
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
|
||||
const vueUseMock = vi.hoisted(() => ({
|
||||
audioInputs: undefined as unknown as { value: MediaDeviceInfo[] },
|
||||
ensurePermissions: vi.fn(async () => {}),
|
||||
startUserMediaStream: vi.fn(),
|
||||
stopStream: vi.fn(),
|
||||
}))
|
||||
|
||||
vi.mock('@vueuse/core', async () => {
|
||||
const vue = await vi.importActual<typeof import('vue')>('vue')
|
||||
vueUseMock.audioInputs = vue.ref<MediaDeviceInfo[]>([])
|
||||
|
||||
return {
|
||||
useDevicesList: () => ({
|
||||
audioInputs: vueUseMock.audioInputs,
|
||||
permissionGranted: vue.ref(false),
|
||||
ensurePermissions: vueUseMock.ensurePermissions,
|
||||
}),
|
||||
useUserMedia: ({ constraints }: { constraints: { value: MediaStreamConstraints } }) => ({
|
||||
stream: vue.shallowRef<MediaStream>(),
|
||||
stop: vueUseMock.stopStream,
|
||||
start: () => vueUseMock.startUserMediaStream(constraints.value),
|
||||
}),
|
||||
}
|
||||
})
|
||||
|
||||
function createAudioInput(deviceId: string): MediaDeviceInfo {
|
||||
return {
|
||||
deviceId,
|
||||
groupId: '',
|
||||
kind: 'audioinput',
|
||||
label: deviceId,
|
||||
toJSON: () => ({}),
|
||||
}
|
||||
}
|
||||
|
||||
function createDeviceNotFoundError() {
|
||||
const error = new Error('Requested device not found')
|
||||
error.name = 'NotFoundError'
|
||||
return error
|
||||
}
|
||||
|
||||
describe('useAudioDevice', () => {
|
||||
afterEach(() => {
|
||||
vueUseMock.audioInputs.value = []
|
||||
vi.clearAllMocks()
|
||||
})
|
||||
|
||||
it('recognizes browser device-not-found errors that are not Error instances', async () => {
|
||||
const { isMissingAudioInputDeviceError } = await import('./audio-device')
|
||||
|
||||
expect(isMissingAudioInputDeviceError({ name: 'NotFoundError' })).toBe(true)
|
||||
expect(isMissingAudioInputDeviceError({ message: 'Requested device not found' })).toBe(true)
|
||||
})
|
||||
|
||||
it('retries with the system default microphone when a persisted device id is stale', async () => {
|
||||
const { useAudioDevice } = await import('./audio-device')
|
||||
const { selectedAudioInput, startStream } = useAudioDevice()
|
||||
selectedAudioInput.value = 'stale-device-id'
|
||||
|
||||
vueUseMock.startUserMediaStream
|
||||
.mockRejectedValueOnce(createDeviceNotFoundError())
|
||||
.mockResolvedValueOnce(undefined)
|
||||
|
||||
await startStream()
|
||||
|
||||
expect(selectedAudioInput.value).toBe('')
|
||||
expect(vueUseMock.startUserMediaStream).toHaveBeenNthCalledWith(1, {
|
||||
audio: {
|
||||
autoGainControl: true,
|
||||
deviceId: { exact: 'stale-device-id' },
|
||||
echoCancellation: true,
|
||||
noiseSuppression: true,
|
||||
},
|
||||
})
|
||||
expect(vueUseMock.startUserMediaStream).toHaveBeenNthCalledWith(2, {
|
||||
audio: {
|
||||
autoGainControl: true,
|
||||
echoCancellation: true,
|
||||
noiseSuppression: true,
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
it('prefers an enumerated default input before falling back to unconstrained audio', async () => {
|
||||
vueUseMock.audioInputs.value = [
|
||||
createAudioInput('default'),
|
||||
createAudioInput('microphone-1'),
|
||||
]
|
||||
|
||||
const { useAudioDevice } = await import('./audio-device')
|
||||
const { selectedAudioInput, startStream } = useAudioDevice()
|
||||
selectedAudioInput.value = 'stale-device-id'
|
||||
|
||||
vueUseMock.startUserMediaStream.mockResolvedValueOnce(undefined)
|
||||
|
||||
await startStream()
|
||||
|
||||
expect(selectedAudioInput.value).toBe('default')
|
||||
expect(vueUseMock.startUserMediaStream).toHaveBeenCalledWith({
|
||||
audio: {
|
||||
autoGainControl: true,
|
||||
deviceId: { exact: 'default' },
|
||||
echoCancellation: true,
|
||||
noiseSuppression: true,
|
||||
},
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -1,32 +1,58 @@
|
||||
import { useDevicesList, useUserMedia } from '@vueuse/core'
|
||||
import { computed, nextTick, ref, watch } from 'vue'
|
||||
|
||||
function resolvePreferredAudioInput(audioInputs: MediaDeviceInfo[]) {
|
||||
return audioInputs.find(device => device.deviceId === 'default')?.deviceId || audioInputs[0]?.deviceId || ''
|
||||
}
|
||||
|
||||
export function isMissingAudioInputDeviceError(error: unknown) {
|
||||
if (!error || typeof error !== 'object')
|
||||
return false
|
||||
|
||||
const { message, name } = error as { message?: unknown, name?: unknown }
|
||||
|
||||
return name === 'NotFoundError'
|
||||
|| name === 'OverconstrainedError'
|
||||
|| (typeof message === 'string' && message.includes('Requested device not found'))
|
||||
}
|
||||
|
||||
export function useAudioDevice(requestPermission: boolean = false) {
|
||||
const { audioInputs, permissionGranted, ensurePermissions } = useDevicesList({ constraints: { audio: true }, requestPermissions: requestPermission })
|
||||
const selectedAudioInput = ref<string>(audioInputs.value.find(device => device.deviceId === 'default')?.deviceId || '')
|
||||
function selectAvailableAudioInput() {
|
||||
if (!audioInputs.value.length)
|
||||
return
|
||||
|
||||
const selectedIsAvailable = audioInputs.value.some(device => device.deviceId === selectedAudioInput.value)
|
||||
if (!selectedAudioInput.value || !selectedIsAvailable)
|
||||
selectedAudioInput.value = resolvePreferredAudioInput(audioInputs.value)
|
||||
}
|
||||
|
||||
const deviceConstraints = computed<MediaStreamConstraints>(() => ({
|
||||
audio: {
|
||||
deviceId: { exact: selectedAudioInput.value },
|
||||
autoGainControl: true,
|
||||
echoCancellation: true,
|
||||
noiseSuppression: true,
|
||||
},
|
||||
audio: selectedAudioInput.value
|
||||
? {
|
||||
deviceId: { exact: selectedAudioInput.value },
|
||||
autoGainControl: true,
|
||||
echoCancellation: true,
|
||||
noiseSuppression: true,
|
||||
}
|
||||
: {
|
||||
autoGainControl: true,
|
||||
echoCancellation: true,
|
||||
noiseSuppression: true,
|
||||
},
|
||||
}))
|
||||
const { stream, stop: stopStream, start: startStream } = useUserMedia({ constraints: deviceConstraints, enabled: false, autoSwitch: true })
|
||||
const { stream, stop: stopStream, start: startUserMediaStream } = useUserMedia({ constraints: deviceConstraints, enabled: false, autoSwitch: true })
|
||||
|
||||
watch(audioInputs, () => {
|
||||
if (selectedAudioInput.value === '' && audioInputs.value.length > 0) {
|
||||
selectedAudioInput.value = audioInputs.value.find(input => input.deviceId === 'default')?.deviceId || audioInputs.value[0].deviceId
|
||||
}
|
||||
selectAvailableAudioInput()
|
||||
})
|
||||
|
||||
function askPermission() {
|
||||
return ensurePermissions()
|
||||
.then(() => nextTick())
|
||||
.then(() => {
|
||||
if (audioInputs.value.length > 0 && !selectedAudioInput.value) {
|
||||
selectedAudioInput.value = audioInputs.value.find(input => input.deviceId === 'default')?.deviceId || audioInputs.value[0].deviceId
|
||||
}
|
||||
selectAvailableAudioInput()
|
||||
})
|
||||
.catch((error) => {
|
||||
console.error('Error ensuring permissions:', error)
|
||||
@@ -34,6 +60,30 @@ export function useAudioDevice(requestPermission: boolean = false) {
|
||||
})
|
||||
}
|
||||
|
||||
async function startStream() {
|
||||
selectAvailableAudioInput()
|
||||
|
||||
try {
|
||||
return await startUserMediaStream()
|
||||
}
|
||||
catch (error) {
|
||||
const fallbackDeviceId = resolvePreferredAudioInput(audioInputs.value)
|
||||
if (fallbackDeviceId && fallbackDeviceId !== selectedAudioInput.value) {
|
||||
selectedAudioInput.value = fallbackDeviceId
|
||||
await nextTick()
|
||||
return await startUserMediaStream()
|
||||
}
|
||||
|
||||
if (selectedAudioInput.value && isMissingAudioInputDeviceError(error)) {
|
||||
selectedAudioInput.value = ''
|
||||
await nextTick()
|
||||
return await startUserMediaStream()
|
||||
}
|
||||
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
audioInputs,
|
||||
selectedAudioInput,
|
||||
|
||||
@@ -0,0 +1,147 @@
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import { shallowRef } from 'vue'
|
||||
|
||||
const mediabunnyMock = vi.hoisted(() => {
|
||||
const audioSources: Array<{ track: MediaStreamTrack, encodingConfig: { codec: string, bitrate: number } }> = []
|
||||
const outputs: Array<{ target: { buffer?: Uint8Array }, finalized: boolean }> = []
|
||||
let startFailuresRemaining = 0
|
||||
|
||||
class FakeBufferTarget {
|
||||
buffer?: Uint8Array
|
||||
}
|
||||
|
||||
class FakeWavOutputFormat {}
|
||||
|
||||
class FakeMediaStreamAudioTrackSource {
|
||||
errorPromise = new Promise<void>(() => {})
|
||||
|
||||
constructor(track: MediaStreamTrack, encodingConfig: { codec: string, bitrate: number }) {
|
||||
audioSources.push({ track, encodingConfig })
|
||||
}
|
||||
}
|
||||
|
||||
class FakeOutput {
|
||||
target: FakeBufferTarget
|
||||
finalized = false
|
||||
|
||||
constructor(options: { target: FakeBufferTarget }) {
|
||||
this.target = options.target
|
||||
outputs.push(this)
|
||||
}
|
||||
|
||||
addAudioTrack() {}
|
||||
|
||||
async getMimeType() {
|
||||
return 'audio/wav'
|
||||
}
|
||||
|
||||
async start() {
|
||||
if (startFailuresRemaining > 0) {
|
||||
startFailuresRemaining -= 1
|
||||
throw new Error('start failed')
|
||||
}
|
||||
|
||||
this.target.buffer = new Uint8Array([outputs.length])
|
||||
}
|
||||
|
||||
async finalize() {
|
||||
this.finalized = true
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
audioSources,
|
||||
outputs,
|
||||
failNextStart: () => {
|
||||
startFailuresRemaining += 1
|
||||
},
|
||||
FakeBufferTarget,
|
||||
FakeMediaStreamAudioTrackSource,
|
||||
FakeOutput,
|
||||
FakeWavOutputFormat,
|
||||
}
|
||||
})
|
||||
|
||||
vi.mock('mediabunny', () => ({
|
||||
BufferTarget: mediabunnyMock.FakeBufferTarget,
|
||||
MediaStreamAudioTrackSource: mediabunnyMock.FakeMediaStreamAudioTrackSource,
|
||||
Output: mediabunnyMock.FakeOutput,
|
||||
QUALITY_MEDIUM: 1,
|
||||
WavOutputFormat: mediabunnyMock.FakeWavOutputFormat,
|
||||
}))
|
||||
|
||||
function createMediaStream() {
|
||||
return {
|
||||
getAudioTracks: () => ([{} as MediaStreamTrack]),
|
||||
} as MediaStream
|
||||
}
|
||||
|
||||
describe('useAudioRecorder', () => {
|
||||
it('records WAV audio with 16-bit PCM for transcription providers', async () => {
|
||||
const { useAudioRecorder } = await import('./audio-recorder')
|
||||
const stream = shallowRef(createMediaStream())
|
||||
|
||||
const { startRecord } = useAudioRecorder(stream)
|
||||
|
||||
await startRecord()
|
||||
|
||||
expect(mediabunnyMock.audioSources.at(-1)?.encodingConfig).toEqual({
|
||||
codec: 'pcm-s16',
|
||||
bitrate: 1,
|
||||
})
|
||||
})
|
||||
|
||||
it('keeps a new recording active while previous stop hooks finish', async () => {
|
||||
const { useAudioRecorder } = await import('./audio-recorder')
|
||||
const stream = shallowRef(createMediaStream())
|
||||
|
||||
const { startRecord, stopRecord, onStopRecord, isRecording } = useAudioRecorder(stream)
|
||||
|
||||
let resolveFirstHook!: () => void
|
||||
let shouldBlockHook = true
|
||||
onStopRecord(async () => {
|
||||
if (!shouldBlockHook)
|
||||
return
|
||||
|
||||
shouldBlockHook = false
|
||||
await new Promise<void>((resolve) => {
|
||||
resolveFirstHook = resolve
|
||||
})
|
||||
})
|
||||
|
||||
await startRecord()
|
||||
expect(isRecording.value).toBe(true)
|
||||
|
||||
const firstStop = stopRecord()
|
||||
await Promise.resolve()
|
||||
expect(isRecording.value).toBe(false)
|
||||
|
||||
await startRecord()
|
||||
expect(isRecording.value).toBe(true)
|
||||
|
||||
const activeSecondOutput = mediabunnyMock.outputs.at(-1)
|
||||
|
||||
resolveFirstHook()
|
||||
await firstStop
|
||||
|
||||
await stopRecord()
|
||||
expect(isRecording.value).toBe(false)
|
||||
|
||||
expect(activeSecondOutput?.finalized).toBe(true)
|
||||
})
|
||||
|
||||
it('resets recorder state after startup fails so recording can be retried', async () => {
|
||||
const { useAudioRecorder } = await import('./audio-recorder')
|
||||
const stream = shallowRef(createMediaStream())
|
||||
|
||||
const { startRecord, isRecording } = useAudioRecorder(stream)
|
||||
mediabunnyMock.failNextStart()
|
||||
|
||||
await expect(startRecord()).rejects.toThrow('start failed')
|
||||
expect(isRecording.value).toBe(false)
|
||||
|
||||
await startRecord()
|
||||
|
||||
expect(isRecording.value).toBe(true)
|
||||
})
|
||||
})
|
||||
@@ -2,8 +2,13 @@ import type { MaybeRefOrGetter } from 'vue'
|
||||
|
||||
import { until } from '@vueuse/core'
|
||||
import { BufferTarget, MediaStreamAudioTrackSource, Output, QUALITY_MEDIUM, WavOutputFormat } from 'mediabunny'
|
||||
import { ref, shallowRef, toRef } from 'vue'
|
||||
import { computed, ref, shallowRef, toRef } from 'vue'
|
||||
|
||||
const TRANSCRIPTION_WAV_CODEC = 'pcm-s16'
|
||||
|
||||
/**
|
||||
* Returns the first audio track from the active microphone stream.
|
||||
*/
|
||||
function getMediaStreamTrack(stream: MediaStream) {
|
||||
const tracks = stream.getAudioTracks()
|
||||
if (!tracks.length)
|
||||
@@ -11,17 +16,24 @@ function getMediaStreamTrack(stream: MediaStream) {
|
||||
return tracks[0]
|
||||
}
|
||||
|
||||
/**
|
||||
* Records microphone input into short WAV blobs for transcription providers.
|
||||
*/
|
||||
export function useAudioRecorder(
|
||||
media: MaybeRefOrGetter<MediaStream | undefined>,
|
||||
) {
|
||||
const mediaRef = toRef(media)
|
||||
const recording = shallowRef<Blob>()
|
||||
|
||||
const mediaOutput = ref<Output>()
|
||||
const mediaFormat = ref<string>()
|
||||
const mediaOutput = shallowRef<Output>()
|
||||
const mediaFormat = shallowRef<string>()
|
||||
const isRecording = computed(() => !!mediaOutput.value)
|
||||
|
||||
const onStopRecordHooks = ref<Array<(recording: Blob | undefined) => Promise<void>>>([])
|
||||
|
||||
/**
|
||||
* Registers a callback that receives each finalized recording blob.
|
||||
*/
|
||||
function onStopRecord(callback: (recording: Blob | undefined) => Promise<void>) {
|
||||
onStopRecordHooks.value.push(callback)
|
||||
// Return unsubscribe function to prevent memory leaks
|
||||
@@ -30,29 +42,55 @@ export function useAudioRecorder(
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Starts recording from the current microphone stream if no recording is active.
|
||||
*/
|
||||
async function startRecord() {
|
||||
if (mediaOutput.value)
|
||||
return
|
||||
|
||||
await until(mediaRef).toBeTruthy()
|
||||
|
||||
const track = await getMediaStreamTrack(mediaRef.value!)
|
||||
mediaOutput.value = new Output({ format: new WavOutputFormat(), target: new BufferTarget() })
|
||||
const output = new Output({ format: new WavOutputFormat(), target: new BufferTarget() })
|
||||
mediaOutput.value = output
|
||||
|
||||
const audioSource = new MediaStreamAudioTrackSource(track, { codec: 'pcm-f32', bitrate: QUALITY_MEDIUM })
|
||||
audioSource.errorPromise.catch(console.error)
|
||||
mediaOutput.value.addAudioTrack(audioSource)
|
||||
try {
|
||||
const audioSource = new MediaStreamAudioTrackSource(track, { codec: TRANSCRIPTION_WAV_CODEC, bitrate: QUALITY_MEDIUM })
|
||||
audioSource.errorPromise.catch(console.error)
|
||||
output.addAudioTrack(audioSource)
|
||||
|
||||
mediaFormat.value = await mediaOutput.value.getMimeType()
|
||||
await mediaOutput.value.start()
|
||||
mediaFormat.value = await output.getMimeType()
|
||||
await output.start()
|
||||
}
|
||||
catch (error) {
|
||||
if (mediaOutput.value === output) {
|
||||
mediaOutput.value = undefined
|
||||
mediaFormat.value = undefined
|
||||
}
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Finalizes the active recording and runs stop hooks without blocking the next recording.
|
||||
*/
|
||||
async function stopRecord() {
|
||||
if (!mediaOutput.value) {
|
||||
const activeOutput = mediaOutput.value
|
||||
const activeFormat = mediaFormat.value
|
||||
if (!activeOutput) {
|
||||
return
|
||||
}
|
||||
|
||||
await mediaOutput.value.finalize()
|
||||
const bufferTarget = mediaOutput.value.target as BufferTarget | undefined
|
||||
// Clear the active output before running transcription hooks so VAD can start the next utterance
|
||||
// while the previous blob is still being sent to the ASR provider.
|
||||
mediaOutput.value = undefined
|
||||
mediaFormat.value = undefined
|
||||
|
||||
await activeOutput.finalize()
|
||||
const bufferTarget = activeOutput.target as BufferTarget | undefined
|
||||
const buffer = bufferTarget?.buffer
|
||||
const audioBlob = buffer ? new Blob([buffer], { type: mediaFormat.value }) : undefined
|
||||
const audioBlob = buffer ? new Blob([buffer], { type: activeFormat }) : undefined
|
||||
|
||||
recording.value = audioBlob
|
||||
|
||||
@@ -66,8 +104,6 @@ export function useAudioRecorder(
|
||||
}
|
||||
}
|
||||
|
||||
mediaOutput.value = undefined
|
||||
|
||||
return audioBlob
|
||||
}
|
||||
|
||||
@@ -76,6 +112,7 @@ export function useAudioRecorder(
|
||||
stopRecord,
|
||||
onStopRecord,
|
||||
|
||||
isRecording,
|
||||
recording,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,3 +2,4 @@ export * from './audio-analyzer'
|
||||
export * from './audio-context'
|
||||
export * from './audio-device'
|
||||
export * from './audio-recorder'
|
||||
export * from './voice-input-session'
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
|
||||
import {
|
||||
createVoiceInputRecordingSegment,
|
||||
resolveActiveVoiceInputRecordingSegmentAfterStop,
|
||||
} from './voice-input-segment'
|
||||
|
||||
describe('voice input recording segment tracking', () => {
|
||||
it('does not clear a newer active segment when an older segment finishes stopping', () => {
|
||||
const stoppedSegment = createVoiceInputRecordingSegment(1, 'volume')
|
||||
const newerActiveSegment = createVoiceInputRecordingSegment(2, 'vad')
|
||||
|
||||
expect(resolveActiveVoiceInputRecordingSegmentAfterStop(newerActiveSegment, stoppedSegment))
|
||||
.toBe(newerActiveSegment)
|
||||
})
|
||||
|
||||
it('clears the active segment when the stopped segment is still current', () => {
|
||||
const stoppedSegment = createVoiceInputRecordingSegment(1, 'manual')
|
||||
|
||||
expect(resolveActiveVoiceInputRecordingSegmentAfterStop(stoppedSegment, stoppedSegment))
|
||||
.toBeUndefined()
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,26 @@
|
||||
export type VoiceInputSessionTrigger = 'manual' | 'vad' | 'volume'
|
||||
|
||||
export interface VoiceInputRecordingSegment {
|
||||
id: number
|
||||
trigger: VoiceInputSessionTrigger
|
||||
}
|
||||
|
||||
export function createVoiceInputRecordingSegment(id: number, trigger: VoiceInputSessionTrigger): VoiceInputRecordingSegment {
|
||||
return { id, trigger }
|
||||
}
|
||||
|
||||
function isSameVoiceInputRecordingSegment(
|
||||
left: VoiceInputRecordingSegment | undefined,
|
||||
right: VoiceInputRecordingSegment | undefined,
|
||||
) {
|
||||
return !!left && !!right && left.id === right.id
|
||||
}
|
||||
|
||||
export function resolveActiveVoiceInputRecordingSegmentAfterStop(
|
||||
activeSegment: VoiceInputRecordingSegment | undefined,
|
||||
stoppedSegment: VoiceInputRecordingSegment | undefined,
|
||||
) {
|
||||
return isSameVoiceInputRecordingSegment(activeSegment, stoppedSegment)
|
||||
? undefined
|
||||
: activeSegment
|
||||
}
|
||||
@@ -0,0 +1,292 @@
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import { ref, shallowRef } from 'vue'
|
||||
|
||||
const audioRecorderMock = vi.hoisted(() => ({
|
||||
isRecording: undefined as unknown as { value: boolean },
|
||||
startRecord: vi.fn(),
|
||||
stopRecord: vi.fn(),
|
||||
}))
|
||||
|
||||
vi.mock('../../workers/vad/process.worklet?worker&url', () => ({
|
||||
default: 'vad-worklet-url',
|
||||
}))
|
||||
|
||||
vi.mock('../../stores/ai/models/vad', async () => {
|
||||
const vue = await vi.importActual<typeof import('vue')>('vue')
|
||||
|
||||
return {
|
||||
useVAD: () => ({
|
||||
init: vi.fn(),
|
||||
dispose: vi.fn(),
|
||||
start: vi.fn(),
|
||||
loaded: vue.ref(true),
|
||||
isSpeech: vue.ref(false),
|
||||
isSpeechProb: vue.ref(0),
|
||||
isSpeechHistory: vue.ref([]),
|
||||
inferenceError: vue.ref(),
|
||||
}),
|
||||
}
|
||||
})
|
||||
|
||||
vi.mock('../../stores/modules/hearing', () => ({
|
||||
useHearingSpeechInputPipeline: () => ({
|
||||
transcribeForRecording: vi.fn(async () => ''),
|
||||
}),
|
||||
}))
|
||||
|
||||
vi.mock('./audio-recorder', async () => {
|
||||
const vue = await vi.importActual<typeof import('vue')>('vue')
|
||||
audioRecorderMock.isRecording = vue.ref(false)
|
||||
|
||||
return {
|
||||
useAudioRecorder: () => ({
|
||||
isRecording: audioRecorderMock.isRecording,
|
||||
startRecord: audioRecorderMock.startRecord,
|
||||
stopRecord: audioRecorderMock.stopRecord,
|
||||
onStopRecord: vi.fn(),
|
||||
}),
|
||||
}
|
||||
})
|
||||
|
||||
function createMediaStream() {
|
||||
return {
|
||||
getAudioTracks: () => ([{} as MediaStreamTrack]),
|
||||
} as MediaStream
|
||||
}
|
||||
|
||||
describe('useVoiceInputSession', () => {
|
||||
afterEach(() => {
|
||||
audioRecorderMock.isRecording.value = false
|
||||
vi.useRealTimers()
|
||||
vi.unstubAllGlobals()
|
||||
vi.clearAllMocks()
|
||||
})
|
||||
|
||||
it('clears the active recorder segment when discarding fails during stop', async () => {
|
||||
const { useVoiceInputSession } = await import('./voice-input-session')
|
||||
|
||||
audioRecorderMock.startRecord.mockImplementation(async () => {
|
||||
audioRecorderMock.isRecording.value = true
|
||||
})
|
||||
audioRecorderMock.stopRecord.mockImplementationOnce(async () => {
|
||||
audioRecorderMock.isRecording.value = false
|
||||
throw new Error('finalize failed')
|
||||
})
|
||||
|
||||
const session = useVoiceInputSession(shallowRef(createMediaStream()), {
|
||||
volumeFallback: { enabled: false },
|
||||
})
|
||||
|
||||
await expect(session.startSegment('manual')).resolves.toBe(true)
|
||||
expect(session.activeRecordingTrigger.value).toBe('manual')
|
||||
|
||||
await expect(session.stop({ flushActiveRecording: false })).rejects.toThrow('finalize failed')
|
||||
|
||||
expect(session.activeRecordingTrigger.value).toBeUndefined()
|
||||
})
|
||||
|
||||
it('reports a failed recorder start without leaving an active segment', async () => {
|
||||
const { useVoiceInputSession } = await import('./voice-input-session')
|
||||
const startupError = new Error('start failed')
|
||||
|
||||
audioRecorderMock.startRecord.mockRejectedValueOnce(startupError)
|
||||
|
||||
const session = useVoiceInputSession(shallowRef(createMediaStream()), {
|
||||
volumeFallback: { enabled: false },
|
||||
})
|
||||
|
||||
await expect(session.startSegment('manual')).resolves.toBe(false)
|
||||
|
||||
expect(session.activeRecordingTrigger.value).toBeUndefined()
|
||||
expect(session.lastError.value).toBe(startupError)
|
||||
})
|
||||
|
||||
it('clears the active segment when the caller start gate rejects', async () => {
|
||||
const { useVoiceInputSession } = await import('./voice-input-session')
|
||||
const gateError = new Error('gate failed')
|
||||
|
||||
const session = useVoiceInputSession(shallowRef(createMediaStream()), {
|
||||
volumeFallback: { enabled: false },
|
||||
canStartSegment: vi.fn()
|
||||
.mockRejectedValueOnce(gateError)
|
||||
.mockResolvedValueOnce(true),
|
||||
})
|
||||
|
||||
await expect(session.startSegment('manual')).resolves.toBe(false)
|
||||
|
||||
expect(session.activeRecordingTrigger.value).toBeUndefined()
|
||||
expect(session.lastError.value).toBe(gateError)
|
||||
|
||||
audioRecorderMock.startRecord.mockImplementationOnce(async () => {
|
||||
audioRecorderMock.isRecording.value = true
|
||||
})
|
||||
|
||||
await expect(session.startSegment('manual')).resolves.toBe(true)
|
||||
expect(session.activeRecordingTrigger.value).toBe('manual')
|
||||
})
|
||||
|
||||
it('clears the active segment when the caller start hook rejects', async () => {
|
||||
const { useVoiceInputSession } = await import('./voice-input-session')
|
||||
const hookError = new Error('start hook failed')
|
||||
|
||||
const session = useVoiceInputSession(shallowRef(createMediaStream()), {
|
||||
volumeFallback: { enabled: false },
|
||||
onSegmentStart: vi.fn().mockRejectedValueOnce(hookError),
|
||||
})
|
||||
|
||||
await expect(session.startSegment('manual')).resolves.toBe(false)
|
||||
|
||||
expect(audioRecorderMock.startRecord).not.toHaveBeenCalled()
|
||||
expect(session.activeRecordingTrigger.value).toBeUndefined()
|
||||
expect(session.lastError.value).toBe(hookError)
|
||||
})
|
||||
|
||||
it('stops and clears the recorder when the caller started hook rejects', async () => {
|
||||
const { useVoiceInputSession } = await import('./voice-input-session')
|
||||
const hookError = new Error('started hook failed')
|
||||
|
||||
audioRecorderMock.startRecord.mockImplementation(async () => {
|
||||
audioRecorderMock.isRecording.value = true
|
||||
})
|
||||
audioRecorderMock.stopRecord.mockImplementation(async () => {
|
||||
audioRecorderMock.isRecording.value = false
|
||||
})
|
||||
|
||||
const session = useVoiceInputSession(shallowRef(createMediaStream()), {
|
||||
volumeFallback: { enabled: false },
|
||||
onSegmentStarted: vi.fn().mockRejectedValueOnce(hookError),
|
||||
})
|
||||
|
||||
await expect(session.startSegment('manual')).resolves.toBe(false)
|
||||
|
||||
expect(audioRecorderMock.stopRecord).toHaveBeenCalledOnce()
|
||||
expect(session.isRecording.value).toBe(false)
|
||||
expect(session.activeRecordingTrigger.value).toBeUndefined()
|
||||
expect(session.lastError.value).toBe(hookError)
|
||||
})
|
||||
|
||||
it('finalizes the recorder when the caller stop hook rejects', async () => {
|
||||
const { useVoiceInputSession } = await import('./voice-input-session')
|
||||
const hookError = new Error('stop hook failed')
|
||||
const onTranscriptionError = vi.fn()
|
||||
|
||||
audioRecorderMock.startRecord.mockImplementation(async () => {
|
||||
audioRecorderMock.isRecording.value = true
|
||||
})
|
||||
audioRecorderMock.stopRecord.mockImplementation(async () => {
|
||||
audioRecorderMock.isRecording.value = false
|
||||
})
|
||||
|
||||
const session = useVoiceInputSession(shallowRef(createMediaStream()), {
|
||||
volumeFallback: { enabled: false },
|
||||
onSegmentStop: vi.fn().mockRejectedValueOnce(hookError),
|
||||
onTranscriptionError,
|
||||
})
|
||||
|
||||
await expect(session.startSegment('manual')).resolves.toBe(true)
|
||||
await expect(session.stopSegment('manual')).resolves.toBeUndefined()
|
||||
|
||||
expect(audioRecorderMock.stopRecord).toHaveBeenCalledOnce()
|
||||
expect(session.isRecording.value).toBe(false)
|
||||
expect(session.activeRecordingTrigger.value).toBeUndefined()
|
||||
expect(session.lastError.value).toBe(hookError)
|
||||
expect(onTranscriptionError).toHaveBeenCalledWith(expect.objectContaining({ error: hookError }))
|
||||
})
|
||||
|
||||
it('stops an active recorder segment after stream mode becomes enabled', async () => {
|
||||
const { useVoiceInputSession } = await import('./voice-input-session')
|
||||
const shouldUseStreamInput = ref(false)
|
||||
|
||||
audioRecorderMock.startRecord.mockImplementation(async () => {
|
||||
audioRecorderMock.isRecording.value = true
|
||||
})
|
||||
audioRecorderMock.stopRecord.mockImplementation(async () => {
|
||||
audioRecorderMock.isRecording.value = false
|
||||
})
|
||||
|
||||
const session = useVoiceInputSession(shallowRef(createMediaStream()), {
|
||||
shouldUseStreamInput,
|
||||
volumeFallback: { enabled: false },
|
||||
})
|
||||
|
||||
await expect(session.startSegment('manual')).resolves.toBe(true)
|
||||
shouldUseStreamInput.value = true
|
||||
await session.stopSegment('manual')
|
||||
|
||||
expect(audioRecorderMock.stopRecord).toHaveBeenCalledOnce()
|
||||
expect(session.isRecording.value).toBe(false)
|
||||
expect(session.activeRecordingTrigger.value).toBeUndefined()
|
||||
})
|
||||
|
||||
it('lets volume fallback finalize a VAD-owned segment after silence', async () => {
|
||||
vi.useFakeTimers()
|
||||
vi.setSystemTime(1000)
|
||||
|
||||
const animationFrames: FrameRequestCallback[] = []
|
||||
const stopRecord = audioRecorderMock.stopRecord.mockImplementation(async () => {
|
||||
audioRecorderMock.isRecording.value = false
|
||||
})
|
||||
audioRecorderMock.startRecord.mockImplementation(async () => {
|
||||
audioRecorderMock.isRecording.value = true
|
||||
})
|
||||
|
||||
class FakeAudioContext {
|
||||
state: AudioContextState = 'running'
|
||||
destination = {}
|
||||
|
||||
createMediaStreamSource() {
|
||||
return {
|
||||
connect: vi.fn(),
|
||||
disconnect: vi.fn(),
|
||||
}
|
||||
}
|
||||
|
||||
createAnalyser() {
|
||||
return {
|
||||
fftSize: 512,
|
||||
smoothingTimeConstant: 0,
|
||||
connect: vi.fn(),
|
||||
disconnect: vi.fn(),
|
||||
getByteTimeDomainData: (data: Uint8Array<ArrayBuffer>) => data.fill(128),
|
||||
}
|
||||
}
|
||||
|
||||
createGain() {
|
||||
return {
|
||||
gain: { value: 1 },
|
||||
connect: vi.fn(),
|
||||
disconnect: vi.fn(),
|
||||
}
|
||||
}
|
||||
|
||||
resume = vi.fn()
|
||||
close = vi.fn()
|
||||
}
|
||||
|
||||
vi.stubGlobal('AudioContext', FakeAudioContext)
|
||||
vi.stubGlobal('requestAnimationFrame', vi.fn((callback: FrameRequestCallback) => {
|
||||
animationFrames.push(callback)
|
||||
return animationFrames.length
|
||||
}))
|
||||
vi.stubGlobal('cancelAnimationFrame', vi.fn())
|
||||
|
||||
const { useVoiceInputSession } = await import('./voice-input-session')
|
||||
const session = useVoiceInputSession(shallowRef(createMediaStream()), {
|
||||
volumeFallback: {
|
||||
enabled: true,
|
||||
stopDelayMs: 10,
|
||||
},
|
||||
})
|
||||
|
||||
await expect(session.startSegment('vad')).resolves.toBe(true)
|
||||
await session.startAutoSegmentation()
|
||||
|
||||
animationFrames.shift()?.(1000)
|
||||
vi.setSystemTime(1011)
|
||||
animationFrames.shift()?.(1011)
|
||||
await Promise.resolve()
|
||||
|
||||
expect(stopRecord).toHaveBeenCalledOnce()
|
||||
expect(session.activeRecordingTrigger.value).toBeUndefined()
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,589 @@
|
||||
import type { MaybeRefOrGetter } from 'vue'
|
||||
|
||||
import type { VoiceInputRecordingSegment, VoiceInputSessionTrigger } from './voice-input-segment'
|
||||
import type { VoiceInputTranscriptionTicket } from './voice-input-transcription-chain'
|
||||
|
||||
import { computed, ref, shallowRef, toRef } from 'vue'
|
||||
|
||||
import workletUrl from '../../workers/vad/process.worklet?worker&url'
|
||||
|
||||
import { useVAD } from '../../stores/ai/models/vad'
|
||||
import { useHearingSpeechInputPipeline } from '../../stores/modules/hearing'
|
||||
import { useAudioRecorder } from './audio-recorder'
|
||||
import {
|
||||
createVoiceInputRecordingSegment,
|
||||
resolveActiveVoiceInputRecordingSegmentAfterStop,
|
||||
} from './voice-input-segment'
|
||||
import { createVoiceInputTranscriptionChain } from './voice-input-transcription-chain'
|
||||
import { startVoiceInputVadDetectionSafely } from './voice-input-vad-startup'
|
||||
|
||||
export type { VoiceInputSessionTrigger } from './voice-input-segment'
|
||||
|
||||
export type VoiceInputSessionLogLevel = 'info' | 'warn' | 'error'
|
||||
|
||||
export interface VoiceInputSessionGate {
|
||||
skip?: boolean
|
||||
reason?: string
|
||||
details?: Record<string, unknown>
|
||||
}
|
||||
|
||||
export interface VoiceInputSessionEvent {
|
||||
trigger: VoiceInputSessionTrigger
|
||||
recording?: Blob
|
||||
text?: string
|
||||
error?: unknown
|
||||
metadata?: Record<string, unknown>
|
||||
gate?: VoiceInputSessionGate
|
||||
}
|
||||
|
||||
export interface VoiceInputSessionVadOptions {
|
||||
threshold?: MaybeRefOrGetter<number>
|
||||
minSilenceDurationMs?: MaybeRefOrGetter<number>
|
||||
speechPadMs?: MaybeRefOrGetter<number>
|
||||
minSpeechDurationMs?: MaybeRefOrGetter<number>
|
||||
}
|
||||
|
||||
export interface VoiceInputSessionVolumeFallbackOptions {
|
||||
enabled?: MaybeRefOrGetter<boolean>
|
||||
startThreshold?: number
|
||||
stopThreshold?: number
|
||||
startFrames?: number
|
||||
stopDelayMs?: number
|
||||
logIntervalMs?: number
|
||||
}
|
||||
|
||||
export interface VoiceInputSessionOptions {
|
||||
shouldUseStreamInput?: MaybeRefOrGetter<boolean>
|
||||
vad?: VoiceInputSessionVadOptions
|
||||
volumeFallback?: VoiceInputSessionVolumeFallbackOptions
|
||||
canStartSegment?: (event: VoiceInputSessionEvent) => boolean | Promise<boolean>
|
||||
inspectBeforeTranscription?: (event: VoiceInputSessionEvent) => VoiceInputSessionGate | Promise<VoiceInputSessionGate | undefined> | undefined
|
||||
inspectAfterTranscription?: (event: VoiceInputSessionEvent) => VoiceInputSessionGate | Promise<VoiceInputSessionGate | undefined> | undefined
|
||||
onLog?: (level: VoiceInputSessionLogLevel, event: string, message: string, details?: Record<string, unknown>) => void
|
||||
onSegmentStart?: (event: VoiceInputSessionEvent) => void | Promise<void>
|
||||
onSegmentStarted?: (event: VoiceInputSessionEvent) => void | Promise<void>
|
||||
onSegmentStop?: (event: VoiceInputSessionEvent) => void | Promise<void>
|
||||
onSegmentStopped?: (event: VoiceInputSessionEvent) => void | Promise<void>
|
||||
onRecordingReady?: (event: VoiceInputSessionEvent) => Record<string, unknown> | void | Promise<Record<string, unknown> | void>
|
||||
onRecordingSkipped?: (event: VoiceInputSessionEvent) => void | Promise<void>
|
||||
onTranscriptionStart?: (event: VoiceInputSessionEvent) => void | Promise<void>
|
||||
onTranscriptionResult?: (event: VoiceInputSessionEvent & { text: string }) => void | Promise<void>
|
||||
onTranscriptionEmpty?: (event: VoiceInputSessionEvent & { text: string }) => void | Promise<void>
|
||||
onTranscriptionError?: (event: VoiceInputSessionEvent & { error: unknown }) => void | Promise<void>
|
||||
}
|
||||
|
||||
const DEFAULT_VOLUME_FALLBACK_START_THRESHOLD = 10
|
||||
const DEFAULT_VOLUME_FALLBACK_STOP_THRESHOLD = 6
|
||||
const DEFAULT_VOLUME_FALLBACK_START_FRAMES = 4
|
||||
const DEFAULT_VOLUME_FALLBACK_STOP_DELAY_MS = 900
|
||||
const DEFAULT_VOLUME_FALLBACK_LOG_INTERVAL_MS = 2000
|
||||
|
||||
function calculateTimeDomainVolumeLevel(dataArray: Uint8Array<ArrayBuffer>) {
|
||||
let sum = 0
|
||||
for (let i = 0; i < dataArray.length; i++) {
|
||||
const centered = (dataArray[i] - 128) / 128
|
||||
sum += centered * centered
|
||||
}
|
||||
|
||||
return Math.min(100, Math.sqrt(sum / dataArray.length) * 100 * 3)
|
||||
}
|
||||
|
||||
/**
|
||||
* Shared voice-input session for both manual STT tests and always-on stage listening.
|
||||
*
|
||||
* Owns:
|
||||
* - recorder-backed segment creation
|
||||
* - VAD-triggered auto segmentation
|
||||
* - volume-triggered fallback segmentation
|
||||
* - record-then-transcribe ASR calls
|
||||
*
|
||||
* Leaves product-specific behavior, such as sending text to chat or updating UI state, to callbacks.
|
||||
*/
|
||||
export function useVoiceInputSession(
|
||||
media: MaybeRefOrGetter<MediaStream | undefined>,
|
||||
options: VoiceInputSessionOptions = {},
|
||||
) {
|
||||
const mediaRef = toRef(media)
|
||||
const shouldUseStreamInput = toRef(options.shouldUseStreamInput ?? false)
|
||||
const volumeFallbackEnabled = toRef(options.volumeFallback?.enabled ?? true)
|
||||
const hearingPipeline = useHearingSpeechInputPipeline()
|
||||
const { transcribeForRecording } = hearingPipeline
|
||||
const recorder = useAudioRecorder(mediaRef)
|
||||
|
||||
const activeRecordingSegment = shallowRef<VoiceInputRecordingSegment>()
|
||||
const activeRecordingTrigger = computed(() => activeRecordingSegment.value?.trigger)
|
||||
const isTranscribing = ref(false)
|
||||
const lastTranscriptionText = ref('')
|
||||
const lastError = ref<unknown>()
|
||||
const transcriptionChain = createVoiceInputTranscriptionChain()
|
||||
const stoppedRecordingSegments: VoiceInputRecordingSegment[] = []
|
||||
let nextRecordingSegmentId = 0
|
||||
let discardNextRecording = false
|
||||
let activeTranscriptionCount = 0
|
||||
|
||||
const {
|
||||
init: initVAD,
|
||||
dispose: disposeVAD,
|
||||
start: startVAD,
|
||||
loaded: vadLoaded,
|
||||
isSpeech: isSpeechVAD,
|
||||
isSpeechProb,
|
||||
isSpeechHistory,
|
||||
inferenceError: vadError,
|
||||
} = useVAD(workletUrl, {
|
||||
threshold: options.vad?.threshold,
|
||||
minSilenceDurationMs: options.vad?.minSilenceDurationMs,
|
||||
speechPadMs: options.vad?.speechPadMs,
|
||||
minSpeechDurationMs: options.vad?.minSpeechDurationMs,
|
||||
onSpeechStart: () => {
|
||||
void startSegment('vad')
|
||||
},
|
||||
onSpeechEnd: () => {
|
||||
void stopSegment('vad')
|
||||
},
|
||||
})
|
||||
|
||||
let volumeFallbackAudioContext: AudioContext | undefined
|
||||
let volumeFallbackSourceNode: MediaStreamAudioSourceNode | undefined
|
||||
let volumeFallbackAnalyserNode: AnalyserNode | undefined
|
||||
let volumeFallbackSilentGainNode: GainNode | undefined
|
||||
let volumeFallbackDataArray: Uint8Array<ArrayBuffer> | undefined
|
||||
let volumeFallbackAnimationFrame: number | undefined
|
||||
let volumeFallbackSpeechFrames = 0
|
||||
let volumeFallbackLastSpeechAt = 0
|
||||
let volumeFallbackLastLogAt = 0
|
||||
|
||||
const isRecording = computed(() => recorder.isRecording.value)
|
||||
|
||||
function log(level: VoiceInputSessionLogLevel, event: string, message: string, details?: Record<string, unknown>) {
|
||||
options.onLog?.(level, event, message, details)
|
||||
}
|
||||
|
||||
function markTranscriptionStarted() {
|
||||
activeTranscriptionCount += 1
|
||||
isTranscribing.value = true
|
||||
}
|
||||
|
||||
function markTranscriptionFinished() {
|
||||
activeTranscriptionCount = Math.max(0, activeTranscriptionCount - 1)
|
||||
isTranscribing.value = activeTranscriptionCount > 0
|
||||
}
|
||||
|
||||
function isStaleTranscriptionTicket(ticket: VoiceInputTranscriptionTicket, trigger: VoiceInputSessionTrigger, phase: string) {
|
||||
if (ticket.isCurrent())
|
||||
return false
|
||||
|
||||
log('info', 'recording-drop-stale-session', 'Dropping stale recorder-backed transcription work after the listening session changed.', {
|
||||
trigger,
|
||||
phase,
|
||||
})
|
||||
return true
|
||||
}
|
||||
|
||||
async function discardActiveRecorderSegment(segment: VoiceInputRecordingSegment) {
|
||||
discardNextRecording = true
|
||||
try {
|
||||
await recorder.stopRecord()
|
||||
}
|
||||
finally {
|
||||
discardNextRecording = false
|
||||
activeRecordingSegment.value = resolveActiveVoiceInputRecordingSegmentAfterStop(activeRecordingSegment.value, segment)
|
||||
}
|
||||
}
|
||||
|
||||
async function startSegment(trigger: VoiceInputSessionTrigger = 'manual') {
|
||||
const event: VoiceInputSessionEvent = { trigger }
|
||||
if (shouldUseStreamInput.value) {
|
||||
log('info', 'segment-start-skipped-streaming', 'Recorder segment start skipped because streaming transcription is active.', { trigger })
|
||||
return false
|
||||
}
|
||||
|
||||
if (isRecording.value || activeRecordingSegment.value) {
|
||||
log('info', 'segment-start-skipped-active', 'Recorder segment start skipped because another segment is already active.', {
|
||||
trigger,
|
||||
activeRecordingTrigger: activeRecordingTrigger.value,
|
||||
})
|
||||
return false
|
||||
}
|
||||
|
||||
const segment = createVoiceInputRecordingSegment(++nextRecordingSegmentId, trigger)
|
||||
activeRecordingSegment.value = segment
|
||||
|
||||
if (options.canStartSegment) {
|
||||
try {
|
||||
if (!await options.canStartSegment(event)) {
|
||||
log('info', 'segment-start-skipped-gate', 'Recorder segment start skipped by caller gate.', { trigger })
|
||||
activeRecordingSegment.value = resolveActiveVoiceInputRecordingSegmentAfterStop(activeRecordingSegment.value, segment)
|
||||
return false
|
||||
}
|
||||
}
|
||||
catch (error) {
|
||||
activeRecordingSegment.value = resolveActiveVoiceInputRecordingSegmentAfterStop(activeRecordingSegment.value, segment)
|
||||
lastError.value = error
|
||||
log('error', 'segment-start-gate-failed', 'Recorder segment start gate failed.', { trigger, error })
|
||||
await options.onTranscriptionError?.({ trigger, error })
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
await options.onSegmentStart?.(event)
|
||||
await recorder.startRecord()
|
||||
|
||||
try {
|
||||
await options.onSegmentStarted?.(event)
|
||||
}
|
||||
catch (error) {
|
||||
await discardActiveRecorderSegment(segment)
|
||||
throw error
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
catch (error) {
|
||||
activeRecordingSegment.value = resolveActiveVoiceInputRecordingSegmentAfterStop(activeRecordingSegment.value, segment)
|
||||
lastError.value = error
|
||||
log('error', 'segment-start-failed', 'Failed to start recorder-backed voice input segment.', { trigger, error })
|
||||
await options.onTranscriptionError?.({ trigger, error })
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
async function stopSegment(trigger: VoiceInputSessionTrigger = 'manual') {
|
||||
const event: VoiceInputSessionEvent = { trigger }
|
||||
const segment = activeRecordingSegment.value
|
||||
|
||||
if (shouldUseStreamInput.value && !isRecording.value && !segment) {
|
||||
log('info', 'segment-stop-skipped-streaming', 'Recorder segment stop skipped because streaming transcription is active.', { trigger })
|
||||
return
|
||||
}
|
||||
|
||||
if (segment && segment.trigger !== trigger) {
|
||||
log('info', 'segment-stop-skipped-trigger-mismatch', 'Recorder segment stop skipped because another detector owns the active segment.', {
|
||||
trigger,
|
||||
activeRecordingTrigger: activeRecordingTrigger.value,
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
if (!isRecording.value) {
|
||||
log('warn', 'segment-stop-without-active-recorder', 'Recorder segment stop requested without an active recording.', { trigger })
|
||||
return
|
||||
}
|
||||
|
||||
const stoppedSegment = segment ?? createVoiceInputRecordingSegment(++nextRecordingSegmentId, trigger)
|
||||
|
||||
try {
|
||||
await options.onSegmentStop?.(event)
|
||||
}
|
||||
catch (error) {
|
||||
lastError.value = error
|
||||
log('error', 'segment-stop-hook-failed', 'Caller stop hook failed; finalizing recorder segment anyway.', { trigger, error })
|
||||
await options.onTranscriptionError?.({ trigger, error })
|
||||
}
|
||||
|
||||
try {
|
||||
stoppedRecordingSegments.push(stoppedSegment)
|
||||
activeRecordingSegment.value = resolveActiveVoiceInputRecordingSegmentAfterStop(activeRecordingSegment.value, stoppedSegment)
|
||||
await recorder.stopRecord()
|
||||
await options.onSegmentStopped?.(event)
|
||||
}
|
||||
catch (error) {
|
||||
const queuedIndex = stoppedRecordingSegments.findIndex(item => item.id === stoppedSegment.id)
|
||||
if (queuedIndex !== -1)
|
||||
stoppedRecordingSegments.splice(queuedIndex, 1)
|
||||
lastError.value = error
|
||||
log('error', 'segment-stop-failed', 'Failed to stop recorder-backed voice input segment.', { trigger, error })
|
||||
await options.onTranscriptionError?.({ trigger, error })
|
||||
}
|
||||
finally {
|
||||
activeRecordingSegment.value = resolveActiveVoiceInputRecordingSegmentAfterStop(activeRecordingSegment.value, stoppedSegment)
|
||||
}
|
||||
}
|
||||
|
||||
async function processRecording(recording: Blob | undefined, trigger: VoiceInputSessionTrigger, ticket: VoiceInputTranscriptionTicket) {
|
||||
const event: VoiceInputSessionEvent = { trigger, recording }
|
||||
|
||||
if (isStaleTranscriptionTicket(ticket, trigger, 'recording-start'))
|
||||
return
|
||||
|
||||
if (!recording || recording.size <= 0) {
|
||||
log('warn', 'recording-drop-empty', 'Dropping empty recorder-backed voice input segment.', { trigger, recording })
|
||||
await options.onRecordingSkipped?.(event)
|
||||
return
|
||||
}
|
||||
|
||||
const metadata = await options.onRecordingReady?.(event) ?? undefined
|
||||
const readyEvent = { ...event, metadata }
|
||||
if (isStaleTranscriptionTicket(ticket, trigger, 'recording-ready'))
|
||||
return
|
||||
|
||||
const beforeGate = await options.inspectBeforeTranscription?.(readyEvent)
|
||||
if (isStaleTranscriptionTicket(ticket, trigger, 'before-transcription-gate'))
|
||||
return
|
||||
|
||||
if (beforeGate?.skip) {
|
||||
log('info', 'recording-drop-before-asr', 'Skipping recorder-backed segment before transcription request.', {
|
||||
trigger,
|
||||
gate: beforeGate,
|
||||
})
|
||||
await options.onRecordingSkipped?.({ ...readyEvent, gate: beforeGate })
|
||||
return
|
||||
}
|
||||
|
||||
markTranscriptionStarted()
|
||||
|
||||
let text = ''
|
||||
try {
|
||||
await options.onTranscriptionStart?.(readyEvent)
|
||||
if (isStaleTranscriptionTicket(ticket, trigger, 'transcription-started'))
|
||||
return
|
||||
|
||||
text = await transcribeForRecording(recording) ?? ''
|
||||
}
|
||||
catch (error) {
|
||||
if (isStaleTranscriptionTicket(ticket, trigger, 'transcription-error'))
|
||||
return
|
||||
|
||||
lastError.value = error
|
||||
log('error', 'recording-transcription-error', 'Transcription provider threw while processing recorder-backed segment.', { trigger, error })
|
||||
await options.onTranscriptionError?.({ ...readyEvent, error })
|
||||
return
|
||||
}
|
||||
finally {
|
||||
markTranscriptionFinished()
|
||||
}
|
||||
|
||||
if (isStaleTranscriptionTicket(ticket, trigger, 'transcription-result'))
|
||||
return
|
||||
|
||||
const resultEvent = { ...readyEvent, text }
|
||||
const afterGate = await options.inspectAfterTranscription?.(resultEvent)
|
||||
if (isStaleTranscriptionTicket(ticket, trigger, 'after-transcription-gate'))
|
||||
return
|
||||
|
||||
if (afterGate?.skip) {
|
||||
log('info', 'recording-drop-after-asr', 'Dropping stale transcription result after transcription request.', {
|
||||
trigger,
|
||||
gate: afterGate,
|
||||
text,
|
||||
})
|
||||
await options.onRecordingSkipped?.({ ...resultEvent, gate: afterGate })
|
||||
return
|
||||
}
|
||||
|
||||
if (!text || !text.trim()) {
|
||||
log('warn', 'recording-transcription-empty', 'Transcription provider returned empty text for recorder-backed segment.', { trigger, text })
|
||||
await options.onTranscriptionEmpty?.(resultEvent)
|
||||
return
|
||||
}
|
||||
|
||||
lastTranscriptionText.value = text
|
||||
await options.onTranscriptionResult?.(resultEvent)
|
||||
}
|
||||
|
||||
recorder.onStopRecord(async (recording) => {
|
||||
if (discardNextRecording) {
|
||||
discardNextRecording = false
|
||||
return
|
||||
}
|
||||
|
||||
const segment = stoppedRecordingSegments.shift()
|
||||
const trigger = segment?.trigger ?? activeRecordingTrigger.value ?? 'manual'
|
||||
await transcriptionChain
|
||||
.enqueue(ticket => processRecording(recording, trigger, ticket))
|
||||
.catch((error) => {
|
||||
lastError.value = error
|
||||
log('error', 'recording-processing-error', 'Voice input recording processing failed.', { trigger, error })
|
||||
})
|
||||
})
|
||||
|
||||
function stopVolumeFallback() {
|
||||
if (volumeFallbackAnimationFrame !== undefined) {
|
||||
cancelAnimationFrame(volumeFallbackAnimationFrame)
|
||||
volumeFallbackAnimationFrame = undefined
|
||||
}
|
||||
|
||||
volumeFallbackSourceNode?.disconnect()
|
||||
volumeFallbackAnalyserNode?.disconnect()
|
||||
volumeFallbackSilentGainNode?.disconnect()
|
||||
volumeFallbackSourceNode = undefined
|
||||
volumeFallbackAnalyserNode = undefined
|
||||
volumeFallbackSilentGainNode = undefined
|
||||
volumeFallbackDataArray = undefined
|
||||
volumeFallbackSpeechFrames = 0
|
||||
volumeFallbackLastSpeechAt = 0
|
||||
volumeFallbackLastLogAt = 0
|
||||
|
||||
if (volumeFallbackAudioContext && volumeFallbackAudioContext.state !== 'closed')
|
||||
void volumeFallbackAudioContext.close()
|
||||
volumeFallbackAudioContext = undefined
|
||||
}
|
||||
|
||||
async function startVolumeFallback(stream: MediaStream) {
|
||||
if (!volumeFallbackEnabled.value || shouldUseStreamInput.value)
|
||||
return
|
||||
|
||||
stopVolumeFallback()
|
||||
|
||||
const startThreshold = options.volumeFallback?.startThreshold ?? DEFAULT_VOLUME_FALLBACK_START_THRESHOLD
|
||||
const stopThreshold = options.volumeFallback?.stopThreshold ?? DEFAULT_VOLUME_FALLBACK_STOP_THRESHOLD
|
||||
const startFrames = options.volumeFallback?.startFrames ?? DEFAULT_VOLUME_FALLBACK_START_FRAMES
|
||||
const stopDelayMs = options.volumeFallback?.stopDelayMs ?? DEFAULT_VOLUME_FALLBACK_STOP_DELAY_MS
|
||||
const logIntervalMs = options.volumeFallback?.logIntervalMs ?? DEFAULT_VOLUME_FALLBACK_LOG_INTERVAL_MS
|
||||
|
||||
try {
|
||||
volumeFallbackAudioContext = new AudioContext({ latencyHint: 'interactive' })
|
||||
if (volumeFallbackAudioContext.state === 'suspended')
|
||||
await volumeFallbackAudioContext.resume()
|
||||
|
||||
volumeFallbackSourceNode = volumeFallbackAudioContext.createMediaStreamSource(stream)
|
||||
volumeFallbackAnalyserNode = volumeFallbackAudioContext.createAnalyser()
|
||||
volumeFallbackAnalyserNode.fftSize = 512
|
||||
volumeFallbackAnalyserNode.smoothingTimeConstant = 0.25
|
||||
volumeFallbackSilentGainNode = volumeFallbackAudioContext.createGain()
|
||||
volumeFallbackSilentGainNode.gain.value = 0
|
||||
volumeFallbackDataArray = new Uint8Array(volumeFallbackAnalyserNode.fftSize) as Uint8Array<ArrayBuffer>
|
||||
|
||||
volumeFallbackSourceNode.connect(volumeFallbackAnalyserNode)
|
||||
volumeFallbackAnalyserNode.connect(volumeFallbackSilentGainNode)
|
||||
volumeFallbackSilentGainNode.connect(volumeFallbackAudioContext.destination)
|
||||
|
||||
log('info', 'volume-fallback-started', 'Volume-based recorder fallback started for record-then-transcribe voice input.', {
|
||||
startThreshold,
|
||||
stopThreshold,
|
||||
stopDelayMs,
|
||||
})
|
||||
|
||||
const analyze = () => {
|
||||
if (!volumeFallbackAnalyserNode || !volumeFallbackDataArray)
|
||||
return
|
||||
|
||||
volumeFallbackAnalyserNode.getByteTimeDomainData(volumeFallbackDataArray)
|
||||
const level = calculateTimeDomainVolumeLevel(volumeFallbackDataArray)
|
||||
const now = Date.now()
|
||||
|
||||
if (now - volumeFallbackLastLogAt >= logIntervalMs) {
|
||||
volumeFallbackLastLogAt = now
|
||||
log('info', 'volume-fallback-level', 'Volume fallback sampled microphone input.', {
|
||||
level: Number(level.toFixed(1)),
|
||||
isRecording: isRecording.value,
|
||||
activeRecordingTrigger: activeRecordingTrigger.value,
|
||||
startThreshold,
|
||||
stopThreshold,
|
||||
})
|
||||
}
|
||||
|
||||
if (shouldUseStreamInput.value) {
|
||||
volumeFallbackSpeechFrames = 0
|
||||
volumeFallbackAnimationFrame = requestAnimationFrame(analyze)
|
||||
return
|
||||
}
|
||||
|
||||
if (!isRecording.value) {
|
||||
if (level >= startThreshold) {
|
||||
volumeFallbackSpeechFrames += 1
|
||||
if (volumeFallbackSpeechFrames >= startFrames) {
|
||||
volumeFallbackLastSpeechAt = now
|
||||
volumeFallbackSpeechFrames = 0
|
||||
log('info', 'volume-fallback-speech-start', 'Volume fallback detected speech; starting recorder segment.', {
|
||||
level: Number(level.toFixed(1)),
|
||||
})
|
||||
void startSegment('volume')
|
||||
}
|
||||
}
|
||||
else {
|
||||
volumeFallbackSpeechFrames = 0
|
||||
}
|
||||
}
|
||||
else if (activeRecordingTrigger.value === 'volume' || activeRecordingTrigger.value === 'vad') {
|
||||
if (level > stopThreshold) {
|
||||
volumeFallbackLastSpeechAt = now
|
||||
}
|
||||
else if (!volumeFallbackLastSpeechAt) {
|
||||
volumeFallbackLastSpeechAt = now
|
||||
}
|
||||
else if (volumeFallbackLastSpeechAt && now - volumeFallbackLastSpeechAt >= stopDelayMs) {
|
||||
const trigger = activeRecordingTrigger.value
|
||||
volumeFallbackLastSpeechAt = 0
|
||||
log('info', 'volume-fallback-speech-end', 'Volume fallback detected silence; finalizing recorder segment.', {
|
||||
level: Number(level.toFixed(1)),
|
||||
silenceMs: stopDelayMs,
|
||||
trigger,
|
||||
})
|
||||
void stopSegment(trigger)
|
||||
}
|
||||
}
|
||||
|
||||
volumeFallbackAnimationFrame = requestAnimationFrame(analyze)
|
||||
}
|
||||
|
||||
volumeFallbackAnimationFrame = requestAnimationFrame(analyze)
|
||||
}
|
||||
catch (error) {
|
||||
stopVolumeFallback()
|
||||
lastError.value = error
|
||||
log('error', 'volume-fallback-start-failed', 'Failed to start volume-based recorder fallback.', { error })
|
||||
}
|
||||
}
|
||||
|
||||
async function startAutoSegmentation() {
|
||||
const stream = mediaRef.value
|
||||
if (!stream)
|
||||
throw new Error('No microphone stream available for voice input')
|
||||
|
||||
await startVoiceInputVadDetectionSafely({
|
||||
init: initVAD,
|
||||
loaded: () => vadLoaded.value,
|
||||
start: startVAD,
|
||||
stream,
|
||||
getError: () => vadError.value,
|
||||
log,
|
||||
})
|
||||
await startVolumeFallback(stream)
|
||||
}
|
||||
|
||||
async function stop(options: { flushActiveRecording?: boolean } = {}) {
|
||||
stopVolumeFallback()
|
||||
disposeVAD()
|
||||
transcriptionChain.reset()
|
||||
stoppedRecordingSegments.length = 0
|
||||
|
||||
if (options.flushActiveRecording && isRecording.value) {
|
||||
await stopSegment(activeRecordingTrigger.value ?? 'manual')
|
||||
await transcriptionChain.idle()
|
||||
transcriptionChain.reset()
|
||||
}
|
||||
else if (isRecording.value) {
|
||||
discardNextRecording = true
|
||||
try {
|
||||
await recorder.stopRecord()
|
||||
}
|
||||
finally {
|
||||
discardNextRecording = false
|
||||
activeRecordingSegment.value = undefined
|
||||
}
|
||||
}
|
||||
else {
|
||||
activeRecordingSegment.value = undefined
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
isRecording,
|
||||
isTranscribing,
|
||||
lastTranscriptionText,
|
||||
lastError,
|
||||
activeRecordingTrigger,
|
||||
isSpeechVAD,
|
||||
isSpeechProb,
|
||||
isSpeechHistory,
|
||||
vadLoaded,
|
||||
vadError,
|
||||
|
||||
startSegment,
|
||||
stopSegment,
|
||||
startAutoSegmentation,
|
||||
stop,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
|
||||
import { createVoiceInputTranscriptionChain } from './voice-input-transcription-chain'
|
||||
|
||||
describe('createVoiceInputTranscriptionChain', () => {
|
||||
it('runs queued transcriptions in order while the session stays current', async () => {
|
||||
const chain = createVoiceInputTranscriptionChain()
|
||||
const resolved: string[] = []
|
||||
|
||||
let finishFirst!: () => void
|
||||
const first = chain.enqueue(async () => {
|
||||
await new Promise<void>((resolve) => {
|
||||
finishFirst = resolve
|
||||
})
|
||||
resolved.push('first')
|
||||
})
|
||||
const second = chain.enqueue(async () => {
|
||||
resolved.push('second')
|
||||
})
|
||||
|
||||
await Promise.resolve()
|
||||
await Promise.resolve()
|
||||
|
||||
expect(resolved).toEqual([])
|
||||
|
||||
finishFirst()
|
||||
await first
|
||||
await second
|
||||
|
||||
expect(resolved).toEqual(['first', 'second'])
|
||||
})
|
||||
|
||||
it('lets fresh transcriptions start after reset without waiting for stale provider work', async () => {
|
||||
const chain = createVoiceInputTranscriptionChain()
|
||||
const resolved: string[] = []
|
||||
|
||||
void chain.enqueue(async () => {
|
||||
await new Promise<void>(() => {})
|
||||
resolved.push('stale')
|
||||
})
|
||||
|
||||
await Promise.resolve()
|
||||
chain.reset()
|
||||
|
||||
await chain.enqueue(async (ticket) => {
|
||||
expect(ticket.isCurrent()).toBe(true)
|
||||
resolved.push('fresh')
|
||||
})
|
||||
|
||||
expect(resolved).toEqual(['fresh'])
|
||||
})
|
||||
|
||||
it('marks running tickets stale after reset so late results cannot publish', async () => {
|
||||
const chain = createVoiceInputTranscriptionChain()
|
||||
let finishFirst!: () => void
|
||||
let firstTicketStillCurrent = true
|
||||
|
||||
const first = chain.enqueue(async (ticket) => {
|
||||
await new Promise<void>((resolve) => {
|
||||
finishFirst = resolve
|
||||
})
|
||||
firstTicketStillCurrent = ticket.isCurrent()
|
||||
})
|
||||
|
||||
await Promise.resolve()
|
||||
chain.reset()
|
||||
finishFirst()
|
||||
await first
|
||||
|
||||
expect(firstTicketStillCurrent).toBe(false)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,54 @@
|
||||
export interface VoiceInputTranscriptionTicket {
|
||||
/** Returns whether this queued transcription still belongs to the active listening session. */
|
||||
isCurrent: () => boolean
|
||||
}
|
||||
|
||||
export interface VoiceInputTranscriptionChain {
|
||||
/** Runs work after earlier current transcription tasks have settled. */
|
||||
enqueue: <T>(task: (ticket: VoiceInputTranscriptionTicket) => Promise<T> | T) => Promise<T | undefined>
|
||||
/** Invalidates pending/running tickets and lets future work start from a fresh tail. */
|
||||
reset: () => void
|
||||
/** Resolves when all currently chained transcription work has settled. */
|
||||
idle: () => Promise<void>
|
||||
}
|
||||
|
||||
export function createVoiceInputTranscriptionChain(): VoiceInputTranscriptionChain {
|
||||
let tail = Promise.resolve()
|
||||
let generation = 0
|
||||
|
||||
function enqueue<T>(task: (ticket: VoiceInputTranscriptionTicket) => Promise<T> | T) {
|
||||
const taskGeneration = generation
|
||||
const ticket: VoiceInputTranscriptionTicket = {
|
||||
isCurrent: () => taskGeneration === generation,
|
||||
}
|
||||
|
||||
const run = tail.then(async () => {
|
||||
if (!ticket.isCurrent())
|
||||
return undefined
|
||||
|
||||
return task(ticket)
|
||||
})
|
||||
|
||||
tail = run.then(
|
||||
() => undefined,
|
||||
() => undefined,
|
||||
)
|
||||
|
||||
return run
|
||||
}
|
||||
|
||||
function reset() {
|
||||
generation += 1
|
||||
tail = Promise.resolve()
|
||||
}
|
||||
|
||||
function idle() {
|
||||
return tail
|
||||
}
|
||||
|
||||
return {
|
||||
enqueue,
|
||||
reset,
|
||||
idle,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
|
||||
import { startVoiceInputVadDetectionSafely } from './voice-input-vad-startup'
|
||||
|
||||
describe('voice input VAD startup', () => {
|
||||
it('returns false and logs when VAD initialization throws', async () => {
|
||||
const init = vi.fn().mockRejectedValue(new Error('vad unavailable'))
|
||||
const start = vi.fn()
|
||||
const log = vi.fn()
|
||||
|
||||
await expect(startVoiceInputVadDetectionSafely({
|
||||
init,
|
||||
loaded: () => false,
|
||||
start,
|
||||
stream: {} as MediaStream,
|
||||
log,
|
||||
})).resolves.toBe(false)
|
||||
|
||||
expect(start).not.toHaveBeenCalled()
|
||||
expect(log).toHaveBeenCalledWith(
|
||||
'error',
|
||||
'vad-init-failed',
|
||||
'VAD initialization failed.',
|
||||
expect.objectContaining({
|
||||
error: expect.any(Error),
|
||||
}),
|
||||
)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,38 @@
|
||||
import type { VoiceInputSessionLogLevel } from './voice-input-session'
|
||||
|
||||
export interface VoiceInputVadStartupOptions {
|
||||
init: () => Promise<void>
|
||||
loaded: () => boolean
|
||||
start: (stream: MediaStream) => Promise<void>
|
||||
stream: MediaStream
|
||||
getError?: () => unknown
|
||||
log?: (level: VoiceInputSessionLogLevel, event: string, message: string, details?: Record<string, unknown>) => void
|
||||
}
|
||||
|
||||
export async function startVoiceInputVadDetectionSafely(options: VoiceInputVadStartupOptions) {
|
||||
try {
|
||||
await options.init()
|
||||
|
||||
if (options.loaded()) {
|
||||
options.log?.('info', 'vad-start', 'VAD initialized successfully; starting against microphone stream.', {
|
||||
stream: options.stream,
|
||||
})
|
||||
await options.start(options.stream)
|
||||
return true
|
||||
}
|
||||
|
||||
const error = options.getError?.()
|
||||
if (error) {
|
||||
options.log?.('error', 'vad-init-failed', 'VAD initialization failed.', {
|
||||
error,
|
||||
})
|
||||
}
|
||||
}
|
||||
catch (error) {
|
||||
options.log?.('error', 'vad-init-failed', 'VAD initialization failed.', {
|
||||
error,
|
||||
})
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
import type { BaseVAD } from './vad'
|
||||
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
|
||||
import { createVADStates } from './vad'
|
||||
|
||||
class FakeAudioNode {
|
||||
connect = vi.fn()
|
||||
disconnect = vi.fn()
|
||||
port = { onmessage: null as ((event: MessageEvent) => void) | null }
|
||||
}
|
||||
|
||||
class FakeAudioContext {
|
||||
state: AudioContextState = 'running'
|
||||
destination = new FakeAudioNode()
|
||||
audioWorklet = {
|
||||
addModule: vi.fn(async () => {}),
|
||||
}
|
||||
|
||||
createMediaStreamSource = vi.fn(() => new FakeAudioNode())
|
||||
createGain = vi.fn(() => ({
|
||||
gain: { value: 1 },
|
||||
connect: vi.fn(),
|
||||
disconnect: vi.fn(),
|
||||
}))
|
||||
|
||||
async resume() {
|
||||
this.state = 'running'
|
||||
}
|
||||
|
||||
suspend = vi.fn(async () => {
|
||||
this.state = 'suspended'
|
||||
})
|
||||
|
||||
close = vi.fn(async () => {
|
||||
this.state = 'closed'
|
||||
})
|
||||
}
|
||||
|
||||
class FakeAudioWorkletNode extends FakeAudioNode {
|
||||
constructor() {
|
||||
super()
|
||||
}
|
||||
}
|
||||
|
||||
function createVADMock(): BaseVAD {
|
||||
return {
|
||||
initialize: vi.fn(async () => {}),
|
||||
processAudio: vi.fn(async () => {}),
|
||||
on: vi.fn(),
|
||||
off: vi.fn(),
|
||||
}
|
||||
}
|
||||
|
||||
describe('createVADStates', () => {
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals()
|
||||
})
|
||||
|
||||
it('does not stop the caller-owned microphone stream when disposing VAD nodes', async () => {
|
||||
// NOTICE:
|
||||
// Vitest node tests do not provide Web Audio constructors.
|
||||
// The regression is about our ownership policy around a caller-owned MediaStream, not browser audio.
|
||||
// Source/context: packages/stage-ui/src/libs/audio/vad.ts dispose previously called track.stop().
|
||||
// Removal condition: replace this with a browser-mode Web Audio lifecycle test.
|
||||
vi.stubGlobal('AudioContext', FakeAudioContext)
|
||||
vi.stubGlobal('AudioWorkletNode', FakeAudioWorkletNode)
|
||||
const stop = vi.fn()
|
||||
const stream = {
|
||||
getTracks: () => [{ stop }],
|
||||
} as unknown as MediaStream
|
||||
|
||||
const manager = createVADStates(createVADMock(), '/vad-worklet.js')
|
||||
await manager.initialize()
|
||||
await manager.start(stream)
|
||||
manager.dispose()
|
||||
|
||||
expect(stop).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('disconnects the previous microphone source before starting a new graph', async () => {
|
||||
// NOTICE:
|
||||
// The page can call start from both the init continuation and the stream/loaded watcher.
|
||||
// This fake Web Audio graph keeps the regression focused on duplicate source-node wiring.
|
||||
// Source/context: apps/stage-tamagotchi/src/renderer/pages/index.vue can restart VAD around stream changes.
|
||||
// Removal condition: replace this with browser-mode Web Audio graph lifecycle coverage.
|
||||
const createdSources: FakeAudioNode[] = []
|
||||
class ReconnectAudioContext extends FakeAudioContext {
|
||||
createMediaStreamSource = vi.fn(() => {
|
||||
const source = new FakeAudioNode()
|
||||
createdSources.push(source)
|
||||
return source
|
||||
})
|
||||
}
|
||||
|
||||
vi.stubGlobal('AudioContext', ReconnectAudioContext)
|
||||
vi.stubGlobal('AudioWorkletNode', FakeAudioWorkletNode)
|
||||
const stream = {
|
||||
getTracks: () => [],
|
||||
} as unknown as MediaStream
|
||||
|
||||
const manager = createVADStates(createVADMock(), '/vad-worklet.js')
|
||||
await manager.initialize()
|
||||
await manager.start(stream)
|
||||
await manager.start(stream)
|
||||
|
||||
expect(createdSources).toHaveLength(2)
|
||||
expect(createdSources[0].disconnect).toHaveBeenCalledTimes(1)
|
||||
expect(createdSources[1].disconnect).not.toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
@@ -60,6 +60,7 @@ export function createVADStates(vad: BaseVAD, vadAudioWorkletUrl: string, option
|
||||
let audioWorkletNode: AudioWorkletNode | null
|
||||
let mediaStream: MediaStream | null
|
||||
let sourceNode: MediaStreamAudioSourceNode | null
|
||||
let silentGainNode: GainNode | null
|
||||
let workletInitialized: boolean
|
||||
|
||||
const {
|
||||
@@ -96,6 +97,20 @@ export function createVADStates(vad: BaseVAD, vadAudioWorkletUrl: string, option
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Disconnects caller-owned microphone graph nodes before rebuilding the input graph.
|
||||
*/
|
||||
function disconnectInputGraph() {
|
||||
if (sourceNode) {
|
||||
sourceNode.disconnect()
|
||||
sourceNode = null
|
||||
}
|
||||
if (silentGainNode) {
|
||||
silentGainNode.disconnect()
|
||||
silentGainNode = null
|
||||
}
|
||||
}
|
||||
|
||||
async function start(stream: MediaStream) {
|
||||
if (!audioContext || !audioWorkletNode) {
|
||||
throw new Error('Audio system not initialized. Call initialize() first.')
|
||||
@@ -107,6 +122,7 @@ export function createVADStates(vad: BaseVAD, vadAudioWorkletUrl: string, option
|
||||
}
|
||||
|
||||
// Request microphone access
|
||||
disconnectInputGraph()
|
||||
mediaStream = stream
|
||||
|
||||
// Create source node and connect to worklet
|
||||
@@ -115,10 +131,10 @@ export function createVADStates(vad: BaseVAD, vadAudioWorkletUrl: string, option
|
||||
|
||||
// Connect worklet to a silent destination (to keep the audio graph active)
|
||||
// Using a GainNode with gain=0 to ensure no sound is output
|
||||
const silentGain = audioContext.createGain()
|
||||
silentGain.gain.value = 0
|
||||
audioWorkletNode.connect(silentGain)
|
||||
silentGain.connect(audioContext.destination)
|
||||
silentGainNode = audioContext.createGain()
|
||||
silentGainNode.gain.value = 0
|
||||
audioWorkletNode.connect(silentGainNode)
|
||||
silentGainNode.connect(audioContext.destination)
|
||||
}
|
||||
catch (error) {
|
||||
console.error('Failed to start microphone:', error)
|
||||
@@ -133,18 +149,14 @@ export function createVADStates(vad: BaseVAD, vadAudioWorkletUrl: string, option
|
||||
}
|
||||
|
||||
function dispose() {
|
||||
if (sourceNode) {
|
||||
sourceNode.disconnect()
|
||||
sourceNode = null
|
||||
}
|
||||
disconnectInputGraph()
|
||||
if (audioWorkletNode) {
|
||||
audioWorkletNode.disconnect()
|
||||
audioWorkletNode = null
|
||||
}
|
||||
if (mediaStream) {
|
||||
mediaStream.getTracks().forEach(track => track.stop())
|
||||
mediaStream = null
|
||||
}
|
||||
// The MediaStream is owned by the caller (settings audio device store). VAD only borrows it
|
||||
// to build an AudioNode graph, so disposing VAD must not stop the microphone device itself.
|
||||
mediaStream = null
|
||||
if (audioContext && audioContext.state !== 'closed') {
|
||||
audioContext.close()
|
||||
}
|
||||
|
||||
@@ -17,11 +17,17 @@ export function withCredentials() {
|
||||
const chatSession = getActivePinia() ? useChatSessionStore() : null
|
||||
if (chatSession?.activeSessionId)
|
||||
headers.set('x-airi-session-id', chatSession.activeSessionId)
|
||||
return globalThis.fetch(input, {
|
||||
|
||||
const requestInit = {
|
||||
...init,
|
||||
headers,
|
||||
credentials: 'omit',
|
||||
})
|
||||
} as RequestInit & { duplex?: 'half' }
|
||||
|
||||
if (init?.body instanceof ReadableStream)
|
||||
requestInit.duplex = 'half'
|
||||
|
||||
return globalThis.fetch(input, requestInit)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
import type {
|
||||
PlaybackEndEvent,
|
||||
PlaybackInterruptEvent,
|
||||
PlaybackItem,
|
||||
PlaybackRejectEvent,
|
||||
PlaybackStartEvent,
|
||||
} from '@proj-airi/pipelines-audio'
|
||||
|
||||
import { describe, expect, it } from 'vitest'
|
||||
|
||||
import { bindSpeakingStateToPlaybackManager } from './playback-speaking-state'
|
||||
|
||||
function createPlaybackItem(): PlaybackItem<AudioBuffer> {
|
||||
return {
|
||||
id: 'playback-1',
|
||||
streamId: 'stream-1',
|
||||
intentId: 'intent-1',
|
||||
segmentId: 'segment-1',
|
||||
sequence: 1,
|
||||
priority: 0,
|
||||
text: 'hello',
|
||||
special: null,
|
||||
audio: {} as AudioBuffer,
|
||||
createdAt: 1000,
|
||||
}
|
||||
}
|
||||
|
||||
function createFakePlaybackManager() {
|
||||
const listeners = {
|
||||
start: [] as Array<(event: PlaybackStartEvent<AudioBuffer>) => void>,
|
||||
end: [] as Array<(event: PlaybackEndEvent<AudioBuffer>) => void>,
|
||||
interrupt: [] as Array<(event: PlaybackInterruptEvent<AudioBuffer>) => void>,
|
||||
reject: [] as Array<(event: PlaybackRejectEvent<AudioBuffer>) => void>,
|
||||
}
|
||||
|
||||
return {
|
||||
listeners,
|
||||
manager: {
|
||||
onStart: (listener: (event: PlaybackStartEvent<AudioBuffer>) => void) => {
|
||||
listeners.start.push(listener)
|
||||
},
|
||||
onEnd: (listener: (event: PlaybackEndEvent<AudioBuffer>) => void) => {
|
||||
listeners.end.push(listener)
|
||||
},
|
||||
onInterrupt: (listener: (event: PlaybackInterruptEvent<AudioBuffer>) => void) => {
|
||||
listeners.interrupt.push(listener)
|
||||
},
|
||||
onReject: (listener: (event: PlaybackRejectEvent<AudioBuffer>) => void) => {
|
||||
listeners.reject.push(listener)
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
describe('bindSpeakingStateToPlaybackManager', () => {
|
||||
it('resets speaking state when playback is interrupted', () => {
|
||||
const playback = createFakePlaybackManager()
|
||||
let speaking = false
|
||||
|
||||
bindSpeakingStateToPlaybackManager(playback.manager, {
|
||||
setSpeaking: (value) => {
|
||||
speaking = value
|
||||
},
|
||||
})
|
||||
|
||||
const item = createPlaybackItem()
|
||||
playback.listeners.start.forEach(listener => listener({ item, startedAt: 1000 }))
|
||||
expect(speaking).toBe(true)
|
||||
|
||||
playback.listeners.interrupt.forEach(listener => listener({ item, reason: 'playback-error', interruptedAt: 1100 }))
|
||||
expect(speaking).toBe(false)
|
||||
})
|
||||
|
||||
it('resets speaking state when playback is rejected before it can finish', () => {
|
||||
const playback = createFakePlaybackManager()
|
||||
let speaking = true
|
||||
|
||||
bindSpeakingStateToPlaybackManager(playback.manager, {
|
||||
setSpeaking: (value) => {
|
||||
speaking = value
|
||||
},
|
||||
})
|
||||
|
||||
const item = createPlaybackItem()
|
||||
playback.listeners.reject.forEach(listener => listener({ item, reason: 'overflow' }))
|
||||
|
||||
expect(speaking).toBe(false)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,53 @@
|
||||
import type {
|
||||
PlaybackEndEvent,
|
||||
PlaybackInterruptEvent,
|
||||
PlaybackRejectEvent,
|
||||
PlaybackStartEvent,
|
||||
} from '@proj-airi/pipelines-audio'
|
||||
|
||||
export interface PlaybackSpeakingStateManager<TAudio> {
|
||||
onStart: (listener: (event: PlaybackStartEvent<TAudio>) => void) => void
|
||||
onEnd: (listener: (event: PlaybackEndEvent<TAudio>) => void) => void
|
||||
onInterrupt: (listener: (event: PlaybackInterruptEvent<TAudio>) => void) => void
|
||||
onReject: (listener: (event: PlaybackRejectEvent<TAudio>) => void) => void
|
||||
}
|
||||
|
||||
export interface PlaybackSpeakingStateHandlers<TAudio> {
|
||||
setSpeaking: (value: boolean) => void
|
||||
onStart?: (event: PlaybackStartEvent<TAudio>) => void
|
||||
}
|
||||
|
||||
/**
|
||||
* Binds assistant speaking state to every terminal playback outcome.
|
||||
*
|
||||
* Use when:
|
||||
* - UI state must show whether assistant audio is currently audible.
|
||||
* - Voice input should be suspended only while playback is actually active.
|
||||
*
|
||||
* Expects:
|
||||
* - Playback managers emit exactly one terminal event for each accepted item.
|
||||
*
|
||||
* Returns:
|
||||
* - Nothing; listeners are registered on the provided manager.
|
||||
*/
|
||||
export function bindSpeakingStateToPlaybackManager<TAudio>(
|
||||
manager: PlaybackSpeakingStateManager<TAudio>,
|
||||
handlers: PlaybackSpeakingStateHandlers<TAudio>,
|
||||
) {
|
||||
manager.onStart((event) => {
|
||||
handlers.setSpeaking(true)
|
||||
handlers.onStart?.(event)
|
||||
})
|
||||
|
||||
manager.onEnd(() => {
|
||||
handlers.setSpeaking(false)
|
||||
})
|
||||
|
||||
manager.onInterrupt(() => {
|
||||
handlers.setSpeaking(false)
|
||||
})
|
||||
|
||||
manager.onReject(() => {
|
||||
handlers.setSpeaking(false)
|
||||
})
|
||||
}
|
||||
@@ -5,17 +5,21 @@ import { resolveVADConfig } from './vad'
|
||||
describe('resolveVADConfig', () => {
|
||||
it('uses safer defaults for threshold and silence duration', () => {
|
||||
expect(resolveVADConfig()).toEqual({
|
||||
speechThreshold: 0.6,
|
||||
exitThreshold: 0.18,
|
||||
minSilenceDurationMs: 800,
|
||||
speechThreshold: 0.52,
|
||||
exitThreshold: 0.156,
|
||||
minSilenceDurationMs: 1200,
|
||||
speechPadMs: 360,
|
||||
minSpeechDurationMs: 300,
|
||||
})
|
||||
})
|
||||
|
||||
it('preserves explicit threshold and silence duration values', () => {
|
||||
expect(resolveVADConfig(0.45, 650)).toEqual({
|
||||
expect(resolveVADConfig(0.45, 650, 420, 500)).toEqual({
|
||||
speechThreshold: 0.45,
|
||||
exitThreshold: 0.135,
|
||||
minSilenceDurationMs: 650,
|
||||
speechPadMs: 420,
|
||||
minSpeechDurationMs: 500,
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@@ -11,21 +11,33 @@ import { createVAD, createVADStates } from '../../../workers/vad'
|
||||
interface UseVADOptions {
|
||||
threshold?: MaybeRefOrGetter<number>
|
||||
minSilenceDurationMs?: MaybeRefOrGetter<number>
|
||||
speechPadMs?: MaybeRefOrGetter<number>
|
||||
minSpeechDurationMs?: MaybeRefOrGetter<number>
|
||||
|
||||
onSpeechStart?: () => void
|
||||
onSpeechEnd?: () => void
|
||||
onSpeechReady?: (event: { buffer: Float32Array, duration: number }) => void
|
||||
}
|
||||
|
||||
const DEFAULT_VAD_THRESHOLD = 0.6
|
||||
const DEFAULT_VAD_MIN_SILENCE_DURATION_MS = 800
|
||||
const DEFAULT_VAD_THRESHOLD = 0.52
|
||||
const DEFAULT_VAD_MIN_SILENCE_DURATION_MS = 1200
|
||||
const DEFAULT_VAD_SPEECH_PAD_MS = 360
|
||||
const DEFAULT_VAD_MIN_SPEECH_DURATION_MS = 300
|
||||
|
||||
export function resolveVADConfig(threshold?: number, minSilenceDurationMs?: number): Pick<BaseVADConfig, 'speechThreshold' | 'exitThreshold' | 'minSilenceDurationMs'> {
|
||||
export function resolveVADConfig(
|
||||
threshold?: number,
|
||||
minSilenceDurationMs?: number,
|
||||
speechPadMs?: number,
|
||||
minSpeechDurationMs?: number,
|
||||
): Pick<BaseVADConfig, 'speechThreshold' | 'exitThreshold' | 'minSilenceDurationMs' | 'speechPadMs' | 'minSpeechDurationMs'> {
|
||||
const resolvedThreshold = threshold ?? DEFAULT_VAD_THRESHOLD
|
||||
|
||||
return {
|
||||
speechThreshold: resolvedThreshold,
|
||||
exitThreshold: resolvedThreshold * 0.3,
|
||||
minSilenceDurationMs: minSilenceDurationMs ?? DEFAULT_VAD_MIN_SILENCE_DURATION_MS,
|
||||
speechPadMs: speechPadMs ?? DEFAULT_VAD_SPEECH_PAD_MS,
|
||||
minSpeechDurationMs: minSpeechDurationMs ?? DEFAULT_VAD_MIN_SPEECH_DURATION_MS,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -33,6 +45,8 @@ export function useVAD(workerUrl: string, options?: UseVADOptions) {
|
||||
const defaultOptions: UseVADOptions = {
|
||||
threshold: ref(DEFAULT_VAD_THRESHOLD),
|
||||
minSilenceDurationMs: ref(DEFAULT_VAD_MIN_SILENCE_DURATION_MS),
|
||||
speechPadMs: ref(DEFAULT_VAD_SPEECH_PAD_MS),
|
||||
minSpeechDurationMs: ref(DEFAULT_VAD_MIN_SPEECH_DURATION_MS),
|
||||
}
|
||||
|
||||
options = merge(defaultOptions, options)
|
||||
@@ -51,6 +65,8 @@ export function useVAD(workerUrl: string, options?: UseVADOptions) {
|
||||
|
||||
const threshold = toRef(options.threshold)
|
||||
const minSilenceDurationMs = toRef(options.minSilenceDurationMs)
|
||||
const speechPadMs = toRef(options.speechPadMs)
|
||||
const minSpeechDurationMs = toRef(options.minSpeechDurationMs)
|
||||
|
||||
async function init() {
|
||||
if (loaded.value || loading.value || manager.value)
|
||||
@@ -60,7 +76,12 @@ export function useVAD(workerUrl: string, options?: UseVADOptions) {
|
||||
inferenceError.value = ''
|
||||
|
||||
try {
|
||||
const vadConfig = resolveVADConfig(threshold.value, minSilenceDurationMs.value)
|
||||
const vadConfig = resolveVADConfig(
|
||||
threshold.value,
|
||||
minSilenceDurationMs.value,
|
||||
speechPadMs.value,
|
||||
minSpeechDurationMs.value,
|
||||
)
|
||||
|
||||
vad.value = await createVAD({
|
||||
sampleRate: 16000,
|
||||
@@ -78,6 +99,10 @@ export function useVAD(workerUrl: string, options?: UseVADOptions) {
|
||||
options?.onSpeechEnd?.()
|
||||
})
|
||||
|
||||
vad.value.on('speech-ready', (event) => {
|
||||
options?.onSpeechReady?.(event)
|
||||
})
|
||||
|
||||
vad.value.on('debug', ({ data }) => {
|
||||
if (data?.probability !== undefined) {
|
||||
isSpeechProb.value = data.probability
|
||||
@@ -149,6 +174,18 @@ export function useVAD(workerUrl: string, options?: UseVADOptions) {
|
||||
}
|
||||
})
|
||||
|
||||
watch(speechPadMs, (newVal) => {
|
||||
if (vad.value && newVal !== undefined) {
|
||||
vad.value.updateConfig({ speechPadMs: newVal })
|
||||
}
|
||||
})
|
||||
|
||||
watch(minSpeechDurationMs, (newVal) => {
|
||||
if (vad.value && newVal !== undefined) {
|
||||
vad.value.updateConfig({ minSpeechDurationMs: newVal })
|
||||
}
|
||||
})
|
||||
|
||||
return {
|
||||
isSpeech,
|
||||
isSpeechProb,
|
||||
@@ -158,6 +195,8 @@ export function useVAD(workerUrl: string, options?: UseVADOptions) {
|
||||
inferenceError,
|
||||
threshold,
|
||||
minSilenceDurationMs,
|
||||
speechPadMs,
|
||||
minSpeechDurationMs,
|
||||
|
||||
init,
|
||||
start,
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
import { createPinia, setActivePinia } from 'pinia'
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
|
||||
const analyticsMock = vi.hoisted(() => ({
|
||||
allowComposableCall: true,
|
||||
trackSttFailed: vi.fn(),
|
||||
trackSttStarted: vi.fn(),
|
||||
trackSttSucceeded: vi.fn(),
|
||||
}))
|
||||
|
||||
vi.mock('../../composables/use-analytics', () => ({
|
||||
useAnalytics: () => {
|
||||
if (!analyticsMock.allowComposableCall)
|
||||
throw new Error('Must be called at the top of a `setup` function')
|
||||
|
||||
return {
|
||||
trackSttFailed: analyticsMock.trackSttFailed,
|
||||
trackSttStarted: analyticsMock.trackSttStarted,
|
||||
trackSttSucceeded: analyticsMock.trackSttSucceeded,
|
||||
}
|
||||
},
|
||||
}))
|
||||
|
||||
vi.mock('@xsai/generate-transcription', () => ({
|
||||
generateTranscription: vi.fn(async () => ({ text: 'hello' })),
|
||||
}))
|
||||
|
||||
vi.mock('vue-i18n', () => ({
|
||||
useI18n: () => ({
|
||||
locale: { value: 'en' },
|
||||
t: (_key: string, fallback?: string) => fallback ?? _key,
|
||||
}),
|
||||
}))
|
||||
|
||||
describe('useHearingStore analytics lifecycle', () => {
|
||||
beforeEach(() => {
|
||||
setActivePinia(createPinia())
|
||||
analyticsMock.allowComposableCall = true
|
||||
analyticsMock.trackSttFailed.mockReset()
|
||||
analyticsMock.trackSttStarted.mockReset()
|
||||
analyticsMock.trackSttSucceeded.mockReset()
|
||||
})
|
||||
|
||||
/**
|
||||
* @example
|
||||
* await hearingStore.transcription(providerId, provider, model, file)
|
||||
*/
|
||||
it('does not call analytics composables when a recording is transcribed later', async () => {
|
||||
const { useHearingStore } = await import('./hearing')
|
||||
const hearingStore = useHearingStore()
|
||||
analyticsMock.allowComposableCall = false
|
||||
|
||||
const result = await hearingStore.transcription(
|
||||
'openai-compatible-audio-transcription',
|
||||
{
|
||||
transcription: () => ({}),
|
||||
} as any,
|
||||
'FunAudioLLM/SenseVoiceSmall',
|
||||
new File(['hello'], 'recording.wav', { type: 'audio/wav' }),
|
||||
)
|
||||
|
||||
expect(result.text).toBe('hello')
|
||||
expect(analyticsMock.trackSttStarted).toHaveBeenCalledWith('openai-compatible-audio-transcription')
|
||||
expect(analyticsMock.trackSttSucceeded).toHaveBeenCalledWith({
|
||||
provider: 'openai-compatible-audio-transcription',
|
||||
latency_ms: expect.any(Number),
|
||||
char_count: 5,
|
||||
stream: false,
|
||||
})
|
||||
}, 10000)
|
||||
})
|
||||
@@ -1,6 +1,15 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
|
||||
import { filterTranscriptionByConfidence, resolveStreamTranscriptionExecutor } from './hearing'
|
||||
import {
|
||||
describeEmptyTranscriptionResponse,
|
||||
filterTranscriptionByConfidence,
|
||||
normalizeGeneratedTranscriptionText,
|
||||
resolveActiveTranscriptionModel,
|
||||
resolveActiveTranscriptionProviderError,
|
||||
resolveStreamTranscriptionExecutor,
|
||||
resolveTranscriptionFileName,
|
||||
resolveTranscriptionProviderOptions,
|
||||
} from './hearing'
|
||||
|
||||
describe('filterTranscriptionByConfidence', () => {
|
||||
const segments = [
|
||||
@@ -41,3 +50,111 @@ describe('resolveStreamTranscriptionExecutor', () => {
|
||||
expect(executor).toBe(resolveStreamTranscriptionExecutor('aliyun-nls-transcription'))
|
||||
})
|
||||
})
|
||||
|
||||
describe('resolveActiveTranscriptionProviderError', () => {
|
||||
/**
|
||||
* @example
|
||||
* resolveActiveTranscriptionProviderError('')
|
||||
*/
|
||||
it('returns a clear setup error when no transcription provider is selected', () => {
|
||||
expect(resolveActiveTranscriptionProviderError('')).toBe('No active transcription provider selected. Select a provider in Settings > Hearing.')
|
||||
})
|
||||
|
||||
/**
|
||||
* @example
|
||||
* resolveActiveTranscriptionProviderError('openai-compatible-audio-transcription')
|
||||
*/
|
||||
it('allows a selected transcription provider', () => {
|
||||
expect(resolveActiveTranscriptionProviderError('openai-compatible-audio-transcription')).toBeUndefined()
|
||||
})
|
||||
})
|
||||
|
||||
describe('resolveActiveTranscriptionModel', () => {
|
||||
/**
|
||||
* @example
|
||||
* resolveActiveTranscriptionModel('', { model: 'FunAudioLLM/SenseVoiceSmall' })
|
||||
*/
|
||||
it('uses the provider config model when the hearing model has not been synced', () => {
|
||||
expect(resolveActiveTranscriptionModel('', { model: 'FunAudioLLM/SenseVoiceSmall' })).toBe('FunAudioLLM/SenseVoiceSmall')
|
||||
})
|
||||
|
||||
/**
|
||||
* @example
|
||||
* resolveActiveTranscriptionModel('whisper-1', { model: 'FunAudioLLM/SenseVoiceSmall' })
|
||||
*/
|
||||
it('prefers the explicit hearing model over the provider config model', () => {
|
||||
expect(resolveActiveTranscriptionModel('whisper-1', { model: 'FunAudioLLM/SenseVoiceSmall' })).toBe('whisper-1')
|
||||
})
|
||||
})
|
||||
|
||||
describe('resolveTranscriptionProviderOptions', () => {
|
||||
/**
|
||||
* @example
|
||||
* resolveTranscriptionProviderOptions({}, 'zh-Hans')
|
||||
*/
|
||||
it('derives a two-letter transcription language from the active UI locale', () => {
|
||||
expect(resolveTranscriptionProviderOptions({}, 'zh-Hans')).toEqual({ language: 'zh' })
|
||||
})
|
||||
|
||||
/**
|
||||
* @example
|
||||
* resolveTranscriptionProviderOptions({ language: 'ja' }, 'zh-Hans')
|
||||
*/
|
||||
it('prefers the provider language when one is configured explicitly', () => {
|
||||
expect(resolveTranscriptionProviderOptions({ language: 'ja' }, 'zh-Hans')).toEqual({ language: 'ja' })
|
||||
})
|
||||
})
|
||||
|
||||
describe('normalizeGeneratedTranscriptionText', () => {
|
||||
/**
|
||||
* @example
|
||||
* normalizeGeneratedTranscriptionText({ result: { text: '你好' } })
|
||||
*/
|
||||
it('reads nested text from OpenAI-compatible provider variants', () => {
|
||||
expect(normalizeGeneratedTranscriptionText({ result: { text: '你好' } })).toBe('你好')
|
||||
})
|
||||
|
||||
/**
|
||||
* @example
|
||||
* normalizeGeneratedTranscriptionText({ segments: [{ text: '你' }, { text: '好' }] })
|
||||
*/
|
||||
it('joins segment text when no top-level text is returned', () => {
|
||||
expect(normalizeGeneratedTranscriptionText({ segments: [{ text: '你' }, { text: '好' }] })).toBe('你好')
|
||||
})
|
||||
|
||||
/**
|
||||
* @example
|
||||
* normalizeGeneratedTranscriptionText({ segments: [{ text: ' Hello' }, { text: ' world' }] })
|
||||
*/
|
||||
it('preserves segment whitespace before trimming the final fallback text', () => {
|
||||
expect(normalizeGeneratedTranscriptionText({ segments: [{ text: ' Hello' }, { text: ' world' }] })).toBe('Hello world')
|
||||
})
|
||||
|
||||
/**
|
||||
* @example
|
||||
* normalizeGeneratedTranscriptionText({ data: { text: '你好' } })
|
||||
*/
|
||||
it('reads data text from provider envelope responses', () => {
|
||||
expect(normalizeGeneratedTranscriptionText({ data: { text: '你好' } })).toBe('你好')
|
||||
})
|
||||
})
|
||||
|
||||
describe('describeEmptyTranscriptionResponse', () => {
|
||||
/**
|
||||
* @example
|
||||
* describeEmptyTranscriptionResponse({ result: { duration: 1 } })
|
||||
*/
|
||||
it('describes response keys when no usable text was returned', () => {
|
||||
expect(describeEmptyTranscriptionResponse({ result: { duration: 1 } })).toContain('keys=result')
|
||||
})
|
||||
})
|
||||
|
||||
describe('resolveTranscriptionFileName', () => {
|
||||
/**
|
||||
* @example
|
||||
* resolveTranscriptionFileName(new File([], 'recording.wav'))
|
||||
*/
|
||||
it('uses the File name so OpenAI-compatible providers can infer the audio format', () => {
|
||||
expect(resolveTranscriptionFileName(new File([], 'recording.wav'))).toBe('recording.wav')
|
||||
})
|
||||
})
|
||||
|
||||
@@ -74,6 +74,7 @@ export type HearingTranscriptionResult = HearingTranscriptionGenerateResult | He
|
||||
|
||||
type HearingTranscriptionInput = File | {
|
||||
file?: File
|
||||
fileName?: string
|
||||
inputAudioStream?: ReadableStream<ArrayBuffer>
|
||||
}
|
||||
|
||||
@@ -94,6 +95,111 @@ export function filterTranscriptionByConfidence(
|
||||
return segments.filter(s => (s?.avg_logprob ?? -Infinity) >= threshold).map(s => s?.text ?? '').join('').trim()
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads a string field from an unknown response object.
|
||||
*/
|
||||
function stringField(value: unknown, key: string, options: { trim?: boolean } = {}) {
|
||||
if (!value || typeof value !== 'object')
|
||||
return ''
|
||||
|
||||
const field = (value as Record<string, unknown>)[key]
|
||||
if (typeof field !== 'string')
|
||||
return ''
|
||||
|
||||
return options.trim === false ? field : field.trim()
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads a nested object field from an unknown response object.
|
||||
*/
|
||||
function objectField(value: unknown, key: string) {
|
||||
if (!value || typeof value !== 'object')
|
||||
return undefined
|
||||
|
||||
const field = (value as Record<string, unknown>)[key]
|
||||
return field && typeof field === 'object' ? field : undefined
|
||||
}
|
||||
|
||||
/**
|
||||
* Normalizes generated transcription text from OpenAI-compatible response variants.
|
||||
*
|
||||
* Before:
|
||||
* - `{ result: { text: "你好" } }`
|
||||
* - `{ segments: [{ text: "你" }, { text: "好" }] }`
|
||||
*
|
||||
* After:
|
||||
* - `"你好"`
|
||||
*/
|
||||
export function normalizeGeneratedTranscriptionText(response: unknown) {
|
||||
const directText = stringField(response, 'text')
|
||||
if (directText)
|
||||
return directText
|
||||
|
||||
for (const envelopeKey of ['result', 'data', 'output']) {
|
||||
const nested = objectField(response, envelopeKey)
|
||||
const nestedText = stringField(nested, 'text')
|
||||
if (nestedText)
|
||||
return nestedText
|
||||
}
|
||||
|
||||
const segments = objectField(response, 'segments') ?? (response && typeof response === 'object' ? (response as Record<string, unknown>).segments : undefined)
|
||||
if (Array.isArray(segments)) {
|
||||
const text = segments
|
||||
.map(segment => stringField(segment, 'text', { trim: false }))
|
||||
.join('')
|
||||
.trim()
|
||||
if (text)
|
||||
return text
|
||||
}
|
||||
|
||||
return ''
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds a compact diagnostic summary for an empty transcription response.
|
||||
*/
|
||||
export function describeEmptyTranscriptionResponse(response: unknown) {
|
||||
if (!response || typeof response !== 'object')
|
||||
return `response=${String(response)}`
|
||||
|
||||
const keys = Object.keys(response as Record<string, unknown>)
|
||||
const nestedKeys = keys
|
||||
.map((key) => {
|
||||
const nested = objectField(response, key)
|
||||
return nested ? `${key}.{${Object.keys(nested as Record<string, unknown>).join(',')}}` : ''
|
||||
})
|
||||
.filter(Boolean)
|
||||
|
||||
return [
|
||||
`keys=${keys.join(',') || '(none)'}`,
|
||||
...(nestedKeys.length ? [`nested=${nestedKeys.join(';')}`] : []),
|
||||
].join(' ')
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves the upload filename for transcription requests.
|
||||
*
|
||||
* Use when:
|
||||
* - OpenAI-compatible providers infer audio format from multipart filenames.
|
||||
*
|
||||
* Expects:
|
||||
* - `file.name` may carry the recorder-generated extension.
|
||||
*
|
||||
* Returns:
|
||||
* - A stable filename with an audio extension.
|
||||
*/
|
||||
export function resolveTranscriptionFileName(file: File, explicitFileName?: string) {
|
||||
const explicit = explicitFileName?.trim()
|
||||
if (explicit)
|
||||
return explicit
|
||||
|
||||
const fileName = file.name.trim()
|
||||
if (fileName)
|
||||
return fileName
|
||||
|
||||
return 'recording.wav'
|
||||
}
|
||||
|
||||
const STREAM_TRANSCRIPTION_EXECUTORS: Record<string, StreamTranscription> = {
|
||||
'aliyun-nls-transcription': streamAliyunTranscription,
|
||||
[OFFICIAL_TRANSCRIPTION_PROVIDER_ID]: streamAliyunTranscription,
|
||||
@@ -104,9 +210,78 @@ export function resolveStreamTranscriptionExecutor(providerId: string): StreamTr
|
||||
return STREAM_TRANSCRIPTION_EXECUTORS[providerId]
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves the setup error for the selected transcription provider.
|
||||
*
|
||||
* Use when:
|
||||
* - A speech pipeline entry point needs to fail before provider instantiation.
|
||||
* - User-facing diagnostics should explain the missing Hearing selection.
|
||||
*
|
||||
* Expects:
|
||||
* - `providerId` is the current `settings/hearing/active-provider` value.
|
||||
*
|
||||
* Returns:
|
||||
* - A setup error when no provider is selected, otherwise `undefined`.
|
||||
*/
|
||||
export function resolveActiveTranscriptionProviderError(providerId: string): string | undefined {
|
||||
if (providerId)
|
||||
return undefined
|
||||
|
||||
return 'No active transcription provider selected. Select a provider in Settings > Hearing.'
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves the transcription model from Hearing state with provider config fallback.
|
||||
*
|
||||
* Use when:
|
||||
* - OpenAI-compatible transcription stores the model in provider settings.
|
||||
* - The Hearing module has not yet synchronized that model into its active model state.
|
||||
*
|
||||
* Expects:
|
||||
* - `activeModel` is the current Hearing model value.
|
||||
* - `providerConfig.model` may contain a provider-scoped model name.
|
||||
*
|
||||
* Returns:
|
||||
* - The explicit Hearing model first, then the provider config model, otherwise an empty string.
|
||||
*/
|
||||
export function resolveActiveTranscriptionModel(activeModel: string, providerConfig?: Record<string, unknown>) {
|
||||
const modelFromHearing = activeModel.trim()
|
||||
if (modelFromHearing)
|
||||
return modelFromHearing
|
||||
|
||||
const modelFromProviderConfig = typeof providerConfig?.model === 'string' ? providerConfig.model.trim() : ''
|
||||
return modelFromProviderConfig
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves extra transcription request options from provider config and UI locale.
|
||||
*
|
||||
* Use when:
|
||||
* - Short ASR recordings need a language hint to avoid multilingual auto-detection drift.
|
||||
* - Provider-specific transcription prompts are configured outside the Hearing active model field.
|
||||
*
|
||||
* Expects:
|
||||
* - `uiLocale` uses a BCP-47-like language tag such as `zh-Hans` or `en-US`.
|
||||
*
|
||||
* Returns:
|
||||
* - OpenAI-compatible transcription options that can be merged into the provider request.
|
||||
*/
|
||||
export function resolveTranscriptionProviderOptions(providerConfig?: Record<string, unknown>, uiLocale = globalThis.navigator?.language ?? '') {
|
||||
const configuredLanguage = typeof providerConfig?.language === 'string' ? providerConfig.language.trim() : ''
|
||||
const localeLanguage = uiLocale.split(/[-_]/)[0]?.trim().toLowerCase() ?? ''
|
||||
const language = configuredLanguage || localeLanguage
|
||||
const prompt = typeof providerConfig?.prompt === 'string' ? providerConfig.prompt.trim() : ''
|
||||
|
||||
return {
|
||||
...(language ? { language } : {}),
|
||||
...(prompt ? { prompt } : {}),
|
||||
}
|
||||
}
|
||||
|
||||
export const useHearingStore = defineStore('hearing-store', () => {
|
||||
const providersStore = useProvidersStore()
|
||||
const { allAudioTranscriptionProvidersMetadata } = storeToRefs(providersStore)
|
||||
const { trackSttStarted, trackSttSucceeded, trackSttFailed } = useAnalytics()
|
||||
|
||||
// State
|
||||
const activeTranscriptionProvider = useLocalStorageManualReset('settings/hearing/active-provider', '')
|
||||
@@ -196,12 +371,12 @@ export const useHearingStore = defineStore('hearing-store', () => {
|
||||
): Promise<HearingTranscriptionResult> {
|
||||
const normalizedInput = (input instanceof File ? { file: input } : input ?? {}) as {
|
||||
file?: File
|
||||
fileName?: string
|
||||
inputAudioStream?: ReadableStream<ArrayBuffer>
|
||||
}
|
||||
const features = providersStore.getTranscriptionFeatures(providerId)
|
||||
const streamExecutor = resolveStreamTranscriptionExecutor(providerId)
|
||||
|
||||
const { trackSttStarted, trackSttSucceeded, trackSttFailed } = useAnalytics()
|
||||
const sttStartedAt = performance.now()
|
||||
trackSttStarted(providerId)
|
||||
|
||||
@@ -275,6 +450,7 @@ export const useHearingStore = defineStore('hearing-store', () => {
|
||||
const response = await generateTranscription({
|
||||
...provider.transcription(model, options?.providerOptions),
|
||||
file: normalizedInput.file,
|
||||
fileName: resolveTranscriptionFileName(normalizedInput.file, normalizedInput.fileName),
|
||||
responseFormat: useVerboseJson ? 'verbose_json' : format,
|
||||
})
|
||||
|
||||
@@ -295,11 +471,12 @@ export const useHearingStore = defineStore('hearing-store', () => {
|
||||
}
|
||||
}
|
||||
|
||||
const fallbackText = typeof response.text === 'string' ? response.text : ''
|
||||
const fallbackText = normalizeGeneratedTranscriptionText(response)
|
||||
emitSucceeded(fallbackText.length, false)
|
||||
return {
|
||||
mode: 'generate',
|
||||
...response,
|
||||
text: fallbackText,
|
||||
}
|
||||
}
|
||||
catch (err) {
|
||||
@@ -355,6 +532,24 @@ export const useHearingSpeechInputPipeline = defineStore('modules:hearing:speech
|
||||
|
||||
let asrSpan: Span | undefined
|
||||
|
||||
function startStreamingAsrSpan(providerId: string) {
|
||||
activeTurnSpan.value?.end()
|
||||
const turnSpan = startSpan(IOSpanNames.InteractionTurn)
|
||||
activeTurnSpan.value = turnSpan
|
||||
asrSpan = startSpan(IOSpanNames.SpeechRecognition, turnSpan, {
|
||||
[IOAttributes.Subsystem]: IOSubsystems.ASR,
|
||||
[IOAttributes.GenAIRequestModel]: providerId,
|
||||
})
|
||||
}
|
||||
|
||||
function endStreamingAsrSpan() {
|
||||
if (!asrSpan)
|
||||
return
|
||||
|
||||
asrSpan.end()
|
||||
asrSpan = undefined
|
||||
}
|
||||
|
||||
const supportsStreamInput = computed(() => {
|
||||
const providerId = activeTranscriptionProvider.value
|
||||
if (!providerId)
|
||||
@@ -543,14 +738,6 @@ export const useHearingSpeechInputPipeline = defineStore('modules:hearing:speech
|
||||
onSentenceEnd?: (delta: string) => void
|
||||
onSpeechEnd?: (text: string) => void
|
||||
}) {
|
||||
activeTurnSpan.value?.end()
|
||||
const turnSpan = startSpan(IOSpanNames.InteractionTurn)
|
||||
activeTurnSpan.value = turnSpan
|
||||
asrSpan = startSpan(IOSpanNames.SpeechRecognition, turnSpan, {
|
||||
[IOAttributes.Subsystem]: IOSubsystems.ASR,
|
||||
[IOAttributes.GenAIRequestModel]: activeTranscriptionProvider.value ?? '',
|
||||
})
|
||||
|
||||
console.info('[Hearing Pipeline] transcribeForMediaStream called', {
|
||||
supportsStreamInput: supportsStreamInput.value,
|
||||
hasStream: !!stream,
|
||||
@@ -567,9 +754,10 @@ export const useHearingSpeechInputPipeline = defineStore('modules:hearing:speech
|
||||
|
||||
try {
|
||||
const providerId = activeTranscriptionProvider.value
|
||||
if (!providerId) {
|
||||
error.value = 'No transcription provider selected'
|
||||
console.error('[Hearing Pipeline] No transcription provider selected')
|
||||
const providerError = resolveActiveTranscriptionProviderError(providerId)
|
||||
if (providerError) {
|
||||
error.value = providerError
|
||||
console.error('[Hearing Pipeline]', providerError)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -619,6 +807,8 @@ export const useHearingSpeechInputPipeline = defineStore('modules:hearing:speech
|
||||
}
|
||||
}
|
||||
|
||||
startStreamingAsrSpan(providerId)
|
||||
|
||||
// Auto-select default model if not selected
|
||||
if (!activeTranscriptionModel.value) {
|
||||
// Try to get models for the provider and select the first one
|
||||
@@ -760,6 +950,8 @@ export const useHearingSpeechInputPipeline = defineStore('modules:hearing:speech
|
||||
}
|
||||
}
|
||||
|
||||
startStreamingAsrSpan(providerId)
|
||||
|
||||
const abortController = new AbortController()
|
||||
let idleTimer: ReturnType<typeof setTimeout> | undefined
|
||||
const bumpIdle = () => {
|
||||
@@ -856,6 +1048,8 @@ export const useHearingSpeechInputPipeline = defineStore('modules:hearing:speech
|
||||
}
|
||||
}
|
||||
catch (err) {
|
||||
endStreamingAsrSpan()
|
||||
|
||||
if (isExpectedStreamStopError(err))
|
||||
return
|
||||
|
||||
@@ -867,33 +1061,58 @@ export const useHearingSpeechInputPipeline = defineStore('modules:hearing:speech
|
||||
async function transcribeForRecording(recording: Blob | null | undefined) {
|
||||
error.value = undefined
|
||||
|
||||
if (!recording)
|
||||
if (!recording) {
|
||||
error.value = 'No recording captured from microphone'
|
||||
return
|
||||
}
|
||||
|
||||
if (recording.size <= 0) {
|
||||
error.value = 'Recording captured from microphone is empty'
|
||||
return
|
||||
}
|
||||
|
||||
try {
|
||||
if (recording && recording.size > 0) {
|
||||
const providerId = activeTranscriptionProvider.value
|
||||
const provider = await providersStore.getProviderInstance<TranscriptionProviderWithExtraOptions<string, any>>(providerId)
|
||||
if (!provider) {
|
||||
throw new Error('Failed to initialize speech provider')
|
||||
}
|
||||
|
||||
// Get model from configuration or use default
|
||||
const model = activeTranscriptionModel.value
|
||||
const result = await hearingStore.transcription(
|
||||
providerId,
|
||||
provider,
|
||||
model,
|
||||
new File([recording], 'recording.wav'),
|
||||
)
|
||||
const text = result.mode === 'stream' ? await result.text : result.text
|
||||
if (!text || !text.trim()) {
|
||||
error.value = 'No transcription result returned from provider'
|
||||
return
|
||||
}
|
||||
|
||||
return text
|
||||
const providerId = activeTranscriptionProvider.value
|
||||
const providerError = resolveActiveTranscriptionProviderError(providerId)
|
||||
if (providerError) {
|
||||
error.value = providerError
|
||||
console.error('[Hearing Pipeline]', providerError)
|
||||
return
|
||||
}
|
||||
|
||||
const provider = await providersStore.getProviderInstance<TranscriptionProviderWithExtraOptions<string, any>>(providerId)
|
||||
if (!provider) {
|
||||
throw new Error('Failed to initialize speech provider')
|
||||
}
|
||||
|
||||
const providerConfig = providersStore.getProviderConfig(providerId)
|
||||
const model = resolveActiveTranscriptionModel(activeTranscriptionModel.value, providerConfig)
|
||||
const providerOptions = resolveTranscriptionProviderOptions(providerConfig)
|
||||
console.info('[Hearing Pipeline] Transcribing recording', {
|
||||
providerId,
|
||||
language: providerOptions.language,
|
||||
model,
|
||||
recordingSize: recording.size,
|
||||
recordingType: recording.type,
|
||||
})
|
||||
const result = await hearingStore.transcription(
|
||||
providerId,
|
||||
provider,
|
||||
model,
|
||||
new File([recording], 'recording.wav', { type: recording.type || 'audio/wav' }),
|
||||
undefined,
|
||||
{ providerOptions },
|
||||
)
|
||||
const text = result.mode === 'stream' ? await result.text : result.text
|
||||
if (!text || !text.trim()) {
|
||||
const responseSummary = result.mode === 'generate'
|
||||
? describeEmptyTranscriptionResponse(result)
|
||||
: 'stream result returned empty text'
|
||||
error.value = `No transcription result returned from provider (${responseSummary})`
|
||||
return
|
||||
}
|
||||
|
||||
return text
|
||||
}
|
||||
catch (err) {
|
||||
error.value = errorMessage(err)
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
|
||||
import { buildOpenAICompatibleProvider } from './openai-compatible-builder'
|
||||
|
||||
describe('buildOpenAICompatibleProvider', () => {
|
||||
/**
|
||||
* @example
|
||||
* provider.transcription('FunAudioLLM/SenseVoiceSmall', { language: 'zh' })
|
||||
*/
|
||||
it('preserves transcription extra options for OpenAI-compatible ASR providers', async () => {
|
||||
const metadata = buildOpenAICompatibleProvider({
|
||||
id: 'test-openai-compatible-transcription',
|
||||
name: 'Test Transcription',
|
||||
nameKey: 'test.transcription.title',
|
||||
description: 'Test transcription provider',
|
||||
descriptionKey: 'test.transcription.description',
|
||||
icon: 'i-lobe-icons:openai',
|
||||
category: 'transcription',
|
||||
creator: () => ({
|
||||
transcription: (model: string) => ({
|
||||
baseURL: 'https://example.com/v1/',
|
||||
model,
|
||||
}),
|
||||
}),
|
||||
})
|
||||
|
||||
const provider = await metadata.createProvider({})
|
||||
|
||||
expect('transcription' in provider).toBe(true)
|
||||
expect((provider as any).transcription('FunAudioLLM/SenseVoiceSmall', { language: 'zh' })).toEqual({
|
||||
baseURL: 'https://example.com/v1/',
|
||||
language: 'zh',
|
||||
model: 'FunAudioLLM/SenseVoiceSmall',
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -35,6 +35,26 @@ function logWarn(...args: unknown[]) {
|
||||
console.warn(...args)
|
||||
}
|
||||
|
||||
/**
|
||||
* Wraps transcription providers so OpenAI audio options like `language` and `prompt` are preserved.
|
||||
*/
|
||||
function withTranscriptionExtraOptions(provider: unknown) {
|
||||
if (!provider || typeof provider !== 'object' || !('transcription' in provider))
|
||||
return provider
|
||||
|
||||
const transcription = (provider as { transcription?: unknown }).transcription
|
||||
if (typeof transcription !== 'function')
|
||||
return provider
|
||||
|
||||
return {
|
||||
...provider,
|
||||
transcription: (model: string, extraOptions?: Record<string, unknown>) => ({
|
||||
...transcription(model),
|
||||
...extraOptions,
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
export function buildOpenAICompatibleProvider(
|
||||
options: Partial<ProviderMetadata> & {
|
||||
id: string
|
||||
@@ -270,7 +290,11 @@ export function buildOpenAICompatibleProvider(
|
||||
createProvider: async (config: { apiKey: string, baseUrl: string }) => {
|
||||
const apiKey = normalizeString(config.apiKey)
|
||||
const baseUrl = normalizeBaseUrl(config.baseUrl)
|
||||
return creator(apiKey, baseUrl)
|
||||
const provider = await creator(apiKey, baseUrl)
|
||||
if (resolvedCategory === 'transcription')
|
||||
return withTranscriptionExtraOptions(provider)
|
||||
|
||||
return provider
|
||||
},
|
||||
capabilities: finalCapabilities,
|
||||
validators: finalValidators,
|
||||
|
||||
@@ -0,0 +1,136 @@
|
||||
import { createTestingPinia } from '@pinia/testing'
|
||||
import { setActivePinia } from 'pinia'
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import { nextTick } from 'vue'
|
||||
|
||||
const storageMock = vi.hoisted(() => ({
|
||||
values: new Map<string, unknown>(),
|
||||
}))
|
||||
|
||||
const audioDeviceMock = vi.hoisted(() => ({
|
||||
audioInputs: { value: [] as MediaDeviceInfo[] },
|
||||
selectedAudioInput: { value: '' },
|
||||
startStream: vi.fn(),
|
||||
stopStream: vi.fn(),
|
||||
askPermission: vi.fn(),
|
||||
}))
|
||||
|
||||
vi.mock('@proj-airi/stage-shared/composables', async () => {
|
||||
const vue = await vi.importActual<typeof import('vue')>('vue')
|
||||
|
||||
return {
|
||||
useLocalStorageManualReset: <T>(key: string, initialValue: T) => {
|
||||
const value = vue.ref((storageMock.values.has(key) ? storageMock.values.get(key) : initialValue) as T)
|
||||
|
||||
storageMock.values.set(key, value.value)
|
||||
vue.watch(value, (newValue) => {
|
||||
storageMock.values.set(key, newValue)
|
||||
}, { flush: 'sync' })
|
||||
|
||||
return Object.assign(value, {
|
||||
reset: () => {
|
||||
value.value = initialValue
|
||||
},
|
||||
})
|
||||
},
|
||||
}
|
||||
})
|
||||
|
||||
vi.mock('../../composables/audio', async () => {
|
||||
const vue = await vi.importActual<typeof import('vue')>('vue')
|
||||
|
||||
return {
|
||||
useAudioDevice: () => ({
|
||||
audioInputs: audioDeviceMock.audioInputs,
|
||||
deviceConstraints: vue.computed(() => ({ audio: true })),
|
||||
selectedAudioInput: audioDeviceMock.selectedAudioInput,
|
||||
startStream: audioDeviceMock.startStream,
|
||||
stopStream: audioDeviceMock.stopStream,
|
||||
stream: vue.shallowRef<MediaStream>(),
|
||||
askPermission: audioDeviceMock.askPermission,
|
||||
}),
|
||||
}
|
||||
})
|
||||
|
||||
function createAudioInput(deviceId: string): MediaDeviceInfo {
|
||||
return {
|
||||
deviceId,
|
||||
groupId: '',
|
||||
kind: 'audioinput',
|
||||
label: deviceId,
|
||||
toJSON: () => ({}),
|
||||
}
|
||||
}
|
||||
|
||||
describe('store settings-audio-devices', () => {
|
||||
beforeEach(() => {
|
||||
setActivePinia(createTestingPinia({ createSpy: vi.fn, stubActions: false }))
|
||||
storageMock.values.clear()
|
||||
audioDeviceMock.audioInputs.value = []
|
||||
audioDeviceMock.selectedAudioInput.value = ''
|
||||
vi.clearAllMocks()
|
||||
})
|
||||
|
||||
afterEach(() => {
|
||||
vi.resetModules()
|
||||
})
|
||||
|
||||
it('starts with the persisted microphone instead of overwriting it with the runtime default', async () => {
|
||||
storageMock.values.set('settings/audio/input', 'microphone-1')
|
||||
storageMock.values.set('settings/audio/input/enabled', true)
|
||||
audioDeviceMock.audioInputs.value = [
|
||||
createAudioInput('default'),
|
||||
createAudioInput('microphone-1'),
|
||||
]
|
||||
audioDeviceMock.selectedAudioInput.value = 'default'
|
||||
|
||||
const startedWith: string[] = []
|
||||
audioDeviceMock.startStream.mockImplementation(async () => {
|
||||
startedWith.push(audioDeviceMock.selectedAudioInput.value)
|
||||
})
|
||||
|
||||
const { useSettingsAudioDevice } = await import('./audio-device')
|
||||
const store = useSettingsAudioDevice()
|
||||
|
||||
store.initialize()
|
||||
await Promise.resolve()
|
||||
|
||||
expect(startedWith).toEqual(['microphone-1'])
|
||||
expect(store.selectedAudioInput).toBe('microphone-1')
|
||||
expect(storageMock.values.get('settings/audio/input')).toBe('microphone-1')
|
||||
})
|
||||
|
||||
it('ignores stale microphone startup failures after a newer start succeeds', async () => {
|
||||
const { useSettingsAudioDevice } = await import('./audio-device')
|
||||
const store = useSettingsAudioDevice()
|
||||
|
||||
let rejectFirstStart!: (error: unknown) => void
|
||||
let resolveSecondStart!: () => void
|
||||
audioDeviceMock.startStream
|
||||
.mockImplementationOnce(() => new Promise<void>((_resolve, reject) => {
|
||||
rejectFirstStart = reject
|
||||
}))
|
||||
.mockImplementationOnce(() => new Promise<void>((resolve) => {
|
||||
resolveSecondStart = resolve
|
||||
}))
|
||||
|
||||
store.enabled = true
|
||||
await nextTick()
|
||||
|
||||
store.enabled = false
|
||||
await nextTick()
|
||||
|
||||
store.enabled = true
|
||||
await nextTick()
|
||||
|
||||
resolveSecondStart()
|
||||
await Promise.resolve()
|
||||
|
||||
rejectFirstStart(new Error('old startup failed'))
|
||||
await Promise.resolve()
|
||||
await nextTick()
|
||||
|
||||
expect(store.enabled).toBe(true)
|
||||
expect(audioDeviceMock.stopStream).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
})
|
||||
@@ -7,10 +7,68 @@ import { useAudioDevice } from '../../composables/audio'
|
||||
let microphonePermissionStatus: PermissionStatus
|
||||
|
||||
export const useSettingsAudioDevice = defineStore('settings-audio-devices', () => {
|
||||
const { audioInputs, deviceConstraints, selectedAudioInput: selectedAudioInputNonPersist, startStream, stopStream, stream, askPermission } = useAudioDevice()
|
||||
const {
|
||||
audioInputs,
|
||||
deviceConstraints,
|
||||
selectedAudioInput: selectedAudioInputNonPersist,
|
||||
startStream: startAudioInputStream,
|
||||
stopStream: stopAudioInputStream,
|
||||
stream,
|
||||
askPermission: askAudioInputPermission,
|
||||
} = useAudioDevice()
|
||||
|
||||
const selectedAudioInputPersist = useLocalStorageManualReset<string>('settings/audio/input', selectedAudioInputNonPersist.value)
|
||||
const audioInputEnabled = useLocalStorageManualReset<boolean>('settings/audio/input/enabled', false)
|
||||
let audioInputStartGeneration = 0
|
||||
|
||||
function syncSelectedAudioInputFromRuntime() {
|
||||
if (selectedAudioInputPersist.value !== selectedAudioInputNonPersist.value)
|
||||
selectedAudioInputPersist.value = selectedAudioInputNonPersist.value
|
||||
}
|
||||
|
||||
function syncSelectedAudioInputToRuntime() {
|
||||
if (selectedAudioInputPersist.value && selectedAudioInputPersist.value !== selectedAudioInputNonPersist.value)
|
||||
selectedAudioInputNonPersist.value = selectedAudioInputPersist.value
|
||||
}
|
||||
|
||||
async function askPermission() {
|
||||
syncSelectedAudioInputToRuntime()
|
||||
await askAudioInputPermission()
|
||||
syncSelectedAudioInputFromRuntime()
|
||||
}
|
||||
|
||||
function createAudioInputStartGeneration() {
|
||||
audioInputStartGeneration += 1
|
||||
return audioInputStartGeneration
|
||||
}
|
||||
|
||||
function invalidateAudioInputStarts() {
|
||||
audioInputStartGeneration += 1
|
||||
}
|
||||
|
||||
async function startStreamForGeneration(generation: number) {
|
||||
syncSelectedAudioInputToRuntime()
|
||||
await startAudioInputStream()
|
||||
|
||||
if (generation === audioInputStartGeneration)
|
||||
syncSelectedAudioInputFromRuntime()
|
||||
}
|
||||
|
||||
async function startStream() {
|
||||
await startStreamForGeneration(createAudioInputStartGeneration())
|
||||
}
|
||||
|
||||
function stopStream() {
|
||||
invalidateAudioInputStarts()
|
||||
stopAudioInputStream()
|
||||
}
|
||||
|
||||
function handleStartStreamError(generation: number, error: unknown, message: string) {
|
||||
console.error(message, error)
|
||||
|
||||
if (generation === audioInputStartGeneration)
|
||||
audioInputEnabled.value = false
|
||||
}
|
||||
|
||||
watch(selectedAudioInputPersist, (newValue) => {
|
||||
selectedAudioInputNonPersist.value = newValue
|
||||
@@ -18,7 +76,10 @@ export const useSettingsAudioDevice = defineStore('settings-audio-devices', () =
|
||||
|
||||
watch(audioInputEnabled, (val) => {
|
||||
if (val) {
|
||||
startStream()
|
||||
const generation = createAudioInputStartGeneration()
|
||||
startStreamForGeneration(generation).catch((error) => {
|
||||
handleStartStreamError(generation, error, 'Unable to start audio input stream:')
|
||||
})
|
||||
}
|
||||
else {
|
||||
stopStream()
|
||||
@@ -42,8 +103,17 @@ export const useSettingsAudioDevice = defineStore('settings-audio-devices', () =
|
||||
const hasSelectedInput = selectedAudioInputPersist.value
|
||||
&& audioInputs.value.some(device => device.deviceId === selectedAudioInputPersist.value)
|
||||
|
||||
if (hasSelectedInput)
|
||||
syncSelectedAudioInputToRuntime()
|
||||
|
||||
if (audioInputEnabled.value && hasSelectedInput) {
|
||||
startStream()
|
||||
const generation = createAudioInputStartGeneration()
|
||||
startStreamForGeneration(generation).catch((error) => {
|
||||
handleStartStreamError(generation, error, 'Unable to initialize audio input stream:')
|
||||
})
|
||||
}
|
||||
else if (selectedAudioInputPersist.value && audioInputs.value.length > 0 && !hasSelectedInput) {
|
||||
selectedAudioInputPersist.value = selectedAudioInputNonPersist.value
|
||||
}
|
||||
if (selectedAudioInputNonPersist.value && !audioInputEnabled.value) {
|
||||
selectedAudioInputPersist.value = selectedAudioInputNonPersist.value
|
||||
|
||||
Reference in New Issue
Block a user