nanakura/moeka-project
1
0
Fork 0
Code Issues Pull Requests 1 Actions 2 Packages Projects Releases Wiki Activity

fix(stage-ui): fix transcription session callback reuse and auto-send issues (#996)

Browse Source
This commit is contained in:
lockrush-dev
2026-01-26 14:59:50 +08:00
committed by GitHub
parent 850792b0a1
commit 4cac97937f
5 changed files with 122 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/stage-pocket/src/pages/index.vue
+5 -1
View File
@@ -46,7 +46,7 @@ const settingsAudioDeviceStore = useSettingsAudioDevice()
const { stream, enabled } = storeToRefs(settingsAudioDeviceStore)
const { startRecord, stopRecord, onStopRecord } = useAudioRecorder(stream)
const hearingPipeline = useHearingSpeechInputPipeline()
const { transcribeForRecording, transcribeForMediaStream } = hearingPipeline
const { transcribeForRecording, transcribeForMediaStream, stopStreamingTranscription } = hearingPipeline
const { supportsStreamInput } = storeToRefs(hearingPipeline)
const providersStore = useProvidersStore()
const consciousnessStore = useConsciousnessStore()
@@ -99,6 +99,8 @@ async function startAudioInteraction() {
async function handleSpeechStart() {
if (shouldUseStreamInput.value && stream.value) {
// Use both callbacks to support incremental updates and final transcript replacement.
// ChatArea uses only onSentenceEnd to avoid re-adding deleted text.
await transcribeForMediaStream(stream.value, {
onSentenceEnd: (delta) => {
const finalText = delta
@@ -139,6 +141,8 @@ function stopAudioInteraction() {
try {
stopOnStopRecord?.()
stopOnStopRecord = undefined
// Stop any active streaming transcription sessions to prevent session leakage
void stopStreamingTranscription(true)
disposeVAD()
}
catch {}
apps/stage-tamagotchi/src/renderer/pages/index.vue
+1
View File
@@ -203,6 +203,7 @@ async function startAudioInteraction() {
return
}
// Use sentence deltas for live captions and speech end for final text.
await transcribeForMediaStream(stream.value, {
onSentenceEnd: (delta) => {
console.info('[Main Page] Received transcription delta:', delta)
packages/stage-layouts/src/components/Widgets/ChatArea.vue
+35 -7
View File
@@ -49,8 +49,18 @@ const shouldUseStreamInput = computed(() => supportsStreamInput.value && !!strea
let autoSendTimeout: ReturnType<typeof setTimeout> | undefined
const pendingAutoSendText = ref('')
function clearPendingAutoSend() {
if (autoSendTimeout) {
clearTimeout(autoSendTimeout)
autoSendTimeout = undefined
}
pendingAutoSendText.value = ''
}
async function debouncedAutoSend(text: string) {
// Double-check auto-send is enabled before proceeding
if (!autoSendEnabled.value) {
clearPendingAutoSend()
return
}
@@ -64,6 +74,12 @@ async function debouncedAutoSend(text: string) {
// Set new timeout
autoSendTimeout = setTimeout(async () => {
// Final check before sending - auto-send might have been disabled while waiting
if (!autoSendEnabled.value) {
clearPendingAutoSend()
return
}
const textToSend = pendingAutoSendText.value.trim()
if (textToSend && autoSendEnabled.value) {
try {
@@ -259,6 +275,8 @@ async function startListening() {
if (!shouldUseStreamInput.value) {
const errorMsg = 'Streaming input not supported by the selected transcription provider. Please select a provider that supports streaming (e.g., Web Speech API).'
console.warn('[ChatArea]', errorMsg)
// Clean up any existing sessions from other pages (e.g., test page) that might interfere
await stopStreamingTranscription(true)
isListening.value = false
return
}
@@ -276,14 +294,18 @@ async function startListening() {
messageInput.value = currentText ? `${currentText} ${delta}` : delta
console.info('[ChatArea] Received transcription delta:', delta)
// Auto-send if enabled
// Auto-send if enabled - check the current value (not captured in closure)
// This ensures we always respect the current setting, even if callbacks are reused
if (autoSendEnabled.value) {
debouncedAutoSend(delta)
}
else {
// If auto-send is disabled, clear any pending auto-send text to prevent accidental sends
clearPendingAutoSend()
}
}
},
// Don't use onSpeechEnd - it re-adds text that users may have deleted
// onSentenceEnd handles all text updates as transcription happens
// Omit onSpeechEnd to avoid re-adding user-deleted text; use sentence deltas only.
})
// Only set listening to true if transcription started successfully
@@ -311,10 +333,7 @@ async function stopListening() {
console.info('[ChatArea] Stopping transcription...')
// Clear auto-send timeout
if (autoSendTimeout) {
clearTimeout(autoSendTimeout)
autoSendTimeout = undefined
}
clearPendingAutoSend()
// Send any pending text immediately if auto-send is enabled
if (autoSendEnabled.value && pendingAutoSendText.value.trim()) {
@@ -367,6 +386,15 @@ watch(stream, async (val) => {
await stopListening()
}
})
// Watch for auto-send setting changes and clear pending sends if disabled
watch(autoSendEnabled, (enabled) => {
if (!enabled) {
// Auto-send was disabled - clear any pending auto-send
clearPendingAutoSend()
console.info('[ChatArea] Auto-send disabled, cleared pending text')
}
})
</script>
<template>
packages/stage-pages/src/pages/settings/modules/hearing.vue
+44 -37
View File
@@ -78,6 +78,8 @@ const shouldUseStreamInput = computed(() => supportsStreamInput.value && !!strea
async function handleSpeechStart() {
if (shouldUseStreamInput.value && stream.value) {
// Use both callbacks to support incremental updates and final transcript replacement.
// ChatArea uses only onSentenceEnd to avoid re-adding deleted text.
await transcribeForMediaStream(stream.value, {
onSentenceEnd: (delta) => {
transcriptions.value.push(delta)
@@ -251,12 +253,40 @@ onStopRecord(async (recording) => {
if (shouldUseStreamInput.value)
return
// Skip onStopRecord handler during STT test - the watch handler handles transcription for tests
if (isTestingSTT.value)
if (!recording || recording.size === 0)
return
if (recording && recording.size > 0)
audios.value.push(recording)
// Handle STT test transcription directly here
if (isTestingSTT.value) {
testStatusMessage.value = 'Transcribing recording...'
isTranscribing.value = true
try {
const result = await transcribeForRecording(recording)
if (result) {
testTranscriptionText.value = result
testStatusMessage.value = 'Transcription complete!'
console.info('STT test transcription result:', result)
}
else {
testTranscriptionError.value = 'No transcription result received'
testStatusMessage.value = 'Transcription failed'
}
}
catch (err) {
testTranscriptionError.value = err instanceof Error ? err.message : String(err)
testStatusMessage.value = `Error: ${testTranscriptionError.value}`
console.error('STT test transcription error:', err)
}
finally {
isTranscribing.value = false
isTestingSTT.value = false
}
return
}
// Normal monitoring mode - add to audios and transcribe
audios.value.push(recording)
const res = await transcribeForRecording(recording)
@@ -399,39 +429,8 @@ async function stopSTTTest() {
}
}
// Watch for recording completion during STT test
watch(() => audios.value.length, async (newLength, oldLength) => {
if (isTestingSTT.value && !shouldUseStreamInput.value && newLength > oldLength) {
// Recording was completed, now transcribe it
const latestRecording = audios.value[audios.value.length - 1]
if (latestRecording) {
testStatusMessage.value = 'Transcribing recording...'
isTranscribing.value = true
try {
const result = await transcribeForRecording(latestRecording)
if (result) {
testTranscriptionText.value = result
testStatusMessage.value = 'Transcription complete!'
console.info('STT test transcription result:', result)
}
else {
testTranscriptionError.value = 'No transcription result received'
testStatusMessage.value = 'Transcription failed'
}
}
catch (err) {
testTranscriptionError.value = err instanceof Error ? err.message : String(err)
testStatusMessage.value = `Error: ${testTranscriptionError.value}`
console.error('STT test transcription error:', err)
}
finally {
isTranscribing.value = false
isTestingSTT.value = false
}
}
}
})
// Note: STT test transcription is now handled directly in onStopRecord handler above
// This watch is kept for potential future use but is no longer needed for STT tests
watch(selectedAudioInput, async () => isMonitoring.value && await setupAudioMonitoring())
@@ -470,6 +469,14 @@ onUnmounted(() => {
stopAudioMonitoring()
disposeVAD()
// Clean up any active transcription sessions when leaving the page
// This prevents stale sessions from interfering with other pages
if (shouldUseStreamInput.value) {
stopStreamingTranscription(true, activeTranscriptionProvider.value).catch((err) => {
console.warn('[Hearing Module] Error cleaning up transcription session on unmount:', err)
})
}
audioCleanups.value.forEach(cleanup => cleanup())
})
</script>
packages/stage-ui/src/stores/modules/hearing.ts
+37 -9
View File
@@ -584,16 +584,30 @@ export const useHearingSpeechInputPipeline = defineStore('modules:hearing:speech
const idleTimeout = options?.idleTimeoutMs ?? DEFAULT_STREAM_IDLE_TIMEOUT
// If a session already exists, just bump the idle timer and reuse the websocket/audio graph.
// If a session exists, reuse it unless new callbacks are provided.
// The stream reader captures callbacks at creation time, so updated callbacks
// require restarting the session to create a new reader.
const existingSession = streamingSession.value
if (existingSession) {
if (existingSession.idleTimer) {
clearTimeout(existingSession.idleTimer)
existingSession.idleTimer = setTimeout(async () => {
await stopStreamingTranscription(false, existingSession.providerId)
}, idleTimeout)
const hasNewCallbacks
= options?.onSentenceEnd !== undefined
|| options?.onSpeechEnd !== undefined
if (hasNewCallbacks) {
console.info('[Hearing Pipeline] New callbacks provided, restarting session')
await stopStreamingTranscription(false, existingSession.providerId)
// Fall through to create a new session with updated callbacks
}
else {
// No callback changes: refresh idle timer and reuse session
if (existingSession.idleTimer) {
clearTimeout(existingSession.idleTimer)
existingSession.idleTimer = setTimeout(async () => {
await stopStreamingTranscription(false, existingSession.providerId)
}, idleTimeout)
}
return
}
return
}
const abortController = new AbortController()
@@ -641,11 +655,23 @@ export const useHearingSpeechInputPipeline = defineStore('modules:hearing:speech
result,
idleTimer,
providerId,
callbacks: {
onSentenceEnd: options?.onSentenceEnd,
onSpeechEnd: options?.onSpeechEnd,
},
}
// Stream out text deltas to caller without tearing down the session.
if (result.mode === 'stream' && result.textStream) {
void (async () => {
// Capture callbacks from the session at the time the reader is created
// This prevents cross-session leakage if the session is restarted before
// this reader finishes (e.g., when navigating between pages or callbacks change)
const sessionCallbacks = {
onSentenceEnd: streamingSession.value?.callbacks?.onSentenceEnd,
onSpeechEnd: streamingSession.value?.callbacks?.onSpeechEnd,
}
let fullText = ''
try {
const reader = result.textStream.getReader()
@@ -656,7 +682,8 @@ export const useHearingSpeechInputPipeline = defineStore('modules:hearing:speech
break
if (value) {
fullText += value
options?.onSentenceEnd?.(value)
// Use captured callbacks to avoid cross-session leakage
sessionCallbacks.onSentenceEnd?.(value)
}
}
}
@@ -664,7 +691,8 @@ export const useHearingSpeechInputPipeline = defineStore('modules:hearing:speech
console.error('Error reading text stream:', err)
}
finally {
options?.onSpeechEnd?.(fullText)
// Use captured callbacks to avoid cross-session leakage
sessionCallbacks.onSpeechEnd?.(fullText)
}
})()
}