fix(stage-tamagotchi): replace streaming caption corrections (#2263)

apps/stage-tamagotchi/src/renderer/composables/useCaptionItems.test.ts

@@ -48,4 +48,34 @@ describe('useCaptionItems', () => {
       vi.useRealTimers()
     }
   })
+
+  // ROOT CAUSE:
+  //
+  // Streaming providers send a complete volatile sentence on each update.
+  // The caption overlay appended every correction as a separate item.
+  it('replaces volatile speaker captions without accumulating corrections', () => {
+    vi.useFakeTimers()
+
+    try {
+      const captions = useCaptionItems({ ttlMs: 1000 })
+
+      captions.add({ operation: 'replace', type: 'caption-speaker', text: '今天天气很号' })
+      vi.advanceTimersByTime(500)
+      captions.add({ operation: 'replace', type: 'caption-speaker', text: '今天天气很好' })
+
+      expect(captions.items.value).toHaveLength(1)
+      expect(captions.items.value[0]?.text).toBe('今天天气很好')
+
+      vi.advanceTimersByTime(500)
+
+      expect(captions.items.value).toHaveLength(1)
+
+      vi.advanceTimersByTime(500)
+
+      expect(captions.items.value).toEqual([])
+    }
+    finally {
+      vi.useRealTimers()
+    }
+  })
 })

apps/stage-tamagotchi/src/renderer/composables/useCaptionItems.ts

@@ -1,8 +1,6 @@
-import { readonly, shallowRef } from 'vue'
+import type { CaptionChannelEvent } from '@proj-airi/stage-shared'
 
-export type CaptionChannelEvent
-  = | { type: 'caption-speaker', text: string }
-    | { type: 'caption-assistant', text: string }
+import { readonly, shallowRef } from 'vue'
 
 export interface CaptionItem {
   /** Stable render key and timer owner for one broadcast caption event. */
@@ -68,21 +66,54 @@ export function useCaptionItems(options: UseCaptionItemsOptions = {}) {
     items.value = items.value.filter(item => item.type !== type)
   }
 
+  function scheduleExpiry(item: CaptionItem) {
+    expiryTimers.set(item.id, setTimeout(() => {
+      remove(item.id)
+    }, ttlMs))
+  }
+
+  function replace(event: CaptionChannelEvent) {
+    const matchedItems = items.value.filter(item => item.type === event.type)
+    const currentItem = matchedItems.at(-1)
+    if (!currentItem) {
+      const item: CaptionItem = {
+        id: nextId++,
+        type: event.type,
+        text: event.text,
+      }
+      items.value = [...items.value, item]
+      scheduleExpiry(item)
+      return
+    }
+
+    for (const item of matchedItems)
+      clearTimer(item.id)
+
+    const replacement = { ...currentItem, text: event.text }
+    items.value = items.value
+      .filter(item => item.type !== event.type || item.id === currentItem.id)
+      .map(item => item.id === currentItem.id ? replacement : item)
+    scheduleExpiry(replacement)
+  }
+
   function add(event: CaptionChannelEvent) {
     if (!event.text.trim()) {
       clearType(event.type)
       return
     }
 
+    if (event.operation === 'replace') {
+      replace(event)
+      return
+    }
+
     const item: CaptionItem = {
       id: nextId++,
       type: event.type,
       text: event.text,
     }
     items.value = [...items.value, item]
-    expiryTimers.set(item.id, setTimeout(() => {
-      remove(item.id)
-    }, ttlMs))
+    scheduleExpiry(item)
   }
 
   function dispose() {

apps/stage-tamagotchi/src/renderer/pages/caption.vue

@@ -1,4 +1,6 @@
 <script setup lang="ts">
+import type { CaptionChannelEvent } from '@proj-airi/stage-shared'
+
 import { defineInvoke } from '@moeru/eventa'
 import { useElectronEventaContext, useElectronMouseAroundWindowBorder, useElectronMouseInWindow } from '@proj-airi/electron-vueuse'
 import { createFadeAnimator, PoppinText } from '@proj-airi/stage-ui/components'
@@ -21,7 +23,6 @@ const { isNearAnyBorder: isAroundWindowBorder } = useElectronMouseAroundWindowBo
 const isAroundWindowBorderFor250Ms = refDebounced(isAroundWindowBorder, 250)
 
 // Broadcast channel for captions
-type CaptionChannelEvent = | { type: 'caption-speaker', text: string } | { type: 'caption-assistant', text: string }
 const { data } = useBroadcastChannel<CaptionChannelEvent, CaptionChannelEvent>({ name: 'airi-caption-overlay' })
 const { items: captionItems, add: addCaptionItem, dispose: disposeCaptionItems } = useCaptionItems({ ttlMs: CAPTION_TEXT_EXPIRY_MS })
 

apps/stage-tamagotchi/src/renderer/pages/index.vue

@@ -1,4 +1,5 @@
 <script setup lang="ts">
+import type { CaptionChannelEvent } from '@proj-airi/stage-shared'
 import type { ModelSettingsRuntimeSnapshot } from '@proj-airi/stage-ui/components/scenarios/settings/model-settings/runtime'
 
 import type { ModelSettingsRuntimeChannelEvent } from '../../shared/model-settings-runtime'
@@ -357,9 +358,6 @@ const voiceInputInteractionLifecycle = createVoiceInputInteractionLifecycle<Stop
 })
 
 // Caption overlay broadcast channel
-type CaptionChannelEvent
-  = | { type: 'caption-speaker', text: string }
-    | { type: 'caption-assistant', text: string }
 const { post: postCaption } = useBroadcastChannel<CaptionChannelEvent, CaptionChannelEvent>({ name: 'airi-caption-overlay' })
 
 /**
@@ -499,8 +497,8 @@ async function ensureLiveAudioInputStream() {
 /**
  * Sends voice captions as best-effort overlay updates without interrupting chat ingestion.
  */
-function postSpeakerCaption(text: string) {
-  const { error } = tryCatch(() => postCaption({ type: 'caption-speaker', text }))
+function postSpeakerCaption(text: string, operation: NonNullable<CaptionChannelEvent['operation']> = 'append') {
+  const { error } = tryCatch(() => postCaption({ operation, type: 'caption-speaker', text }))
   if (error)
     console.warn('[Main Page] Failed to post voice input caption:', error)
 }
@@ -529,16 +527,24 @@ function handleStreamingSentenceEnd(delta: string) {
   if (!finalText || !finalText.trim())
     return
 
-  postSpeakerCaption(finalText)
+  postSpeakerCaption(finalText, 'replace')
   void sendVoiceInputTextToChat(finalText)
 }
 
+/** Replaces the speaker caption with the provider's current volatile transcript. */
+function handleStreamingTranscriptionUpdate(text: string) {
+  if (isVoiceInputSuppressed())
+    return
+
+  postSpeakerCaption(text, 'replace')
+}
+
 /** Publishes the provider's final streaming-ASR text to the caption overlay. */
 function handleStreamingSpeechEnd(text: string) {
   if (isVoiceInputSuppressed())
     return
 
-  postSpeakerCaption(text)
+  postSpeakerCaption(text, 'replace')
 }
 
 /** Reads the listening generation attached to recorder-backed transcription metadata. */
@@ -593,6 +599,7 @@ async function startAudioInteractionConsumers() {
       consumerId: transcriptionConsumerId,
       onSentenceEnd: handleStreamingSentenceEnd,
       onSpeechEnd: handleStreamingSpeechEnd,
+      onTranscriptionUpdate: handleStreamingTranscriptionUpdate,
     })
 
     if (inspectVoiceInputStreamingRequestGate().skip) {

packages/stage-shared/src/caption.ts

@@ -0,0 +1,9 @@
+/** A caption update sent through the cross-window caption channel. */
+export interface CaptionChannelEvent {
+  /** Controls whether the overlay appends text or replaces the current source text. */
+  operation?: 'append' | 'replace'
+  /** Text rendered by the caption overlay. Empty text clears this source. */
+  text: string
+  /** Identifies the speaker that owns this caption text. */
+  type: 'caption-speaker' | 'caption-assistant'
+}

packages/stage-shared/src/index.ts

@@ -1,4 +1,5 @@
 export * from './artistry'
+export * from './caption'
 export * from './env-vars'
 export * from './environment'
 export * from './error-message'

packages/stage-ui/src/components/scenes/Stage.vue

@@ -1,6 +1,7 @@
 <script setup lang="ts">
 import type { Live2DLipSync, Live2DLipSyncOptions } from '@proj-airi/model-driver-lipsync'
 import type { Profile } from '@proj-airi/model-driver-lipsync/shared/wlipsync'
+import type { CaptionChannelEvent } from '@proj-airi/stage-shared'
 import type { VrmInteractionTarget } from '@proj-airi/stage-ui-three'
 import type { SpeechProviderWithExtraOptions } from '@xsai-ext/providers/utils'
 import type { UnElevenLabsOptions } from 'unspeech'
@@ -157,9 +158,6 @@ watch([stageModelRenderer, stageModelSelected, stageModelSelectedUrl], () => {
 })
 
 // Caption + Presentation broadcast channels
-type CaptionChannelEvent
-  = | { type: 'caption-speaker', text: string }
-    | { type: 'caption-assistant', text: string }
 const { post: postCaption } = useBroadcastChannel<CaptionChannelEvent, CaptionChannelEvent>({ name: 'airi-caption-overlay' })
 const assistantCaption = ref('')