fix(stage-tamagotchi): sync hearing text with chat input (#2264)

apps/stage-tamagotchi/src/renderer/components/InteractiveArea.vue

@@ -21,10 +21,12 @@ import { useRouter } from 'vue-router'
 
 import JournalToolCallBlock from './chat-tool-renderers/journal-tool-call-block.vue'
 
+import { useHearingInputChannel } from '../composables/use-hearing-input-channel'
 import { artistryToolReferences, widgetToolReferences } from '../stores/tools'
 
 const router = useRouter()
 const messageInput = ref('')
+useHearingInputChannel(messageInput)
 const lastEnterTime = ref(0)
 const attachments = ref<{ type: 'image', data: string, mimeType: string, url: string }[]>([])
 

apps/stage-tamagotchi/src/renderer/composables/use-hearing-input-channel.test.ts

@@ -0,0 +1,82 @@
+import type { HearingInputChannelEvent } from '@proj-airi/stage-shared'
+
+import { hearingInputChannelName } from '@proj-airi/stage-shared'
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+import { nextTick, ref, shallowRef } from 'vue'
+
+import { useHearingInputChannel } from './use-hearing-input-channel'
+
+const broadcastChannelMock = vi.hoisted(() => ({
+  useBroadcastChannel: vi.fn(),
+}))
+
+vi.mock('@vueuse/core', () => ({
+  useBroadcastChannel: broadcastChannelMock.useBroadcastChannel,
+}))
+
+describe('useHearingInputChannel', () => {
+  let data: ReturnType<typeof shallowRef<HearingInputChannelEvent | undefined>>
+
+  beforeEach(() => {
+    data = shallowRef<HearingInputChannelEvent>()
+    broadcastChannelMock.useBroadcastChannel.mockReset()
+    broadcastChannelMock.useBroadcastChannel.mockReturnValue({ data })
+  })
+
+  it('listens on the shared Hearing input channel', () => {
+    useHearingInputChannel(ref(''))
+
+    expect(broadcastChannelMock.useBroadcastChannel).toHaveBeenCalledWith({
+      name: hearingInputChannelName,
+    })
+  })
+
+  it('replaces Provider revisions and clears only the owned suffix', async () => {
+    const input = ref('manual note')
+    useHearingInputChannel(input)
+
+    data.value = { operation: 'replace', sourceId: 'utterance-1', text: 'hello' }
+    await nextTick()
+    expect(input.value).toBe('manual note hello')
+
+    data.value = { operation: 'replace', sourceId: 'utterance-1', text: 'hello world' }
+    await nextTick()
+    expect(input.value).toBe('manual note hello world')
+
+    data.value = { operation: 'clear', sourceId: 'utterance-1' }
+    await nextTick()
+    expect(input.value).toBe('manual note')
+  })
+
+  it('ignores stale cleanup after a new Provider utterance starts', async () => {
+    const input = ref('')
+    useHearingInputChannel(input)
+
+    data.value = { operation: 'replace', sourceId: 'utterance-1', text: 'first' }
+    await nextTick()
+    data.value = { operation: 'replace', sourceId: 'utterance-2', text: 'second' }
+    await nextTick()
+    expect(input.value).toBe('second')
+
+    data.value = { operation: 'clear', sourceId: 'utterance-1' }
+    await nextTick()
+    expect(input.value).toBe('second')
+
+    data.value = { operation: 'clear', sourceId: 'utterance-2' }
+    await nextTick()
+    expect(input.value).toBe('')
+  })
+
+  it('does not replace text after the user edits the Provider-owned suffix', async () => {
+    const input = ref('')
+    useHearingInputChannel(input)
+
+    data.value = { operation: 'replace', sourceId: 'utterance-1', text: 'draft' }
+    await nextTick()
+    input.value = 'user edit'
+
+    data.value = { operation: 'replace', sourceId: 'utterance-1', text: 'provider revision' }
+    await nextTick()
+    expect(input.value).toBe('user edit')
+  })
+})

apps/stage-tamagotchi/src/renderer/composables/use-hearing-input-channel.ts

@@ -0,0 +1,39 @@
+import type { HearingInputChannelEvent } from '@proj-airi/stage-shared'
+import type { Ref } from 'vue'
+
+import { hearingInputChannelName } from '@proj-airi/stage-shared'
+import { useStreamingTranscriptionInput } from '@proj-airi/stage-ui/composables/use-streaming-transcription-input'
+import { useBroadcastChannel } from '@vueuse/core'
+import { watch } from 'vue'
+
+/** Applies cross-window Hearing updates to one editable chat input. */
+export function useHearingInputChannel(input: Ref<string>) {
+  const streamingInput = useStreamingTranscriptionInput(input)
+  const { data } = useBroadcastChannel<HearingInputChannelEvent, HearingInputChannelEvent>({
+    name: hearingInputChannelName,
+  })
+  let activeSourceId: string | undefined
+
+  watch(data, (event) => {
+    if (!event)
+      return
+
+    if (event.operation === 'replace') {
+      if (!event.text.trim())
+        return
+
+      if (activeSourceId && activeSourceId !== event.sourceId)
+        streamingInput.clear()
+
+      activeSourceId = event.sourceId
+      streamingInput.replace(event.text)
+      return
+    }
+
+    if (event.sourceId !== activeSourceId)
+      return
+
+    streamingInput.clear()
+    activeSourceId = undefined
+  })
+}

apps/stage-tamagotchi/src/renderer/pages/index.vue

@@ -1,5 +1,5 @@
 <script setup lang="ts">
-import type { CaptionChannelEvent } from '@proj-airi/stage-shared'
+import type { CaptionChannelEvent, HearingInputChannelEvent } from '@proj-airi/stage-shared'
 import type { ModelSettingsRuntimeSnapshot } from '@proj-airi/stage-ui/components/scenarios/settings/model-settings/runtime'
 
 import type { ModelSettingsRuntimeChannelEvent } from '../../shared/model-settings-runtime'
@@ -14,7 +14,7 @@ import {
   useElectronRelativeMouse,
 } from '@proj-airi/electron-vueuse'
 import { createTranscriptBuffer } from '@proj-airi/pipelines-audio'
-import { IS_DEV } from '@proj-airi/stage-shared'
+import { hearingInputChannelName, IS_DEV } from '@proj-airi/stage-shared'
 import { useModelStore, useThreeSceneIsTransparentAtPoint } from '@proj-airi/stage-ui-three'
 import { HoloCoupon } from '@proj-airi/stage-ui/components'
 import {
@@ -359,6 +359,46 @@ const voiceInputInteractionLifecycle = createVoiceInputInteractionLifecycle<Stop
 
 // Caption overlay broadcast channel
 const { post: postCaption } = useBroadcastChannel<CaptionChannelEvent, CaptionChannelEvent>({ name: 'airi-caption-overlay' })
+const { post: postHearingInput } = useBroadcastChannel<HearingInputChannelEvent, HearingInputChannelEvent>({ name: hearingInputChannelName })
+const hearingInputClearTimers = new Map<ReturnType<typeof setTimeout>, string>()
+let hearingInputSequence = 0
+let activeHearingInputSourceId: string | undefined
+
+function currentHearingInputSourceId() {
+  activeHearingInputSourceId ??= `stage-tamagotchi:${++hearingInputSequence}`
+  return activeHearingInputSourceId
+}
+
+function postHearingInputEvent(event: HearingInputChannelEvent) {
+  const { error } = tryCatch(() => postHearingInput(event))
+  if (error)
+    console.warn('[Main Page] Failed to post Hearing input text:', error)
+}
+
+function replaceHearingInput(text: string) {
+  postHearingInputEvent({
+    operation: 'replace',
+    sourceId: currentHearingInputSourceId(),
+    text,
+  })
+}
+
+function clearHearingInput(sourceId = activeHearingInputSourceId) {
+  if (!sourceId)
+    return
+
+  postHearingInputEvent({ operation: 'clear', sourceId })
+  if (sourceId === activeHearingInputSourceId)
+    activeHearingInputSourceId = undefined
+}
+
+function scheduleHearingInputClear(sourceId: string) {
+  const timer = setTimeout(() => {
+    hearingInputClearTimers.delete(timer)
+    clearHearingInput(sourceId)
+  }, 250)
+  hearingInputClearTimers.set(timer, sourceId)
+}
 
 /**
  * Reports a voice input pipeline failure to both the console and visible app UI.
@@ -527,6 +567,10 @@ function handleStreamingSentenceEnd(delta: string) {
   if (!finalText || !finalText.trim())
     return
 
+  const sourceId = currentHearingInputSourceId()
+  replaceHearingInput(finalText)
+  scheduleHearingInputClear(sourceId)
+  activeHearingInputSourceId = undefined
   postSpeakerCaption(finalText, 'replace')
   void sendVoiceInputTextToChat(finalText)
 }
@@ -536,6 +580,7 @@ function handleStreamingTranscriptionUpdate(text: string) {
   if (isVoiceInputSuppressed())
     return
 
+  replaceHearingInput(text)
   postSpeakerCaption(text, 'replace')
 }
 
@@ -627,6 +672,7 @@ async function stopAudioInteractionConsumers(options: StopAudioInteractionOption
   const flushTranscript = options.flushTranscript ?? true
 
   clearAssistantSpeechResumeTimer()
+  clearHearingInput()
   voiceInputGeneration += 1
   removeStreamingTranscriptionConsumer(transcriptionConsumerId)
 
@@ -696,6 +742,12 @@ onMounted(() => {
 })
 
 onUnmounted(() => {
+  for (const [timer, sourceId] of hearingInputClearTimers) {
+    clearTimeout(timer)
+    clearHearingInput(sourceId)
+  }
+  hearingInputClearTimers.clear()
+  clearHearingInput()
   postModelSettingsRuntimeEvent({
     type: 'owner-gone',
     ownerInstanceId: modelSettingsRuntimeOwnerInstanceId,

packages/stage-shared/src/hearing.ts

@@ -0,0 +1,19 @@
+/** Shared channel name for streaming Hearing text shown in editable chat input. */
+export const hearingInputChannelName = 'airi-hearing-input'
+
+/** A streaming Hearing update sent from the microphone owner to a chat window. */
+export type HearingInputChannelEvent
+  = | {
+    /** Removes text owned by the matching Provider utterance. */
+    operation: 'clear'
+    /** Correlates delayed cleanup with the matching Provider utterance. */
+    sourceId: string
+  }
+  | {
+    /** Replaces the current text owned by this Provider utterance. */
+    operation: 'replace'
+    /** Correlates transcript revisions with the matching Provider utterance. */
+    sourceId: string
+    /** Contains the complete current utterance. */
+    text: string
+  }

packages/stage-shared/src/index.ts

@@ -4,6 +4,7 @@ export * from './env-vars'
 export * from './environment'
 export * from './error-message'
 export * from './export-csv'
+export * from './hearing'
 export * from './perf/io-trace'
 export * from './perf/tracer'
 export * from './url'