feat(stage-tamagotchi): caption for audio, playback will sync chat completion messages with tts

apps/stage-tamagotchi/src/renderer/components/ChatHistory.vue

@@ -1,8 +1,9 @@
 <script setup lang="ts">
 import { MarkdownRenderer } from '@proj-airi/stage-ui/components'
 import { useChatStore } from '@proj-airi/stage-ui/stores/chat'
+import { useBroadcastChannel } from '@vueuse/core'
 import { storeToRefs } from 'pinia'
-import { nextTick, ref } from 'vue'
+import { nextTick, ref, watch } from 'vue'
 import { useI18n } from 'vue-i18n'
 
 const chatHistoryRef = ref<HTMLDivElement>()
@@ -12,6 +13,24 @@ const { messages, sending, streamingMessage } = storeToRefs(useChatStore())
 
 const { onBeforeMessageComposed, onTokenLiteral } = useChatStore()
 
+// Presentation channel: show assistant text only when corresponding TTS segment starts
+type PresentEvent
+  = | { type: 'assistant-reset' }
+    | { type: 'assistant-append', text: string }
+const { data: presentEvent } = useBroadcastChannel<PresentEvent, PresentEvent>({ name: 'airi-chat-present' })
+const presentSlices = ref<string[]>([])
+
+watch(presentEvent, (ev) => {
+  if (!ev)
+    return
+  if (ev.type === 'assistant-reset') {
+    presentSlices.value = []
+  }
+  else if (ev.type === 'assistant-append') {
+    presentSlices.value.push(ev.text)
+  }
+})
+
 onBeforeMessageComposed(async () => {
   // Scroll down to the new sent message
   nextTick().then(() => {
@@ -113,21 +132,24 @@ onTokenLiteral(async () => {
         <div>
           <span text-xs text="primary-400/90 dark:primary-600/90" font-normal class="inline <sm:hidden">{{ t('stage.chat.message.character-name.airi') }}</span>
         </div>
-        <div v-if="streamingMessage.content" class="break-words" text="primary-700 dark:primary-200">
-          <div v-for="(slice, sliceIndex) in streamingMessage.slices" :key="sliceIndex">
-            <div v-if="slice.type === 'tool-call'">
-              <div
-                p="1" border="1 solid primary-200" rounded-lg m="y-1" bg="primary-100"
-              >
-                Called: <code>{{ slice.toolCall.toolName }}</code>
-              </div>
+        <div v-if="presentSlices.length > 0 || streamingMessage.content" class="break-words" text="primary-700 dark:primary-200">
+          <!-- Prefer presentation slices if available; fallback to normal streaming -->
+          <template v-if="presentSlices.length > 0">
+            <div v-for="(text, idx) in presentSlices" :key="`present-${idx}`">
+              <MarkdownRenderer :content="text" />
             </div>
-            <div v-else-if="slice.type === 'tool-call-result'" /> <!-- this line should be unreachable -->
-            <MarkdownRenderer
-              v-else
-              :content="slice.text"
-            />
-          </div>
+          </template>
+          <template v-else>
+            <div v-for="(slice, sliceIndex) in streamingMessage.slices" :key="sliceIndex">
+              <div v-if="slice.type === 'tool-call'">
+                <div p="1" border="1 solid primary-200" rounded-lg m="y-1" bg="primary-100">
+                  Called: <code>{{ slice.toolCall.toolName }}</code>
+                </div>
+              </div>
+              <div v-else-if="slice.type === 'tool-call-result'" />
+              <MarkdownRenderer v-else :content="slice.text" />
+            </div>
+          </template>
         </div>
         <div v-else i-eos-icons:three-dots-loading />
       </div>

apps/stage-tamagotchi/src/renderer/components/Widgets/ControlsIsland/IndicatorMicVolume.vue

@@ -0,0 +1,76 @@
+<script setup lang="ts">
+import { useAudioAnalyzer } from '@proj-airi/stage-ui/composables'
+import { useAudioContext } from '@proj-airi/stage-ui/stores/audio'
+import { useSettingsAudioDevice } from '@proj-airi/stage-ui/stores/settings'
+import { storeToRefs } from 'pinia'
+import { computed, onMounted, onUnmounted, watch } from 'vue'
+
+const props = withDefaults(defineProps<{ colorClass?: string }>(), { colorClass: 'text-primary-500 dark:text-primary-200' })
+const settingsAudio = useSettingsAudioDevice()
+const { stream, enabled } = storeToRefs(settingsAudio)
+
+const { audioContext } = storeToRefs(useAudioContext())
+const { startAnalyzer, stopAnalyzer, volumeLevel } = useAudioAnalyzer()
+
+let source: MediaStreamAudioSourceNode | undefined
+
+const normalized = computed(() => Math.min(1, (volumeLevel.value ?? 0) / 100))
+
+function teardown() {
+  try { source?.disconnect() }
+  catch {}
+  source = undefined
+  stopAnalyzer()
+}
+
+async function setup() {
+  teardown()
+  if (!enabled.value || !stream.value)
+    return
+
+  const ctx = audioContext.value
+  // Ensure context is running
+  if (ctx.state === 'suspended')
+    await ctx.resume()
+  const analyser = startAnalyzer(ctx)
+  if (!analyser)
+    return
+  source = ctx.createMediaStreamSource(stream.value)
+  source.connect(analyser)
+}
+
+onMounted(() => {
+  watch([enabled, stream], () => setup(), { immediate: true })
+})
+
+onUnmounted(() => teardown())
+</script>
+
+<template>
+  <div :class="['flex items-center justify-center', props.colorClass]">
+    <!-- Inline SVG mic with level fill using gradient stops -->
+    <svg width="24" height="24" viewBox="0 0 256 256" aria-hidden="true">
+      <defs>
+        <linearGradient id="micLevel" x1="0%" y1="0%" x2="0%" y2="100%">
+          <stop offset="0%" stop-color="currentColor" stop-opacity="0" />
+          <stop :offset="`${100 - Math.round(normalized * 100)}%`" stop-color="currentColor" stop-opacity="0" />
+          <stop :offset="`${100 - Math.round(normalized * 100)}%`" stop-color="currentColor" stop-opacity="0.95" />
+          <stop offset="100%" stop-color="currentColor" stop-opacity="0.95" />
+        </linearGradient>
+      </defs>
+      <!-- Fill with level gradient -->
+      <path
+        fill="url(#micLevel)"
+        d="M128 176a48.05 48.05 0 0 0 48-48V64a48 48 0 0 0-96 0v64a48.05 48.05 0 0 0 48 48M96 64a32 32 0 0 1 64 0v64a32 32 0 0 1-64 0Zm40 143.6V240a8 8 0 0 1-16 0v-32.4A80.11 80.11 0 0 1 48 128a8 8 0 0 1 16 0a64 64 0 0 0 128 0a8 8 0 0 1 16 0a80.11 80.11 0 0 1-72 79.6"
+      />
+      <!-- Outline -->
+      <path
+        fill="none"
+        stroke="currentColor"
+        stroke-opacity="1"
+        stroke-width="2"
+        d="M128 176a48.05 48.05 0 0 0 48-48V64a48 48 0 0 0-96 0v64a48.05 48.05 0 0 0 48 48M96 64a32 32 0 0 1 64 0v64a32 32 0 0 1-64 0Zm40 143.6V240a8 8 0 0 1-16 0v-32.4A80.11 80.11 0 0 1 48 128a8 8 0 0 1 16 0a64 64 0 0 0 128 0a8 8 0 0 1 16 0a80.11 80.11 0 0 1-72 79.6"
+      />
+    </svg>
+  </div>
+</template>

apps/stage-tamagotchi/src/renderer/components/Widgets/ControlsIsland/index.vue

@@ -10,6 +10,7 @@ import { ref } from 'vue'
 import HearingPermissionStatus from '../../../components/HearingPermissionStatus.vue'
 import ControlButton from './ControlButton.vue'
 import ControlButtonTooltip from './ControlButtonTooltip.vue'
+import IndicatorMicVolume from './IndicatorMicVolume.vue'
 
 import { electronOpenSettings, electronStartDraggingWindow } from '../../../../shared/eventa'
 import { isLinux } from '../../../utils/platform'
@@ -51,12 +52,14 @@ defineExpose({ hearingDialogOpen })
 
       <ControlButtonTooltip>
         <HearingConfigDialog v-model:show="hearingDialogOpen">
-          <ControlButton>
-            <Transition name="fade" mode="out-in">
-              <div v-if="isAudioEnabled" i-ph:microphone size-5 text="neutral-800 dark:neutral-300" />
-              <div v-else i-ph:microphone-slash size-5 text="neutral-800 dark:neutral-300" />
-            </Transition>
-          </ControlButton>
+          <div class="relative">
+            <ControlButton>
+              <Transition name="fade" mode="out-in">
+                <IndicatorMicVolume v-if="isAudioEnabled" size-5 />
+                <div v-else i-ph:microphone-slash size-5 text="neutral-800 dark:neutral-300" />
+              </Transition>
+            </ControlButton>
+          </div>
           <template #extra>
             <HearingPermissionStatus />
           </template>

apps/stage-tamagotchi/src/renderer/pages/caption.vue

@@ -1,11 +1,20 @@
 <script setup lang="ts">
 import { defineInvoke } from '@unbird/eventa'
 import { createContext } from '@unbird/eventa/adapters/electron/renderer'
-import { onMounted, ref } from 'vue'
+import { useBroadcastChannel } from '@vueuse/core'
+import { onMounted, ref, watch } from 'vue'
 
 import { captionGetIsFollowingWindow, captionIsFollowingWindowChanged } from '../../shared/eventa'
 
 const attached = ref(true)
+const speakerText = ref('')
+const assistantText = ref('')
+
+// Broadcast channel for captions
+type CaptionChannelEvent
+  = | { type: 'caption-speaker', text: string }
+    | { type: 'caption-assistant', text: string }
+const { data } = useBroadcastChannel<CaptionChannelEvent, CaptionChannelEvent>({ name: 'airi-caption-overlay' })
 
 const { context } = createContext(window.electron.ipcRenderer)
 const getAttached = defineInvoke(context, captionGetIsFollowingWindow)
@@ -23,12 +32,27 @@ onMounted(async () => {
     })
   }
   catch {}
+
+  try {
+    // Update texts from broadcast channel
+    watch(data, (event) => {
+      if (!event)
+        return
+      if (event.type === 'caption-speaker') {
+        speakerText.value = event.text
+      }
+      else if (event.type === 'caption-assistant') {
+        assistantText.value = event.text
+      }
+    }, { immediate: true })
+  }
+  catch {}
 })
 </script>
 
 <template>
   <div class="pointer-events-none h-full w-full flex items-end justify-center">
-    <div class="pointer-events-auto relative select-none rounded-xl bg-primary-950/10 px-3 py-2 shadow-md backdrop-blur-md dark:bg-neutral-900/70">
+    <div class="pointer-events-auto relative select-none rounded-xl px-3 py-2">
       <div
         v-show="!attached"
         class="[-webkit-app-region:drag] absolute left-1/2 h-[14px] w-[36px] border border-[rgba(125,125,125,0.35)] rounded-[10px] bg-[rgba(125,125,125,0.28)] backdrop-blur-[6px] -top-2 -translate-x-1/2"
@@ -36,11 +60,21 @@ onMounted(async () => {
       >
         <div class="absolute left-1/2 top-1/2 h-[3px] w-4 rounded-full bg-[rgba(255,255,255,0.85)] -translate-x-1/2 -translate-y-1/2" />
       </div>
-      <div
-        class="content text-primary-50 tracking-widest font-cute text-stroke-4 text-stroke-primary-300/50 text-shadow-lg text-shadow-color-primary-700/50"
-        :style="{ paintOrder: 'stroke fill', fontSize: '2rem' }"
-      >
-        This is a test message caption overlay.
+
+      <div class="max-w-[80vw] flex flex-col gap-1">
+        <div
+          v-if="speakerText"
+          class="rounded-md px-2 py-1 text-[1.1rem] text-neutral-50 font-medium text-shadow-lg text-shadow-color-neutral-900/60"
+        >
+          {{ speakerText }}
+        </div>
+        <div
+          v-if="assistantText"
+          class="rounded-md px-2 py-1 text-[1.35rem] text-primary-50 font-semibold text-stroke-4 text-stroke-primary-300/50 text-shadow-lg text-shadow-color-primary-700/50"
+          :style="{ paintOrder: 'stroke fill' }"
+        >
+          {{ assistantText }}
+        </div>
       </div>
     </div>
   </div>

apps/stage-tamagotchi/src/renderer/pages/index.vue

@@ -13,7 +13,7 @@ import { useConsciousnessStore } from '@proj-airi/stage-ui/stores/modules/consci
 import { useHearingSpeechInputPipeline } from '@proj-airi/stage-ui/stores/modules/hearing'
 import { useProvidersStore } from '@proj-airi/stage-ui/stores/providers'
 import { useSettingsAudioDevice } from '@proj-airi/stage-ui/stores/settings'
-import { debouncedRef, watchPausable } from '@vueuse/core'
+import { debouncedRef, useBroadcastChannel, watchPausable } from '@vueuse/core'
 import { storeToRefs } from 'pinia'
 import { computed, onUnmounted, ref, toRef, watch } from 'vue'
 
@@ -115,6 +115,12 @@ const {
 
 let stopOnStopRecord: (() => void) | undefined
 
+// Caption overlay broadcast channel
+type CaptionChannelEvent
+  = | { type: 'caption-speaker', text: string }
+    | { type: 'caption-assistant', text: string }
+const { post: postCaption } = useBroadcastChannel<CaptionChannelEvent, CaptionChannelEvent>({ name: 'airi-caption-overlay' })
+
 async function startAudioInteraction() {
   try {
     await initVAD()
@@ -127,6 +133,9 @@ async function startAudioInteraction() {
       if (!text || !text.trim())
         return
 
+      // Update caption overlay speaker text via BroadcastChannel
+      postCaption({ type: 'caption-speaker', text })
+
       try {
         const provider = await providersStore.getProviderInstance(activeChatProvider.value)
         if (!provider || !activeChatModel.value)
@@ -176,6 +185,8 @@ watch([stream, () => vadLoaded.value], async ([s, loaded]) => {
     }
   }
 })
+
+// Assistant caption is broadcast from Stage.vue via the same channel
 </script>
 
 <template>

packages/stage-ui/src/components/scenes/Stage.vue

@@ -9,6 +9,7 @@ import { drizzle } from '@proj-airi/drizzle-duckdb-wasm'
 import { getImportUrlBundles } from '@proj-airi/drizzle-duckdb-wasm/bundles/import-url-browser'
 import { withBase } from '@proj-airi/stage-shared'
 import { ThreeScene, useModelStore } from '@proj-airi/stage-ui-three'
+import { useBroadcastChannel } from '@vueuse/core'
 // import { createTransformers } from '@xsai-transformers/embed'
 // import embedWorkerURL from '@xsai-transformers/embed/worker?worker&url'
 // import { embed } from '@xsai/embed'
@@ -64,7 +65,7 @@ const { textSegmentationQueue } = storeToRefs(textSegmentationStore)
 clearTextSegmentationHooks()
 
 const characterSpeechPlaybackQueue = usePipelineCharacterSpeechPlaybackQueueStore()
-const { connectAudioContext, connectAudioAnalyser, clearAll } = characterSpeechPlaybackQueue
+const { connectAudioContext, connectAudioAnalyser, clearAll, onPlaybackStarted } = characterSpeechPlaybackQueue
 const { currentAudioSource, playbackQueue } = storeToRefs(characterSpeechPlaybackQueue)
 
 const settingsStore = useSettings()
@@ -85,6 +86,18 @@ const vrmStore = useModelStore()
 
 const showStage = ref(true)
 
+// Caption + Presentation broadcast channels
+type CaptionChannelEvent
+  = | { type: 'caption-speaker', text: string }
+    | { type: 'caption-assistant', text: string }
+const { post: postCaption } = useBroadcastChannel<CaptionChannelEvent, CaptionChannelEvent>({ name: 'airi-caption-overlay' })
+const assistantCaption = ref('')
+
+type PresentEvent
+  = | { type: 'assistant-reset' }
+    | { type: 'assistant-append', text: string }
+const { post: postPresent } = useBroadcastChannel<PresentEvent, PresentEvent>({ name: 'airi-chat-present' })
+
 // TODO: duplicate calls may happen if this component mounted multiple times
 live2dStore.onShouldUpdateView(async () => {
   showStage.value = false
@@ -216,6 +229,10 @@ onBeforeMessageComposed(async () => {
   clearAll()
   setupAnalyser()
   setupLipSync()
+  // Reset assistant caption for a new message
+  assistantCaption.value = ''
+  postCaption({ type: 'caption-assistant', text: '' })
+  postPresent({ type: 'assistant-reset' })
 })
 
 onBeforeSend(async () => {
@@ -223,6 +240,7 @@ onBeforeSend(async () => {
 })
 
 onTokenLiteral(async (literal) => {
+  // Only push to segmentation; visual presentation happens on playback start
   textSegmentationQueue.value.enqueue(literal)
 })
 
@@ -264,6 +282,12 @@ function canvasElement() {
 defineExpose({
   canvasElement,
 })
+
+onPlaybackStarted(({ text }) => {
+  assistantCaption.value += ` ${text}`
+  postCaption({ type: 'caption-assistant', text: assistantCaption.value })
+  postPresent({ type: 'assistant-append', text })
+})
 </script>
 
 <template>

packages/stage-ui/src/composables/queues.ts

@@ -105,14 +105,14 @@ export function useDelayMessageQueue() {
 
 export const usePipelineCharacterSpeechPlaybackQueueStore = defineStore('pipelines:character:speech', () => {
   // Hooks
-  const onPlaybackStartedHooks = ref<Array<() => Promise<void> | void>>([])
-  const onPlaybackFinishedHooks = ref<Array<() => Promise<void> | void>>([])
+  const onPlaybackStartedHooks = ref<Array<(payload: { text: string }) => Promise<void> | void>>([])
+  const onPlaybackFinishedHooks = ref<Array<(payload: { text: string }) => Promise<void> | void>>([])
 
   // Hooks registers
-  function onPlaybackStarted(hook: () => Promise<void> | void) {
+  function onPlaybackStarted(hook: (payload: { text: string }) => Promise<void> | void) {
     onPlaybackStartedHooks.value.push(hook)
   }
-  function onPlaybackFinished(hook: () => Promise<void> | void) {
+  function onPlaybackFinished(hook: (payload: { text: string }) => Promise<void> | void) {
     onPlaybackFinishedHooks.value.push(hook)
   }
 
@@ -162,16 +162,12 @@ export const usePipelineCharacterSpeechPlaybackQueueStore = defineStore('pipelin
             source.connect(audioAnalyser.value!)
 
             // Start playing the audio
-            for (const hook of onPlaybackStartedHooks.value) {
-              hook()
-            }
+            for (const hook of onPlaybackStartedHooks.value) hook({ text: ctx.data.text })
 
             currentAudioSource.value = source
             source.start(0)
             source.onended = () => {
-              for (const hook of onPlaybackFinishedHooks.value) {
-                hook()
-              }
+              for (const hook of onPlaybackFinishedHooks.value) hook({ text: ctx.data.text })
               if (currentAudioSource.value === source) {
                 currentAudioSource.value = undefined
               }