refactor(stage-ui): transcription playground

packages/i18n/src/locales/en/settings.yaml

@@ -437,6 +437,9 @@ pages:
       xai:
         description: x.ai
         title: xAI
+      transcriptions:
+        playground:
+          title: Transcription Playground
     title: Providers
   scene:
     description: Configure the environment where the character lives

packages/i18n/src/locales/zh-Hans/settings.yaml

@@ -415,6 +415,9 @@ pages:
       xai:
         description: X.AI
         title: xAI
+      transcriptions:
+        playground:
+          title: 实验平台
     title: 服务来源
   scene:
     description: 配置角色所在环境

packages/stage-ui/src/components/Scenarios/Providers/TranscriptionPlayground.vue

@@ -2,11 +2,12 @@
 import type { GenerateTranscriptionResult } from '@xsai/generate-transcription'
 
 import { FieldRange, FieldSelect } from '@proj-airi/ui'
-import { useDevicesList } from '@vueuse/core'
+import { until, useDevicesList, useUserMedia } from '@vueuse/core'
 import { computed, nextTick, onMounted, onUnmounted, ref, watch } from 'vue'
+import { useI18n } from 'vue-i18n'
 
 import { useAudioRecorder } from '../../../composables/audio/audio-recorder'
-import { LevelMeter, ThresholdMeter } from '../../Gadgets'
+import { LevelMeter, TestDummyMarker, ThresholdMeter } from '../../Gadgets'
 import { Button } from '../../Misc'
 
 const props = defineProps<{
@@ -16,25 +17,24 @@ const props = defineProps<{
   apiKeyConfigured?: boolean
 }>()
 
+const { t } = useI18n()
 const devices = useDevicesList({ constraints: { audio: true }, requestPermissions: true })
-
+const selectedAudioInput = ref<string>(devices.audioInputs.value[0]?.deviceId || '')
+const deviceConstraints = computed<MediaStreamConstraints>(() => ({ audio: { deviceId: { exact: selectedAudioInput.value } } }))
+const { stream, stop, start, enabled } = useUserMedia({ constraints: deviceConstraints, enabled: false, autoSwitch: false })
 const audioInputs = computed(() => devices.audioInputs.value)
 
-const selectedAudioInput = ref<string>(devices.audioInputs.value[0]?.deviceId || '')
 const speakingThreshold = ref(25) // 0-100 (for volume-based fallback)
-
-const errorMessage = ref<string>('')
-
-const volumeLevel = ref(0) // 0-100
-
 const isMonitoring = ref(false)
 const isSpeaking = ref(false)
 
+const errorMessage = ref<string>('')
+
 const audioContext = ref<AudioContext>()
-const mediaStream = ref<MediaStream>()
 const analyser = ref<AnalyserNode>()
 const dataArray = ref<Uint8Array>()
 const animationFrame = ref<number>()
+const volumeLevel = ref(0) // 0-100
 
 const audios = ref<Blob[]>([])
 const audioCleanups = ref<(() => void)[]>([])
@@ -47,32 +47,21 @@ const audioURLs = computed(() => {
 })
 const transcriptions = ref<string[]>([])
 
-const { startRecord, stopRecord } = useAudioRecorder(mediaStream)
+const { startRecord, stopRecord, onStop } = useAudioRecorder(stream)
 
 // Audio monitoring
 async function setupAudioMonitoring() {
   try {
-    if (!selectedAudioInput.value) {
-      console.warn('No audio input device selected')
-      return
-    }
-
     // Clean up existing connections
     await stopAudioMonitoring()
 
-    // Get user media with selected device
-    mediaStream.value = await navigator.mediaDevices.getUserMedia({
-      audio: {
-        deviceId: selectedAudioInput.value,
-        echoCancellation: true,
-        noiseSuppression: true,
-        autoGainControl: true,
-      },
-    })
+    enabled.value = true
+    await start()
+    await until(stream).toBeTruthy()
 
     // Create audio context
     audioContext.value = new AudioContext()
-    const source = audioContext.value.createMediaStreamSource(mediaStream.value)
+    const source = audioContext.value.createMediaStreamSource(stream.value!)
 
     // Create analyser for volume detection
     analyser.value = audioContext.value.createAnalyser()
@@ -103,9 +92,9 @@ async function stopAudioMonitoring() {
   }
 
   // Stop media stream
-  if (mediaStream.value) {
-    mediaStream.value.getTracks().forEach(track => track.stop())
-    mediaStream.value = undefined
+  if (stream.value) {
+    stream.value.getTracks().forEach(track => track.stop())
+    stream.value = undefined
   }
 
   // Close audio context
@@ -114,6 +103,9 @@ async function stopAudioMonitoring() {
     audioContext.value = undefined
   }
 
+  await stopRecord()
+  await stop()
+
   analyser.value = undefined
   dataArray.value = undefined
   volumeLevel.value = 0
@@ -152,7 +144,7 @@ watch(selectedAudioInput, async () => {
 
 watch(audioInputs, () => {
   if (!selectedAudioInput.value && audioInputs.value.length > 0) {
-    selectedAudioInput.value = audioInputs.value[0]?.deviceId
+    selectedAudioInput.value = audioInputs.value.find(input => input.deviceId === 'default')?.deviceId || audioInputs.value[0].deviceId
   }
 })
 
@@ -165,11 +157,22 @@ async function toggleMonitoring() {
   }
   else {
     await stopAudioMonitoring()
-    const audio = await stopRecord()
-    audios.value.push(audio)
 
-    const res = await props.generateTranscription(new File([audio], 'audio.wav', { type: 'audio/wav' }))
-    transcriptions.value.push(res.text)
+    onStop(async (recording) => {
+      try {
+        if (recording) {
+          audios.value.push(recording)
+          const res = await props.generateTranscription(new File([recording], 'recording.wav'))
+          transcriptions.value.push(res.text)
+        }
+      }
+      catch (err) {
+        errorMessage.value = err instanceof Error ? err.message : String(err)
+        console.error('Error generating transcription:', errorMessage.value)
+      }
+    })
+
+    await stopRecord()
 
     isMonitoring.value = false
   }
@@ -187,7 +190,7 @@ const speakingIndicatorClass = computed(() => {
 onMounted(() => {
   devices.ensurePermissions().then(() => nextTick()).then(() => {
     if (audioInputs.value.length > 0 && !selectedAudioInput.value) {
-      selectedAudioInput.value = audioInputs.value[0]?.deviceId
+      selectedAudioInput.value = audioInputs.value.find(input => input.deviceId === 'default')?.deviceId || audioInputs.value[0].deviceId
     }
   })
 })
@@ -199,6 +202,15 @@ onUnmounted(() => {
 
 <template>
   <div w-full pt-1>
+    <h2 class="mb-4 text-lg text-neutral-500 md:text-2xl dark:text-neutral-400" w-full>
+      <div class="inline-flex items-center gap-4">
+        <TestDummyMarker />
+        <div>
+          {{ t('settings.pages.providers.provider.transcriptions.playground.title') }}
+        </div>
+      </div>
+    </h2>
+
     <!-- Audio Input Selection -->
     <div mb-2>
       <FieldSelect
@@ -215,21 +227,8 @@ onUnmounted(() => {
       />
     </div>
 
-    <Button class="mb-4" w-full @click="toggleMonitoring">
-      {{ isMonitoring ? 'Stop Monitoring' : 'Start Monitoring' }}
-    </Button>
-
-    <div>
-      <div v-for="(audio, index) in audioURLs" :key="index" class="mb-2">
-        <audio :src="audio" controls class="w-full" />
-        <div v-if="transcriptions[index]" class="mt-2 text-sm text-neutral-500 dark:text-neutral-400">
-          {{ transcriptions[index] }}
-        </div>
-      </div>
-    </div>
-
     <!-- Audio Level Visualization -->
-    <div v-if="isMonitoring" class="space-y-3">
+    <div class="space-y-3">
       <!-- Volume Meter -->
       <LevelMeter :level="volumeLevel" label="Input Level" />
 
@@ -266,5 +265,18 @@ onUnmounted(() => {
         </span>
       </div>
     </div>
+
+    <Button class="my-4" w-full @click="toggleMonitoring">
+      {{ isMonitoring ? 'Stop Monitoring' : 'Start Monitoring' }}
+    </Button>
+
+    <div>
+      <div v-for="(audio, index) in audioURLs" :key="index" class="mb-2">
+        <audio :src="audio" controls class="w-full" />
+        <div v-if="transcriptions[index]" class="mt-2 text-sm text-neutral-500 dark:text-neutral-400">
+          {{ transcriptions[index] }}
+        </div>
+      </div>
+    </div>
   </div>
 </template>

packages/stage-ui/src/components/Scenarios/Providers/TranscriptionProviderSettings.vue

@@ -10,6 +10,7 @@ import {
   ProviderBaseUrlInput,
   ProviderBasicSettings,
   ProviderSettingsContainer,
+  ProviderSettingsLayout2,
 } from '.'
 import { useProvidersStore } from '../../../stores'
 

packages/stage-ui/src/composables/audio/audio-recorder.ts

@@ -1,49 +1,96 @@
 import type { MaybeRefOrGetter } from 'vue'
 
 import { until } from '@vueuse/core'
-import { ref, toRef } from 'vue'
+import { ref, shallowRef, toRef, watch } from 'vue'
+
+export function useDownload(url: string, fileName: string) {
+  const link = document.createElement('a')
+  link.href = url
+  link.download = fileName
+  document.body.appendChild(link)
+  link.click()
+  document.body.removeChild(link)
+}
 
 export function useAudioRecorder(
   media: MaybeRefOrGetter<MediaStream | undefined>,
-  start: () => Promise<void> = () => Promise.resolve(),
 ) {
   const audioRecorder = ref<MediaRecorder>()
   const mediaRef = toRef(media)
+  const recordingChunk = shallowRef<Blob[]>([])
+  const recording = ref<Blob>()
+
+  const onStopHooks = ref<Array<(recording: Blob | undefined) => Promise<void>>>([])
 
   async function startRecord() {
-    await start()
     await until(mediaRef).toBeTruthy()
-
     if (!mediaRef.value) {
-      console.error('No media media available')
       return
     }
 
+    recording.value = undefined
+    recordingChunk.value = []
+
     audioRecorder.value = new MediaRecorder(mediaRef.value)
     audioRecorder.value.start()
+
+    audioRecorder.value.onerror = (event) => {
+      console.error('Error recording audio:', event)
+    }
+
+    audioRecorder.value.onstop = () => {
+      if (recordingChunk.value.length > 0) {
+        const blob = new Blob(recordingChunk.value, { type: audioRecorder.value?.mimeType })
+        recording.value = blob
+
+        if (blob.size > 0) {
+          useDownload(URL.createObjectURL(blob), `recording-${Date.now()}.webm`)
+        }
+      }
+      else {
+        recording.value = undefined
+      }
+
+      for (const hook of onStopHooks.value) {
+        hook(recording.value)
+      }
+    }
+
+    audioRecorder.value.ondataavailable = (event) => {
+      recordingChunk.value.push(event.data)
+    }
   }
 
-  function stopRecord() {
-    return new Promise<Blob>((resolve, reject) => {
-      if (!audioRecorder.value) {
-        return
-      }
+  function onStop(callback: (recording: Blob | undefined) => Promise<void>) {
+    onStopHooks.value.push(callback)
+  }
 
-      audioRecorder.value.onerror = (event) => {
-        console.error('Error recording audio:', event)
-        reject(event)
-      }
-      audioRecorder.value.ondataavailable = (event) => {
-        const audioBlob = event.data
-        resolve(audioBlob)
-      }
+  async function stopRecord() {
+    if (!audioRecorder.value) {
+      return []
+    }
 
+    audioRecorder.value?.stop()
+    return recordingChunk.value
+  }
+
+  watch(mediaRef, () => {
+    if (audioRecorder.value && audioRecorder.value.state === 'recording') {
       audioRecorder.value.stop()
-    })
-  }
+    }
+
+    audioRecorder.value = undefined
+
+    if (mediaRef.value && mediaRef.value.active) {
+      startRecord()
+    }
+  })
 
   return {
     startRecord,
     stopRecord,
+    onStop,
+    recordingChunk,
+    recording,
   }
 }