nanakura/moeka-project
1
0
Fork 0
Code Issues Pull Requests 1 Actions 2 Packages Projects Releases Wiki Activity

refactor(server): streaming tts support model switch

Browse Source
This commit is contained in:
RainbowBird
2026-05-18 23:35:38 +08:00
parent 2854e0accc
commit efe0f8ac19
10 changed files with 35 additions and 194 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/server/src/routes/openai/v1/index.ts
+10 -2
View File
@@ -647,11 +647,16 @@ export function createV1Routes(
* configured yet so the client can render "no voices" instead of
* exploding.
*/
async function handleListStreamingVoices(_c: Context<HonoEnv>) {
async function handleListStreamingVoices(c: Context<HonoEnv>) {
const upstream = await configKV.getOptional('STREAMING_TTS_UPSTREAM')
if (!upstream || !upstream.baseURL)
return Response.json({ voices: [], recommended: {} })
// Pass through the api_resource_id (e.g. `seed-tts-2.0`). unspeech
// filters the embedded Volcengine catalogue server-side; absent model
// means "return everything streaming-safe".
const model = c.req.query('model')
let voicesURL: string
try {
const u = new URL(upstream.baseURL)
@@ -659,7 +664,10 @@ export function createV1Routes(
// stream and the REST voices endpoint on the same listener.
u.protocol = u.protocol === 'wss:' ? 'https:' : 'http:'
u.pathname = '/api/voices'
u.search = '?provider=volcengine'
const params = new URLSearchParams({ provider: 'volcengine' })
if (model)
params.set('model', model)
u.search = `?${params.toString()}`
voicesURL = u.toString()
}
catch (err) {
packages/stage-pages/src/pages/settings/providers/speech/official-provider-speech-streaming.vue
+4 -6
View File
@@ -48,7 +48,7 @@ const voicesLoading = ref(false)
async function loadVoices() {
voicesLoading.value = true
try {
await speechStore.loadVoicesForProvider(providerId)
await speechStore.loadVoicesForProvider(providerId, model.value)
}
finally {
voicesLoading.value = false
@@ -62,11 +62,9 @@ onMounted(async () => {
await loadVoices()
})
// Reload voices when the model variant changes. The streaming provider
// shares one voice catalogue across model variants (both Seed-TTS 2.0 and
// 1.0 expose the same `zh_female_*` ids), so this is mostly a refresh —
// but it keeps the flow consistent with provider pages where the variant
// actually does change the catalogue.
// Volcengine TTS 1.0 and 2.0 ship different voice catalogues (mars/moon/ICL
// vs uranus/saturn; see unspeech voices.go). Re-fetch on model change so the
// list switches accordingly.
watch(model, async () => {
await loadVoices()
})
packages/stage-ui/src/components/scenarios/providers/speech-playground-openai-compatible.vue
-5
View File
@@ -178,11 +178,6 @@ defineExpose({
{{ errorMessage }}
</div>
<audio v-if="audioUrl" ref="audioPlayer" :src="audioUrl" controls class="mt-2 w-full" />
<SpeechStreamingPlayground
:text="testText"
:voice="voice"
:generate-speech="generateSpeech"
/>
</div>
<!-- Slot for additional provider-specific UI in the playground -->
<slot />
packages/stage-ui/src/components/scenarios/providers/speech-playground.vue
-8
View File
@@ -5,8 +5,6 @@ import { FieldCheckbox, FieldCombobox } from '@proj-airi/ui'
import { computed, onUnmounted, ref, watch } from 'vue'
import { useI18n } from 'vue-i18n'
import SpeechStreamingPlayground from './speech-streaming-playground.vue'
import { TestDummyMarker } from '../../gadgets'
const props = defineProps<{
@@ -195,12 +193,6 @@ defineExpose({
{{ errorMessage }}
</div>
<audio v-if="audioUrl" ref="audioPlayer" :src="audioUrl" controls class="mt-2 w-full" />
<SpeechStreamingPlayground
:text="testText"
:voice="selectedVoice"
:generate-speech="generateSpeech"
/>
</div>
<!-- Slot for additional provider-specific UI in the playground -->
<slot />
packages/stage-ui/src/components/scenarios/providers/speech-streaming-playground.vue
-165
View File
@@ -1,165 +0,0 @@
<script setup lang="ts">
import type { TTSInputChunk } from '../../../utils/tts'
import { createQueue } from '@proj-airi/stream-kit'
import { animate } from 'animejs'
import { ref } from 'vue'
import { useAudioContext } from '../../../stores/audio'
import { chunkTTSInput } from '../../../utils/tts'
const props = defineProps<{
text: string
// Provider-specific handlers (provided from parent)
generateSpeech: (input: string, voice: string, useSSML: boolean) => Promise<ArrayBuffer>
voice: string
}>()
const { audioContext } = useAudioContext()
const nowSpeaking = ref(false)
const ttsInputChunks = ref<TTSInputChunk[]>([])
const speechGenerationIndex = ref(-1)
const audioQueue = createQueue<{ audioBuffer: AudioBuffer, text: string }>({
handlers: [
(ctx) => {
return new Promise((resolve) => {
const source = audioContext.createBufferSource()
source.buffer = ctx.data.audioBuffer
source.connect(audioContext.destination)
nowSpeaking.value = true
source.start(0)
source.onended = () => {
nowSpeaking.value = false
resolve()
}
})
},
],
})
async function handleSpeechGeneration(ctx: { data: string }) {
speechGenerationIndex.value++
try {
const input = ctx.data
const res = await props.generateSpeech(input, props.voice, false)
const audioBuffer = await audioContext.decodeAudioData(res)
audioQueue.enqueue({ audioBuffer, text: ctx.data })
}
catch (error) {
console.error('Speech generation failed:', error)
}
}
const ttsQueue = createQueue<string>({ handlers: [handleSpeechGeneration] })
async function testStreaming() {
speechGenerationIndex.value = -1
for await (const chunk of chunkTTSInput(props.text, { boost: 1, minimumWords: 4, maximumWords: 12 })) {
if (!chunk.text)
continue
ttsQueue.enqueue(chunk.text)
}
}
async function testChunking() {
const chunks: TTSInputChunk[] = []
const stream = new ReadableStream<Uint8Array>({
start(controller) {
controller.enqueue(new TextEncoder().encode(props.text))
controller.close()
},
})
for await (const chunk of chunkTTSInput(stream.getReader(), { boost: 1, minimumWords: 4, maximumWords: 12 })) {
chunks.push(chunk)
}
ttsInputChunks.value = chunks
}
</script>
<template>
<div class="flex items-center gap-1 text-sm font-medium">
Streaming Playground
</div>
<div flex="~ row" gap-4>
<button
border="neutral-800 dark:neutral-200 solid 2" transition="border duration-250 ease-in-out"
rounded-lg px-4 text="neutral-100 dark:neutral-900" py-2 text-sm
bg="neutral-700 dark:neutral-300" @click="testChunking"
>
<div flex="~ row" items-center gap-2>
<div i-solar:round-double-alt-arrow-right-bold-duotone />
<span>Test chunking</span>
</div>
</button>
<button
v-if="ttsInputChunks.length > 0"
border="neutral-800 dark:neutral-200 solid 2" transition="border duration-250 ease-in-out"
rounded-lg px-4 text="neutral-100 dark:neutral-900" py-2 text-sm
bg="neutral-700 dark:neutral-300" @click="testStreaming"
>
<div flex="~ row" items-center gap-2>
<div i-solar:round-double-alt-arrow-right-bold-duotone />
<span>Test streaming</span>
</div>
</button>
</div>
<div flex="~ col gap-2 items-start" py-4>
<div
v-for="(chunk, i) in ttsInputChunks"
:key="i"
flex="~ row gap-2 items-center"
>
<div
flex="~ row gap-2 items-center"
rounded-xl px-2 py-1.5
:class="{
'bg-neutral-100 dark:bg-neutral-800': speechGenerationIndex < i,
'bg-neutral-200 dark:bg-neutral-700': speechGenerationIndex >= i,
}"
>
<span ml-1>{{ chunk.text }}</span>
<span
rounded-full px-2 py-.5 text-nowrap text-xs
b="~ dashed"
:class="{
'b-green text-green': chunk.reason === 'boost',
'b-orange text-orange': chunk.reason === 'limit',
'b-red text-red': chunk.reason === 'hard',
'b-purple text-purple': chunk.reason === 'flush',
}"
>
{{ chunk.words }} words,
{{ chunk.reason }}
</span>
</div>
<Transition
:css="false"
@enter="(el) => animate(el, {
opacity: [0, 1],
translateX: [10, 0],
duration: 200,
ease: 'inOut',
})"
>
<div
v-if="speechGenerationIndex >= i"
tag="div"
flex="~ row items-center gap-1"
text-sm
>
<div i-solar-check-circle-line-duotone />
<div>Queued</div>
</div>
</Transition>
</div>
</div>
</template>
packages/stage-ui/src/libs/providers/providers/official/index.ts
+10 -2
View File
@@ -223,15 +223,23 @@ export const providerOfficialSpeechStreaming = defineProvider({
},
]
},
listVoices: async (): Promise<VoiceInfo[]> => {
listVoices: async (_config, _provider, model): Promise<VoiceInfo[]> => {
// Streaming voices live behind a dedicated endpoint
// (`/audio/voices/streaming`) because they come from a separate
// configKV entry (`STREAMING_TTS_UPSTREAM`) than the HTTP TTS
// `?model=...` lookup. The server proxies to unspeech's
// `/api/voices?provider=volcengine`, which ships an embed-time
// catalogue without requiring credentials.
//
// `model` here is the unspeech-routed id (e.g. `volcengine/seed-tts-2.0`).
// unspeech expects the bare `api_resource_id` for its filter, so we
// strip the backend prefix before forwarding.
const apiResourceId = model?.includes('/') ? model.split('/', 2)[1] : model
const voicesURL = new URL(`${SERVER_URL}/api/v1/audio/voices/streaming`)
if (apiResourceId)
voicesURL.searchParams.set('model', apiResourceId)
const res = await globalThis.fetch(
`${SERVER_URL}/api/v1/audio/voices/streaming`,
voicesURL.toString(),
{ headers: authHeaders() },
)
if (!res.ok)
packages/stage-ui/src/libs/providers/types.ts
+6 -1
View File
@@ -47,7 +47,12 @@ export interface ProviderOnboardingField {
export interface ProviderExtraMethods<TConfig> {
listModels?: (config: TConfig, provider: ProviderInstance) => Promise<ModelInfo[]>
listVoices?: (config: TConfig, provider: ProviderInstance) => Promise<VoiceInfo[]>
/**
* Returns the voice catalogue. `model` lets providers whose voices vary by
* model variant (Volcengine streaming TTS 1.0 vs 2.0 differ in catalogue)
* narrow the result. Providers with a single catalogue ignore it.
*/
listVoices?: (config: TConfig, provider: ProviderInstance, model?: string) => Promise<VoiceInfo[]>
loadModel?: (config: TConfig, provider: ProviderInstance, hooks?: { onProgress?: (progress: ProgressInfo) => Promise<void> | void }) => Promise<void>
}
packages/stage-ui/src/stores/modules/speech.ts
+2 -2
View File
@@ -83,7 +83,7 @@ export const useSpeechStore = defineStore('speech', () => {
return ['elevenlabs', 'microsoft-speech', 'azure-speech'].includes(activeSpeechProvider.value)
})
async function loadVoicesForProvider(provider: string) {
async function loadVoicesForProvider(provider: string, model?: string) {
if (!provider) {
return []
}
@@ -92,7 +92,7 @@ export const useSpeechStore = defineStore('speech', () => {
speechProviderError.value = null
try {
const voices = await providersStore.getProviderMetadata(provider).capabilities.listVoices?.(providersStore.getProviderConfig(provider)) || []
const voices = await providersStore.getProviderMetadata(provider).capabilities.listVoices?.(providersStore.getProviderConfig(provider), model) || []
// Reassign to trigger reactivity when adding/updating provider entries
availableVoices.value = {
...availableVoices.value,
packages/stage-ui/src/stores/providers.ts
+1 -1
View File
@@ -137,7 +137,7 @@ export interface ProviderMetadata {
| Promise<TranscriptionProviderWithExtraOptions>
capabilities: {
listModels?: (config: Record<string, unknown>) => Promise<ModelInfo[]>
listVoices?: (config: Record<string, unknown>) => Promise<VoiceInfo[]>
listVoices?: (config: Record<string, unknown>, model?: string) => Promise<VoiceInfo[]>
loadModel?: (config: Record<string, unknown>, hooks?: { onProgress?: (progress: ProgressInfo) => Promise<void> | void }) => Promise<void>
}
validators: {
packages/stage-ui/src/stores/providers/converters.ts
+2 -2
View File
@@ -161,10 +161,10 @@ export function convertProviderDefinitionToMetadata(
}
},
listVoices: definition.extraMethods?.listVoices
? async (config) => {
? async (config, model) => {
const provider = await definition.createProvider(config as any)
try {
return await definition.extraMethods!.listVoices!(config as any, provider)
return await definition.extraMethods!.listVoices!(config as any, provider, model)
}
finally {
await (provider as { dispose?: () => Promise<void> | void }).dispose?.()