From 6f63ce96e94ae0b4c63e03e5b306e594831c3b92 Mon Sep 17 00:00:00 2001 From: RainbowBird Date: Wed, 20 May 2026 01:42:36 +0800 Subject: [PATCH] feat(server): server-controlled default streaming TTS model MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The streaming speech provider settings page was seeding its model picker with a hardcoded `volcengine/seed-tts-2.0`, which contradicts the just-landed "streaming model catalog comes from the server" contract. Now the default also comes from configKV. - `UNSPEECH_UPSTREAM.streaming` gains `defaultModel?: string`. - `GET /api/v1/audio/models/streaming` response gains `default: string | null` reading that field. - admin slice apply: `streaming.defaultModel` survives key/URL rotation alongside `streaming.models`. - `providerOfficialSpeechStreaming.listModels` stashes the value in a module-scope ref, exposed via `getDefaultStreamingModel()`. - Streaming speech settings page reads the helper instead of the hardcoded id; falls back to the first server-returned model if the operator hasn't curated a default. `handleGenerateSpeech` no longer has a `seed-tts-2.0` backstop — a missing backend prefix in the selected model id now throws instead of silently defaulting. --- .../src/routes/admin/config/router/index.ts | 6 ++ apps/server/src/routes/openai/v1/index.ts | 25 +++-- .../server/src/routes/openai/v1/route.test.ts | 56 +++++++---- .../src/services/adapters/config-kv.test.ts | 34 ++++++- .../server/src/services/adapters/config-kv.ts | 16 ++- .../server/src/services/adapters/tts/azure.ts | 2 +- .../adapters/tts/dashscope-cosyvoice.ts | 2 +- .../src/services/adapters/tts/index.test.ts | 12 +-- .../src/services/adapters/tts/volcengine.ts | 2 +- .../domain/admin/router-config/index.ts | 21 ++-- .../tests/admin-router-config.test.ts | 12 +++ .../src/services/domain/llm-router/router.ts | 77 +++++++++------ .../domain/llm-router/tests/router.test.ts | 43 ++++++++ .../src/pages/settings/modules/speech.vue | 12 ++- .../official-provider-speech-streaming.vue | 37 ++++--- .../stage-ui/src/components/scenes/Stage.vue | 15 ++- .../src/composables/use-auth-provider-sync.ts | 66 ++++++++++++- .../src/libs/providers/providers/index.ts | 5 + .../providers/providers/official/index.ts | 98 +++++++++++++------ .../src/stores/modules/speech.test.ts | 67 ++++++++++++- .../stage-ui/src/stores/modules/speech.ts | 81 +++++++++++++-- packages/stage-ui/src/stores/providers.ts | 22 +++++ 22 files changed, 569 insertions(+), 142 deletions(-) diff --git a/apps/server/src/routes/admin/config/router/index.ts b/apps/server/src/routes/admin/config/router/index.ts index 66ef6926a..ee1d7e4cf 100644 --- a/apps/server/src/routes/admin/config/router/index.ts +++ b/apps/server/src/routes/admin/config/router/index.ts @@ -91,6 +91,12 @@ const UnspeechSliceSchema = object({ ), plaintextKey: pipe(string(), nonEmpty('streaming.plaintextKey is required'), maxLength(MAX_KEY_LENGTH)), keyEntryId: optional(pipe(string(), nonEmpty(), maxLength(200), NO_PIPE)), + models: optional(array(object({ + id: pipe(string(), nonEmpty('streaming.models[].id is required'), maxLength(200)), + name: optional(pipe(string(), nonEmpty(), maxLength(200))), + description: optional(pipe(string(), nonEmpty(), maxLength(500))), + }))), + defaultModel: optional(pipe(string(), nonEmpty('streaming.defaultModel must not be empty'), maxLength(200))), })), }) diff --git a/apps/server/src/routes/openai/v1/index.ts b/apps/server/src/routes/openai/v1/index.ts index 97f94befa..144c1fd6d 100644 --- a/apps/server/src/routes/openai/v1/index.ts +++ b/apps/server/src/routes/openai/v1/index.ts @@ -290,7 +290,6 @@ export function createV1Routes( logger.withError(err).warn('Failed to close stream writer') } - // Extract usage from final SSE data lines let usage: UsageInfo = {} try { const lines = tailBuffer.split('\n').filter(l => l.startsWith('data: ') && !l.includes('[DONE]')) @@ -702,14 +701,9 @@ export function createV1Routes( } async function handleListTTSModels(_c: Context) { - // Surface the concrete TTS models the operator has configured plus the - // `auto` alias. Clients need real model ids to pass `?model=` to - // `/audio/voices`, otherwise the voice catalog endpoint can never resolve - // anything beyond the DEFAULT_TTS_MODEL catalog — which is the bug that - // hid the Azure voices from the UI. - // - // `auto` is kept on top as an explicit "use the operator default" knob - // for clients that don't care which concrete model handles them. + // Surface the concrete TTS models the operator has configured. The UI + // should select an explicit model id so voice catalog requests stay + // model-scoped instead of hiding behind DEFAULT_TTS_MODEL. const config = await configKV.getOrThrow('LLM_ROUTER_CONFIG') // `LLM_ROUTER_CONFIG` is `optional()` at the schema, so its inferred type // tolerates `undefined`. `getOrThrow` already throws on missing entries, @@ -717,29 +711,32 @@ export function createV1Routes( // a TS narrowing aid. const modelIds = Object.keys(config?.tts?.models ?? {}).sort() return Response.json({ - models: [ - { id: 'auto', name: 'Auto' }, - ...modelIds.map(id => ({ id, name: id })), - ], + models: modelIds.map(id => ({ id, name: id })), }) } async function handleListStreamingTTSModels(_c: Context) { const unspeech = await configKV.getOptional('UNSPEECH_UPSTREAM') const models = unspeech?.streaming?.models ?? [] + // `available` is the operator-controlled visibility switch the client gates + // the streaming provider on. It tracks whether `UNSPEECH_UPSTREAM.streaming` + // is configured at all — not whether `models[]` happens to be empty — so an + // operator who has wired the upstream but not yet curated models still + // surfaces the provider rather than silently hiding it. return Response.json({ + available: !!unspeech?.streaming?.baseURL, models: models.map(m => ({ id: m.id, name: m.name ?? m.id, description: m.description, })), + default: unspeech?.streaming?.defaultModel ?? null, }) } const chatGuard = configGuard(configKV, ['FLUX_PER_REQUEST'], 'Service is not available yet') const ttsGuard = configGuard(configKV, ['FLUX_PER_1K_CHARS_TTS'], 'TTS service is not available yet') - // 60 requests per minute per user for LLM completions const completionsRateLimit = rateLimiter({ max: 60, windowSec: 60, metrics: rateLimitMetrics, routeLabel: 'openai.completions' }) // OpenAI-compatible surface (mounted at /api/v1/openai). Only routes that diff --git a/apps/server/src/routes/openai/v1/route.test.ts b/apps/server/src/routes/openai/v1/route.test.ts index d4d52fd18..a34a58cf1 100644 --- a/apps/server/src/routes/openai/v1/route.test.ts +++ b/apps/server/src/routes/openai/v1/route.test.ts @@ -11,8 +11,6 @@ import { afterAll, describe, expect, it, vi } from 'vitest' import { createV1Routes } from '.' import { ApiError } from '../../../utils/error' -// --- Mock helpers --- - function createMockFluxService(flux = 100): FluxService { return { getFlux: vi.fn(async () => ({ userId: 'user-1', flux })), @@ -165,8 +163,6 @@ function createTestApp( const testUser = { id: 'user-1', name: 'Test User', email: 'test@example.com' } -// --- Tests --- - describe('v1CompletionsRoutes', () => { const originalFetch = globalThis.fetch @@ -319,12 +315,10 @@ describe('v1CompletionsRoutes', () => { const data = await res.json() as { id: string } expect(data.id).toBe('chatcmpl-1') - // Verify flux was debited via billingService expect(billingService.consumeFluxForLLM).toHaveBeenCalledWith( expect.objectContaining({ userId: 'user-1', amount: 1 }), ) - // Verify upstream was called with correct URL and resolved model expect(globalThis.fetch).toHaveBeenCalledWith( 'http://mock-gateway/chat/completions', expect.objectContaining({ @@ -406,13 +400,11 @@ describe('v1CompletionsRoutes', () => { ) expect(res.status).toBe(500) - // Post-billing: no charge on failed requests expect(billingService.consumeFluxForLLM).not.toHaveBeenCalled() }) it('should return 503 when config keys are missing', async () => { const configKV = createMockConfigKV() - // Override getOptional to return null for required keys configKV.getOptional = vi.fn(async () => null) const app = createTestApp(createMockFluxService(), configKV) @@ -748,7 +740,7 @@ describe('v1CompletionsRoutes', () => { }) describe('gET /api/v1/audio/models', () => { - it('exposes auto alias plus every configured tts model id', async () => { + it('exposes every configured tts model id', async () => { const app = createTestApp( createMockFluxService(), createMockConfigKV({ @@ -771,14 +763,13 @@ describe('v1CompletionsRoutes', () => { expect(res.status).toBe(200) const data = await res.json() as { models: { id: string, name: string }[] } - expect(data.models[0]).toEqual({ id: 'auto', name: 'Auto' }) - expect(data.models.slice(1).map(m => m.id)).toEqual([ + expect(data.models.map(m => m.id)).toEqual([ 'alibaba/cosyvoice-v2', 'microsoft/v1', ]) }) - it('returns only the auto alias when no tts models are configured', async () => { + it('returns an empty list when no tts models are configured', async () => { const app = createTestApp( createMockFluxService(), createMockConfigKV({ @@ -793,7 +784,7 @@ describe('v1CompletionsRoutes', () => { expect(res.status).toBe(200) const data = await res.json() as { models: { id: string, name: string }[] } - expect(data.models).toEqual([{ id: 'auto', name: 'Auto' }]) + expect(data.models).toEqual([]) }) it('should return 401 when unauthenticated', async () => { @@ -805,7 +796,7 @@ describe('v1CompletionsRoutes', () => { }) describe('gET /api/v1/audio/models/streaming', () => { - it('returns the operator-configured streaming model catalog', async () => { + it('returns the operator-configured streaming model catalog + default', async () => { const app = createTestApp( createMockFluxService(), createMockConfigKV({ @@ -818,6 +809,7 @@ describe('v1CompletionsRoutes', () => { { id: 'volcengine/seed-tts-2.0', name: 'Volcengine Seed-TTS 2.0', description: 'TTS 2.0' }, { id: 'volcengine/seed-tts-1.0' }, ], + defaultModel: 'volcengine/seed-tts-2.0', }, }, }), @@ -829,11 +821,37 @@ describe('v1CompletionsRoutes', () => { ) expect(res.status).toBe(200) - const data = await res.json() as { models: { id: string, name: string, description?: string }[] } + const data = await res.json() as { available: boolean, models: { id: string, name: string, description?: string }[], default: string | null } + expect(data.available).toBe(true) expect(data.models).toEqual([ { id: 'volcengine/seed-tts-2.0', name: 'Volcengine Seed-TTS 2.0', description: 'TTS 2.0' }, { id: 'volcengine/seed-tts-1.0', name: 'volcengine/seed-tts-1.0' }, ]) + expect(data.default).toBe('volcengine/seed-tts-2.0') + }) + + it('returns default: null when operator has not set a streaming default', async () => { + const app = createTestApp( + createMockFluxService(), + createMockConfigKV({ + UNSPEECH_UPSTREAM: { + restBaseURL: 'http://unspeech.local:5933', + streaming: { + baseURL: 'wss://unspeech.local', + keys: [{ id: 'k1', ciphertext: 'enc' }], + models: [{ id: 'volcengine/seed-tts-2.0', name: 'Vol' }], + }, + }, + }), + ) + + const res = await app.fetch( + new Request('http://localhost/api/v1/audio/models/streaming', { method: 'GET' }), + { user: testUser } as any, + ) + + const data = await res.json() as { default: string | null } + expect(data.default).toBeNull() }) it('returns an empty list when UNSPEECH_UPSTREAM is unset', async () => { @@ -845,11 +863,12 @@ describe('v1CompletionsRoutes', () => { ) expect(res.status).toBe(200) - const data = await res.json() as { models: unknown[] } + const data = await res.json() as { available: boolean, models: unknown[] } + expect(data.available).toBe(false) expect(data.models).toEqual([]) }) - it('returns an empty list when streaming subtree has no models', async () => { + it('reports available: true with empty models when streaming subtree has no models', async () => { const app = createTestApp( createMockFluxService(), createMockConfigKV({ @@ -869,7 +888,8 @@ describe('v1CompletionsRoutes', () => { ) expect(res.status).toBe(200) - const data = await res.json() as { models: unknown[] } + const data = await res.json() as { available: boolean, models: unknown[] } + expect(data.available).toBe(true) expect(data.models).toEqual([]) }) diff --git a/apps/server/src/services/adapters/config-kv.test.ts b/apps/server/src/services/adapters/config-kv.test.ts index b73c31fbd..1d2a5e345 100644 --- a/apps/server/src/services/adapters/config-kv.test.ts +++ b/apps/server/src/services/adapters/config-kv.test.ts @@ -21,8 +21,6 @@ describe('configKVService', () => { service = createConfigKVService(redis as any) }) - // --- get --- - it('get should throw 503 when key is not set', async () => { await expect(service.getOrThrow('FLUX_PER_1K_CHARS_TTS')) .rejects @@ -43,8 +41,6 @@ describe('configKVService', () => { expect(redis.get).toHaveBeenCalledWith(configRedisKey('FLUX_PER_REQUEST')) }) - // --- getOptional --- - it('getOptional should return schema default when key has one', async () => { const value = await service.getOptional('FLUX_PER_REQUEST') expect(value).toBe(5) @@ -62,7 +58,35 @@ describe('configKVService', () => { expect(value).toBe(200) }) - // --- set --- + it('getOptional should throw CONFIG_INVALID when Redis contains malformed JSON', async () => { + // ROOT CAUSE: + // + // If an operator edits config:LLM_ROUTER_CONFIG directly with invalid JSON, + // JSON.parse used to throw SyntaxError through the request handler and log + // it as an unhandled 500. + // + // We fixed this by translating stored config parse/validation failures into + // a stable API error at the configKV boundary. + redis._store.set(configRedisKey('LLM_ROUTER_CONFIG'), '{"llm":{}') + + await expect(service.getOptional('LLM_ROUTER_CONFIG')) + .rejects + .toMatchObject({ + statusCode: 503, + errorCode: 'CONFIG_INVALID', + }) + }) + + it('getOptional should throw CONFIG_INVALID when Redis contains schema-invalid JSON', async () => { + redis._store.set(configRedisKey('FLUX_PER_REQUEST'), JSON.stringify('5')) + + await expect(service.getOptional('FLUX_PER_REQUEST')) + .rejects + .toMatchObject({ + statusCode: 503, + errorCode: 'CONFIG_INVALID', + }) + }) it('set should write value to Redis with prefix', async () => { await service.set('FLUX_PER_REQUEST', 10) diff --git a/apps/server/src/services/adapters/config-kv.ts b/apps/server/src/services/adapters/config-kv.ts index 7c992881d..46a26cdda 100644 --- a/apps/server/src/services/adapters/config-kv.ts +++ b/apps/server/src/services/adapters/config-kv.ts @@ -1,6 +1,7 @@ import type Redis from 'ioredis' import type { InferOutput } from 'valibot' +import { errorMessageFrom } from '@moeru/std' import { any, array, boolean, check, nonEmpty, number, object, optional, parse, picklist, pipe, record, regex, string } from 'valibot' import { createServiceUnavailableError } from '../../utils/error' @@ -71,6 +72,7 @@ export const streamingTtsUpstreamSchema = object({ })), [], ), + defaultModel: optional(string()), }) export const unspeechUpstreamSchema = object({ @@ -158,7 +160,19 @@ type ConfigDefinitions = { type ConfigKey = keyof ConfigDefinitions function parseValue(key: K, raw: string): ConfigDefinitions[K] { - return parse(ConfigEntrySchemas[key], JSON.parse(raw)) as ConfigDefinitions[K] + try { + return parse(ConfigEntrySchemas[key], JSON.parse(raw)) as ConfigDefinitions[K] + } + catch (error) { + throw createServiceUnavailableError( + 'Service configuration is invalid', + 'CONFIG_INVALID', + { + key, + message: errorMessageFrom(error) ?? 'Unknown config parse error', + }, + ) + } } function serializeValue(key: K, value: ConfigDefinitions[K]): string { diff --git a/apps/server/src/services/adapters/tts/azure.ts b/apps/server/src/services/adapters/tts/azure.ts index b2bb39f58..2fe16b86c 100644 --- a/apps/server/src/services/adapters/tts/azure.ts +++ b/apps/server/src/services/adapters/tts/azure.ts @@ -228,7 +228,7 @@ export const azureAdapter: TtsAdapter = { if (!ctx.keyPlaintext) throw createServiceUnavailableError('azure tts key not configured', 'AZURE_TTS_NOT_CONFIGURED') - const url = `${ctx.unspeechBaseURL.replace(/\/+$/, '')}/api/voices?backend=microsoft®ion=${encodeURIComponent(ctx.region)}` + const url = `${ctx.unspeechBaseURL.replace(/\/+$/, '')}/api/voices?provider=microsoft®ion=${encodeURIComponent(ctx.region)}` let response: Response try { diff --git a/apps/server/src/services/adapters/tts/dashscope-cosyvoice.ts b/apps/server/src/services/adapters/tts/dashscope-cosyvoice.ts index 6c044bd9d..271e27506 100644 --- a/apps/server/src/services/adapters/tts/dashscope-cosyvoice.ts +++ b/apps/server/src/services/adapters/tts/dashscope-cosyvoice.ts @@ -109,7 +109,7 @@ export const dashscopeCosyvoiceAdapter: TtsAdapter = { // (unspeech/pkg/backend/alibaba/voices.go `//go:embed voices.json`), // so this call is in-memory on unspeech's side and only crosses a TCP // hop. No upstream credential is required. - const url = `${ctx.unspeechBaseURL.replace(/\/+$/, '')}/api/voices?backend=alibaba` + const url = `${ctx.unspeechBaseURL.replace(/\/+$/, '')}/api/voices?provider=alibaba` let response: Response try { diff --git a/apps/server/src/services/adapters/tts/index.test.ts b/apps/server/src/services/adapters/tts/index.test.ts index 9f940dd02..8d327ab7c 100644 --- a/apps/server/src/services/adapters/tts/index.test.ts +++ b/apps/server/src/services/adapters/tts/index.test.ts @@ -61,7 +61,7 @@ describe('getAdapter', () => { }) describe('dashscopeCosyvoiceAdapter.getVoiceCatalog', () => { - it('calls unspeech with backend=alibaba (no Bearer)', async () => { + it('calls unspeech with provider=alibaba (no Bearer)', async () => { const adapter = getAdapter('dashscope-cosyvoice') const fetchImpl = vi.fn(async () => new Response(JSON.stringify({ voices: [{ id: 'longxiaochun_v2', name: 'Longxiaochun v2' }], @@ -75,7 +75,7 @@ describe('dashscopeCosyvoiceAdapter.getVoiceCatalog', () => { expect(voices).toEqual([{ id: 'longxiaochun_v2', name: 'Longxiaochun v2' }]) const [calledUrl, init] = (fetchImpl as unknown as { mock: { calls: [string, RequestInit][] } }).mock.calls[0] - expect(calledUrl).toBe('http://unspeech.local/api/voices?backend=alibaba') + expect(calledUrl).toBe('http://unspeech.local/api/voices?provider=alibaba') const headers = (init.headers ?? {}) as Record expect(headers.Authorization).toBeUndefined() }) @@ -92,7 +92,7 @@ describe('dashscopeCosyvoiceAdapter.getVoiceCatalog', () => { }) describe('volcengineAdapter.getVoiceCatalog', () => { - it('calls unspeech with backend=volcengine and forwards adapterParams.model as ?model=', async () => { + it('calls unspeech with provider=volcengine and forwards adapterParams.model as ?model=', async () => { const adapter = getAdapter('volcengine') const fetchImpl = vi.fn(async () => new Response(JSON.stringify({ voices: [{ id: 'zh_female_x', name: 'X' }], @@ -106,7 +106,7 @@ describe('volcengineAdapter.getVoiceCatalog', () => { expect(voices).toEqual([{ id: 'zh_female_x', name: 'X' }]) const [calledUrl] = (fetchImpl as unknown as { mock: { calls: [string, RequestInit][] } }).mock.calls[0] - expect(calledUrl).toBe('http://unspeech.local/api/voices?backend=volcengine&model=seed-tts-2.0') + expect(calledUrl).toBe('http://unspeech.local/api/voices?provider=volcengine&model=seed-tts-2.0') }) it('omits ?model= when adapterParams.model is not set', async () => { @@ -118,7 +118,7 @@ describe('volcengineAdapter.getVoiceCatalog', () => { fetchImpl, }) const [calledUrl] = (fetchImpl as unknown as { mock: { calls: [string, RequestInit][] } }).mock.calls[0] - expect(calledUrl).toBe('http://unspeech.local/api/voices?backend=volcengine') + expect(calledUrl).toBe('http://unspeech.local/api/voices?provider=volcengine') }) }) @@ -140,7 +140,7 @@ describe('azureAdapter.getVoiceCatalog', () => { expect(voices).toEqual([{ id: 'en-US-AvaMultilingualNeural', name: 'Ava' }]) expect(fetchImpl).toHaveBeenCalledTimes(1) const [calledUrl, init] = (fetchImpl as unknown as { mock: { calls: [string, RequestInit][] } }).mock.calls[0] - expect(calledUrl).toBe('http://unspeech.local:5933/api/voices?backend=microsoft®ion=eastasia') + expect(calledUrl).toBe('http://unspeech.local:5933/api/voices?provider=microsoft®ion=eastasia') const headers = init.headers as Record expect(headers.Authorization).toBe('Bearer subscription-key-XYZ') }) diff --git a/apps/server/src/services/adapters/tts/volcengine.ts b/apps/server/src/services/adapters/tts/volcengine.ts index 6d7170ef5..e2083568f 100644 --- a/apps/server/src/services/adapters/tts/volcengine.ts +++ b/apps/server/src/services/adapters/tts/volcengine.ts @@ -120,7 +120,7 @@ export const volcengineAdapter: TtsAdapter = { // further by `compatible_models` — adapterParams.model is the operator- // configured resource id (e.g. `seed-tts-2.0`). const url = new URL(`${ctx.unspeechBaseURL.replace(/\/+$/, '')}/api/voices`) - url.searchParams.set('backend', 'volcengine') + url.searchParams.set('provider', 'volcengine') const apiResourceId = typeof ctx.adapterParams?.model === 'string' ? ctx.adapterParams.model : undefined diff --git a/apps/server/src/services/domain/admin/router-config/index.ts b/apps/server/src/services/domain/admin/router-config/index.ts index ea78e4785..ec5e10c8b 100644 --- a/apps/server/src/services/domain/admin/router-config/index.ts +++ b/apps/server/src/services/domain/admin/router-config/index.ts @@ -96,6 +96,10 @@ export interface UnspeechSliceInput { plaintextKey: string /** @default 'volcengine-prod-1' */ keyEntryId?: string + /** Operator-curated streaming models exposed to the frontend picker. */ + models?: Array<{ id: string, name?: string, description?: string }> + /** Server-curated default streaming model id. */ + defaultModel?: string } } @@ -262,7 +266,8 @@ export function buildUnspeechSlice(input: UnspeechSliceInput, envelope: Envelope baseURL: input.streaming.upstreamURL, keys: [{ id: keyEntryId, ciphertext }], adapterParams: {}, - models: [], + models: input.streaming.models ?? [], + defaultModel: input.streaming.defaultModel, }, }, } @@ -451,18 +456,22 @@ export function createAdminRouterConfigService(deps: AdminRouterConfigDeps) { nextRouterConfig = buildNextRouterConfig(input.mode, existing, llmTtsSlices) } - // Step 3: build the next UNSPEECH_UPSTREAM. Streaming `models` carry the - // operator-curated catalog and must survive key/URL rotation, so we read - // existing and graft them onto the new value when the slice's streaming + // Step 3: build the next UNSPEECH_UPSTREAM. Streaming `models` + + // `defaultModel` are operator-curated and must survive key/URL rotation, + // so we graft them from the existing entry when the slice's streaming // subtree is set (otherwise there's nothing to merge into). let nextUnspeech: UnspeechUpstream | undefined if (unspeechSlice) { const existing = await deps.configKV.getOptional('UNSPEECH_UPSTREAM') const newValue = unspeechSlice.value - if (newValue.streaming && existing?.streaming?.models?.length) { + if (newValue.streaming && existing?.streaming) { nextUnspeech = { ...newValue, - streaming: { ...newValue.streaming, models: existing.streaming.models }, + streaming: { + ...newValue.streaming, + models: existing.streaming.models?.length ? existing.streaming.models : newValue.streaming.models, + defaultModel: existing.streaming.defaultModel ?? newValue.streaming.defaultModel, + }, } } else { diff --git a/apps/server/src/services/domain/admin/router-config/tests/admin-router-config.test.ts b/apps/server/src/services/domain/admin/router-config/tests/admin-router-config.test.ts index 45234973d..d8d18c8a9 100644 --- a/apps/server/src/services/domain/admin/router-config/tests/admin-router-config.test.ts +++ b/apps/server/src/services/domain/admin/router-config/tests/admin-router-config.test.ts @@ -356,11 +356,23 @@ describe('createAdminRouterConfigService', () => { streaming: { upstreamURL: 'wss://unspeech.example/v1/audio/speech/stream', plaintextKey: 'volc', + models: [ + { id: 'volcengine/seed-tts-2.0', name: 'Seed-TTS 2.0', description: 'Low-latency streaming TTS' }, + ], + defaultModel: 'volcengine/seed-tts-2.0', }, }], }) expect(kv.store.has('UNSPEECH_UPSTREAM')).toBe(true) + expect(kv.store.get('UNSPEECH_UPSTREAM')).toMatchObject({ + streaming: { + models: [ + { id: 'volcengine/seed-tts-2.0', name: 'Seed-TTS 2.0', description: 'Low-latency streaming TTS' }, + ], + defaultModel: 'volcengine/seed-tts-2.0', + }, + }) expect(kv.store.has('LLM_ROUTER_CONFIG')).toBe(false) expect(result.invalidatedKeys).toEqual(['UNSPEECH_UPSTREAM']) expect(captured.map(p => JSON.parse(p.payload).key)).toEqual(['UNSPEECH_UPSTREAM']) diff --git a/apps/server/src/services/domain/llm-router/router.ts b/apps/server/src/services/domain/llm-router/router.ts index 72fd166e7..321a726ef 100644 --- a/apps/server/src/services/domain/llm-router/router.ts +++ b/apps/server/src/services/domain/llm-router/router.ts @@ -1,6 +1,7 @@ import type { Buffer } from 'node:buffer' import type Redis from 'ioredis' +import type { Voice } from 'unspeech' import type { GatewayMetrics } from '../../../otel' import type { EnvelopeCrypto } from '../../../utils/envelope-crypto' @@ -176,6 +177,7 @@ export function createLlmRouterService(options: CreateLlmRouterServiceOptions) { const logger = useLogger('llm-router').useGlobalConfig() const fetchImpl = options.fetchImpl ?? globalThis.fetch const configLoader = createConfigLoader({ configKV: options.configKV, ttlMs: options.configCacheTtlMs }) + const ttsVoiceCatalogLoads = new Map>() /** * Run one upstream's key list in order, returning either: @@ -652,44 +654,55 @@ export function createLlmRouterService(options: CreateLlmRouterServiceOptions) { } } - const unspeechBaseURL = (await options.configKV.getOrThrow('UNSPEECH_UPSTREAM')).restBaseURL + const existingLoad = ttsVoiceCatalogLoads.get(cacheKey) + if (existingLoad != null) + return existingLoad - // Live providers (Azure) need the decrypted Azure subscription key + region; - // static-catalog providers (alibaba, volcengine) ignore both. The router - // decrypts unconditionally so the adapter doesn't have to know which - // category it's in — adapters that don't need creds just won't read them. - const region = typeof upstream.adapterParams?.region === 'string' - ? upstream.adapterParams.region - : undefined + const load = (async () => { + const unspeechBaseURL = (await options.configKV.getOrThrow('UNSPEECH_UPSTREAM')).restBaseURL - const keyEntry = upstream.keys[0] - const plaintext = slice.model.provider === 'azure' - ? options.envelopeCrypto.decryptKey(keyEntry.ciphertext, { modelName, keyEntryId: keyEntry.id }) - : undefined + // Live providers (Azure) need the decrypted Azure subscription key + region; + // static-catalog providers (alibaba, volcengine) ignore both. The router + // decrypts unconditionally so the adapter doesn't have to know which + // category it's in — adapters that don't need creds just won't read them. + const region = typeof upstream.adapterParams?.region === 'string' + ? upstream.adapterParams.region + : undefined - try { - const voices = await adapter.getVoiceCatalog({ - keyPlaintext: plaintext, - region, - adapterParams: upstream.adapterParams ?? {}, - unspeechBaseURL, - fetchImpl, - }) + const keyEntry = upstream.keys[0] + const plaintext = slice.model.provider === 'azure' + ? options.envelopeCrypto.decryptKey(keyEntry.ciphertext, { modelName, keyEntryId: keyEntry.id }) + : undefined - // Cache only on success — failure responses must NOT be persisted or - // the next admin reconfigure would have to wait out the TTL even after - // fixing credentials. - const ttl = options.ttsVoiceCacheTtlSeconds ?? ttsVoicesCacheTtl(slice.model.provider) - await options.redis.set(cacheKey, JSON.stringify(voices), 'EX', ttl) - .catch((err) => { - logger.withError(err).withFields({ cacheKey }).warn('failed to write tts voices cache') + try { + const voices = await adapter.getVoiceCatalog({ + keyPlaintext: plaintext, + region, + adapterParams: upstream.adapterParams ?? {}, + unspeechBaseURL, + fetchImpl, }) - return voices - } - finally { - plaintext?.fill(0) - } + // Cache only on success — failure responses must NOT be persisted or + // the next admin reconfigure would have to wait out the TTL even after + // fixing credentials. + const ttl = options.ttsVoiceCacheTtlSeconds ?? ttsVoicesCacheTtl(slice.model.provider) + await options.redis.set(cacheKey, JSON.stringify(voices), 'EX', ttl) + .catch((err) => { + logger.withError(err).withFields({ cacheKey }).warn('failed to write tts voices cache') + }) + + return voices + } + finally { + plaintext?.fill(0) + } + })().finally(() => { + ttsVoiceCatalogLoads.delete(cacheKey) + }) + + ttsVoiceCatalogLoads.set(cacheKey, load) + return load } /** diff --git a/apps/server/src/services/domain/llm-router/tests/router.test.ts b/apps/server/src/services/domain/llm-router/tests/router.test.ts index dcc75e3b2..3cfda3ca6 100644 --- a/apps/server/src/services/domain/llm-router/tests/router.test.ts +++ b/apps/server/src/services/domain/llm-router/tests/router.test.ts @@ -754,5 +754,48 @@ describe('createLlmRouterService', () => { // adapter's `Error & { status }` was read as undefined. expect(fallbackCalls[0][1]).toMatchObject({ reason: '401' }) }) + + it('listTtsVoices deduplicates concurrent cold-cache upstream fetches per model', async () => { + // ROOT CAUSE: + // + // Azure voice catalogs are cached after a successful fetch, but concurrent + // cold-cache requests used to miss Redis together and each hit unspeech's + // microsoft voices endpoint. That can amplify one settings-page open into + // several Azure voices/list calls and trigger upstream 429. + // + // We fixed this by sharing the in-flight catalog load for the same + // provider/model cache key. Failures are still returned to every caller and + // are not cached. + const { config, crypto } = makeTtsConfig({ + upstreams: [{ baseURL: 'https://az.example', keyIds: ['kA1'], adapterParams: { region: 'eastasia' } }], + }) + + let resolveFetch!: () => void + const fetchImpl = vi.fn(() => new Promise((resolve) => { + resolveFetch = () => resolve(happyResponse({ + voices: [{ id: 'en-US-AvaMultilingualNeural', name: 'Ava' }], + })) + })) + + const router = createLlmRouterService({ + configKV: makeConfigKV(config), + envelopeCrypto: crypto, + gatewayMetrics: null, + fetchImpl, + redis: makeRedisStub(), + }) + + const first = router.listTtsVoices('tts-test') + const second = router.listTtsVoices('tts-test') + + await vi.waitFor(() => { + expect(fetchImpl).toHaveBeenCalledTimes(1) + }) + resolveFetch() + + const [firstVoices, secondVoices] = await Promise.all([first, second]) + expect(firstVoices.map(voice => voice.id)).toEqual(['en-US-AvaMultilingualNeural']) + expect(secondVoices.map(voice => voice.id)).toEqual(['en-US-AvaMultilingualNeural']) + }) }) }) diff --git a/packages/stage-pages/src/pages/settings/modules/speech.vue b/packages/stage-pages/src/pages/settings/modules/speech.vue index e29fc3a67..88883f73a 100644 --- a/packages/stage-pages/src/pages/settings/modules/speech.vue +++ b/packages/stage-pages/src/pages/settings/modules/speech.vue @@ -87,13 +87,13 @@ function syncOpenAICompatibleSettings() { onMounted(async () => { await providersStore.loadModelsForConfiguredProviders() - await speechStore.loadVoicesForProvider(activeSpeechProvider.value) + speechStore.ensureActiveSpeechModel() + await speechStore.loadVoicesForProvider(activeSpeechProvider.value, activeSpeechModel.value || undefined) syncOpenAICompatibleSettings() }) watch(activeSpeechProvider, async (newProvider, oldProvider) => { await providersStore.loadModelsForConfiguredProviders() - await speechStore.loadVoicesForProvider(newProvider) // Reset model and voice when switching providers (but not on initial load) if (oldProvider !== undefined && oldProvider !== newProvider) { @@ -102,12 +102,18 @@ watch(activeSpeechProvider, async (newProvider, oldProvider) => { activeSpeechVoice.value = undefined } + // Re-seed the streaming default model after the reset above so its voices + // load model-scoped (the server only returns recommended voices for an + // explicit ?model=). No-op for other providers / when a model is selected. + speechStore.ensureActiveSpeechModel() + await speechStore.loadVoicesForProvider(newProvider, activeSpeechModel.value || undefined) + syncOpenAICompatibleSettings() }) watch(activeSpeechModel, async () => { if (activeSpeechProvider.value) { - await speechStore.loadVoicesForProvider(activeSpeechProvider.value) + await speechStore.loadVoicesForProvider(activeSpeechProvider.value, activeSpeechModel.value || undefined) } }) diff --git a/packages/stage-pages/src/pages/settings/providers/speech/official-provider-speech-streaming.vue b/packages/stage-pages/src/pages/settings/providers/speech/official-provider-speech-streaming.vue index cc2a8c283..97b192d63 100644 --- a/packages/stage-pages/src/pages/settings/providers/speech/official-provider-speech-streaming.vue +++ b/packages/stage-pages/src/pages/settings/providers/speech/official-provider-speech-streaming.vue @@ -4,7 +4,7 @@ import { ProviderSettingsLayout, SpeechPlayground, } from '@proj-airi/stage-ui/components' -import { streamingSynthesize } from '@proj-airi/stage-ui/libs' +import { getDefaultStreamingModel, streamingSynthesize } from '@proj-airi/stage-ui/libs' import { useAuthStore } from '@proj-airi/stage-ui/stores/auth' import { useSpeechStore } from '@proj-airi/stage-ui/stores/modules/speech' import { useProvidersStore } from '@proj-airi/stage-ui/stores/providers' @@ -23,18 +23,19 @@ const { isAuthenticated, credits, needsLogin } = storeToRefs(authStore) const providerId = 'official-provider-speech-streaming' const providerMetadata = computed(() => providersStore.getProviderMetadata(providerId)) -const defaultModel = 'volcengine/seed-tts-2.0' const providerConfig = computed(() => providersStore.getProviderConfig(providerId)) -// Model picker. Pulled from the provider's `extraMethods.listModels` — -// today that's two hard-coded Volcengine variants, but the picker stays -// generic so adding ICL / other backends later doesn't need UI changes. +// Model picker. The catalog and the default model id both come from the +// server's `/api/v1/audio/models/streaming` response (operator-controlled +// via `UNSPEECH_UPSTREAM.streaming`); no client-side hardcoded defaults so +// adding ICL / other backends doesn't need a UI release. const providerModels = computed(() => providersStore.getModelsForProvider(providerId)) const modelsLoading = computed(() => providersStore.isLoadingModels[providerId] || false) +const serverDefaultModel = ref(null) const model = computed({ get(): string { - return (providerConfig.value?.model as string | undefined) ?? defaultModel + return (providerConfig.value?.model as string | undefined) ?? serverDefaultModel.value ?? '' }, set(val: string) { providerConfig.value.model = val @@ -57,8 +58,13 @@ async function loadVoices() { onMounted(async () => { await providersStore.fetchModelsForProvider(providerId) - if (!providerConfig.value.model) - providerConfig.value.model = defaultModel + // `getDefaultStreamingModel()` is populated by the provider's listModels() + // (just ran via fetchModelsForProvider). If the operator hasn't curated a + // default server-side, fall back to the first model the server returned + // so the picker always has something selected. + serverDefaultModel.value = getDefaultStreamingModel() ?? providerModels.value[0]?.id ?? null + if (!providerConfig.value.model && serverDefaultModel.value) + providerConfig.value.model = serverDefaultModel.value await loadVoices() }) @@ -76,11 +82,18 @@ watch(model, async () => { // per-preview because there's no LLM token stream here — we just send // one `text` frame containing the static preview prompt. async function handleGenerateSpeech(input: string, voiceId: string, _useSSML: boolean): Promise { - const requestedModel = model.value || defaultModel + const requestedModel = model.value + if (!requestedModel) + throw new Error('No streaming TTS model selected and server returned no default') // `model` looks like `volcengine/seed-tts-2.0`. The trailing path is - // forwarded as Volcengine's `api_resource_id` so the upstream knows - // which model variant to use; matches the wiring in `Stage.vue`. - const apiResourceId = requestedModel.includes('/') ? requestedModel.split('/', 2)[1] : 'seed-tts-2.0' + // forwarded as Volcengine's `api_resource_id` so the upstream knows which + // model variant to use; matches the wiring in `Stage.vue`. We require the + // `/` shape and refuse anything else — silently picking + // a fallback resource id hides config drift. + const slashIndex = requestedModel.indexOf('/') + if (slashIndex < 0) + throw new Error(`Streaming model id missing backend prefix: ${requestedModel}`) + const apiResourceId = requestedModel.slice(slashIndex + 1) const result = await streamingSynthesize({ model: requestedModel, voice: voiceId, diff --git a/packages/stage-ui/src/components/scenes/Stage.vue b/packages/stage-ui/src/components/scenes/Stage.vue index 001d014bf..a7f68f39b 100644 --- a/packages/stage-ui/src/components/scenes/Stage.vue +++ b/packages/stage-ui/src/components/scenes/Stage.vue @@ -32,7 +32,7 @@ import { useIOTraceBridge } from '../../composables/use-io-trace-bridge' import { initIOTracer } from '../../composables/use-io-tracer' import { useSpeechPipelineAnalytics } from '../../composables/use-speech-pipeline-analytics' import { Emotion, EMOTION_EmotionMotionName_value, EMOTION_VRMExpressionName_value, EmotionThinkMotionName } from '../../constants/emotions' -import { getDefinedProvider } from '../../libs/providers/providers' +import { getDefaultStreamingModel, getDefinedProvider } from '../../libs/providers/providers' import { createStageTtsSession } from '../../libs/speech/tts-session' import { useAudioContext, useSpeakingStore } from '../../stores/audio' import { useBackgroundStore } from '../../stores/background' @@ -575,8 +575,17 @@ function buildStreamingSnapshot(): StreamingSessionSnapshot | null { const voiceId = activeSpeechVoice.value?.id if (!voiceId) return null - const sessionModel = (activeSpeechModel.value as string | undefined) || 'volcengine/seed-tts-2.0' - const apiResourceId = sessionModel.includes('/') ? sessionModel.split('/', 2)[1] : 'seed-tts-2.0' + // Resolve the concrete streaming model id. The active speech model is only + // valid here when it carries the `/` shape the ws + // upstream expects — the HTTP TTS `auto` alias (and an empty selection after + // a provider switch) must NOT reach the bridge, so fall back to the + // server-curated default instead of a hardcoded id. Returns null (segmenter + // fallback) when neither resolves, rather than guessing a resource id. + const activeModel = activeSpeechModel.value as string | undefined + const sessionModel = activeModel?.includes('/') ? activeModel : getDefaultStreamingModel() + if (!sessionModel?.includes('/')) + return null + const apiResourceId = sessionModel.split('/', 2)[1] // TTS 2.0 / ICL 2.0 ship subtitles asynchronously relative to audio // (per the wire spec), so chunk-on-sentence-end would drop frames. // Buffer the entire session and decode at session.finished instead. diff --git a/packages/stage-ui/src/composables/use-auth-provider-sync.ts b/packages/stage-ui/src/composables/use-auth-provider-sync.ts index 698304c21..97ed6b620 100644 --- a/packages/stage-ui/src/composables/use-auth-provider-sync.ts +++ b/packages/stage-ui/src/composables/use-auth-provider-sync.ts @@ -1,6 +1,7 @@ import { nextTick } from 'vue' import { initializeAuth } from '../libs/auth' +import { getStreamingTtsAvailable } from '../libs/providers' import { useAuthStore } from '../stores/auth' import { useConsciousnessStore } from '../stores/modules/consciousness' import { useHearingStore } from '../stores/modules/hearing' @@ -16,6 +17,14 @@ const AUTH_ACTIVATED_PROVIDERS: Array<{ id: string, module: 'consciousness' | 's { id: 'official-provider-speech', module: 'speech' }, ] +// The streaming TTS provider is NOT in the static list above because its +// visibility is operator-controlled: `UNSPEECH_UPSTREAM.streaming` may be +// unconfigured server-side. It's bootstrapped separately (see +// `syncStreamingSpeechProvider`) — probed on sign-in, then force-configured +// only when the server reports it available, mirroring how the HTTP TTS +// provider uses `forceProviderConfigured` but gating it on a server signal. +const STREAMING_SPEECH_PROVIDER_ID = 'official-provider-speech-streaming' + /** * Glue layer: uses auth lifecycle hooks to activate/deactivate * official providers. Providers themselves know nothing about auth. @@ -60,7 +69,7 @@ export function useAuthProviderSync() { case 'speech': if (!speechStore.activeSpeechProvider || speechStore.activeSpeechProvider === 'speech-noop') { speechStore.activeSpeechProvider = id - speechStore.activeSpeechModel = 'auto' + speechStore.activeSpeechModel = '' } break case 'hearing': @@ -85,8 +94,54 @@ export function useAuthProviderSync() { catch (err) { console.error('error loading models for official providers', err) } + + await syncStreamingSpeechProvider() }) + // Bootstrap the streaming TTS provider from the server's availability signal. + // Probing populates `getStreamingTtsAvailable()` (and the default model / + // voices) via the provider's listModels(). The availability override drives + // the provider's presence in the available/configured lists (and thus the + // settings card + picker); force-configure makes it selectable. It is never + // set as the active speech provider — the HTTP TTS provider stays default. + async function syncStreamingSpeechProvider() { + if (providersStore.getProviderMetadata(STREAMING_SPEECH_PROVIDER_ID) == null) + return + + await providersStore.fetchModelsForProvider(STREAMING_SPEECH_PROVIDER_ID) + + const available = getStreamingTtsAvailable() + providersStore.setProviderAvailabilityOverride(STREAMING_SPEECH_PROVIDER_ID, available) + + if (available) { + providersStore.forceProviderConfigured(STREAMING_SPEECH_PROVIDER_ID) + // The speech-module watcher skips voice loading for streaming until it's + // confirmed configured (avoids a pre-probe request on reload), so when + // streaming is the persisted active provider, load its voices now that + // it's confirmed available. + if (speechStore.activeSpeechProvider === STREAMING_SPEECH_PROVIDER_ID) { + speechStore.ensureStreamingDefaultModel() + await speechStore.loadVoicesForProvider(STREAMING_SPEECH_PROVIDER_ID, speechStore.activeSpeechModel || undefined) + } + return + } + + providersStore.setProviderUnconfigured(STREAMING_SPEECH_PROVIDER_ID) + // `setProviderUnconfigured` blanks `validatedCredentialHash`, which makes + // the speech-module reset watcher skip its own clear. So when the server + // now reports streaming unavailable on an authenticated reload (no logout + // event fires), clear a stale active streaming selection here. + clearActiveStreamingSelection() + } + + function clearActiveStreamingSelection() { + if (speechStore.activeSpeechProvider !== STREAMING_SPEECH_PROVIDER_ID) + return + speechStore.activeSpeechProvider = '' + speechStore.activeSpeechModel = '' + speechStore.activeSpeechVoiceId = '' + } + authStore.onLogout(() => { hasSynced = false @@ -94,6 +149,15 @@ export function useAuthProviderSync() { providersStore.setProviderUnconfigured(id) } + // Streaming TTS is bootstrapped outside AUTH_ACTIVATED_PROVIDERS, so reset + // it explicitly. `setProviderUnconfigured` blanks `validatedCredentialHash`, + // which makes the speech-module watcher skip its own reset (it guards + // against racing initial validation), so clear the active selection here + // too when streaming was the active provider. + clearActiveStreamingSelection() + providersStore.setProviderUnconfigured(STREAMING_SPEECH_PROVIDER_ID) + providersStore.setProviderAvailabilityOverride(STREAMING_SPEECH_PROVIDER_ID, false) + // Reset active provider/model if they belong to an auth-activated provider for (const { id, module } of AUTH_ACTIVATED_PROVIDERS) { switch (module) { diff --git a/packages/stage-ui/src/libs/providers/providers/index.ts b/packages/stage-ui/src/libs/providers/providers/index.ts index cab4497a2..e78daf50a 100644 --- a/packages/stage-ui/src/libs/providers/providers/index.ts +++ b/packages/stage-ui/src/libs/providers/providers/index.ts @@ -34,6 +34,11 @@ import './cloudflare-workers-ai' import './azure-ai-foundry' import './official' +export { + getDefaultStreamingModel, + getStreamingTtsAvailable, +} from './official' + export { getDefinedProvider, listProviders, diff --git a/packages/stage-ui/src/libs/providers/providers/official/index.ts b/packages/stage-ui/src/libs/providers/providers/official/index.ts index 961eb42f7..5eb308b8b 100644 --- a/packages/stage-ui/src/libs/providers/providers/official/index.ts +++ b/packages/stage-ui/src/libs/providers/providers/official/index.ts @@ -2,7 +2,7 @@ import type { Ref, WatchSource } from 'vue' import type { ModelInfo, VoiceInfo } from '../../../../stores/providers' -import { watch } from 'vue' +import { ref, watch } from 'vue' import { z } from 'zod' import { getAuthToken } from '../../../../libs/auth' @@ -13,11 +13,34 @@ import { createOfficialAudioProvider, createOfficialOpenAIProvider, OFFICIAL_ICO export const OFFICIAL_SPEECH_PROVIDER_ID = 'official-provider-speech' export const OFFICIAL_SPEECH_STREAMING_PROVIDER_ID = 'official-provider-speech-streaming' -// Locale → voice id map recommended by the server. Populated by listVoices() -// from the /audio/voices response's `recommended` field so the auto-pick can -// prefer a curated default per locale. Falls back to language + first-voice -// matching when the server returns no recommendations. -let recommendedVoicesByLocale: Record = {} +// Locale → voice id map recommended by the server, keyed by provider id. +// Populated by each speech provider's listVoices() from the response's +// `recommended` field so the auto-pick can prefer a curated default per +// locale. Keyed per provider because the HTTP and streaming providers have +// independent catalogs and recommendation buckets. Falls back to language + +// first-voice matching when the server returns no recommendations. +const recommendedVoicesByProvider: Record> = {} + +// Server-curated default streaming model id, populated by the streaming +// provider's listModels(). Pages that need to seed an initial model selection +// read this via getDefaultStreamingModel() instead of hardcoding an id. +let defaultStreamingModelId: string | null = null + +export function getDefaultStreamingModel(): string | null { + return defaultStreamingModelId +} + +// Operator-controlled visibility switch for the streaming provider. The server +// reports it via `/api/v1/audio/models/streaming` (`available`), and the +// auth-activation glue gates `forceProviderConfigured` on this so the provider +// only surfaces when `UNSPEECH_UPSTREAM.streaming` is configured server-side. +// Reactive so the providers store re-derives configured speech providers when +// the probe resolves after sign-in. +const streamingTtsAvailable = ref(false) + +export function getStreamingTtsAvailable(): boolean { + return streamingTtsAvailable.value +} const officialConfigSchema = z.object({}) @@ -107,9 +130,8 @@ export const providerOfficialSpeech = defineProvider({ listVoices: async (_config, _provider, model): Promise => { // Voice catalogs are model-scoped on the server side. Pass the active // model through so Azure / cosyvoice / future provider voices route to - // the right adapter. `auto` defers to the server's DEFAULT_TTS_MODEL, - // but it MUST be sent explicitly — an absent `model` is treated as a - // client bug and returns 400. + // the right adapter. If model discovery has not completed yet, keep the + // legacy `auto` request as a startup fallback. const target = model && model.length > 0 ? model : 'auto' const url = new URL(`${SERVER_URL}/api/v1/audio/voices`) url.searchParams.set('model', target) @@ -137,7 +159,7 @@ export const providerOfficialSpeech = defineProvider({ // Refresh the server-side recommendation map. Done here rather than // threading it through the return value because the auto-pick watcher // lives in this module and reads the same singleton. - recommendedVoicesByLocale = (data.recommended && typeof data.recommended === 'object') ? data.recommended : {} + recommendedVoicesByProvider[OFFICIAL_SPEECH_PROVIDER_ID] = (data.recommended && typeof data.recommended === 'object') ? data.recommended : {} if (!Array.isArray(data.voices)) throw new Error('audio voices upstream returned malformed body') @@ -155,10 +177,9 @@ export const providerOfficialSpeech = defineProvider({ // NOTICE: deliberately dropping `compatible_models`. The official // provider resolves voices through the server's /audio/voices?model= // endpoint, which already returns only voices valid for the active - // model (or the DEFAULT_TTS_MODEL when the client's selection is the - // 'auto' alias). Re-applying the client-side filter on top would - // zero out the list because upstream compatibility ids never match - // 'auto'. See packages/stage-pages/.../speech.vue filter predicate. + // model. Re-applying the client-side filter on top can zero out the + // list when upstream compatibility ids differ from AIRI's router ids. + // See packages/stage-pages/.../speech.vue filter predicate. languages: Array.isArray(v.languages) ? v.languages : [], } }) @@ -214,17 +235,26 @@ export const providerOfficialSpeechStreaming = defineProvider({ extraMethods: { listModels: async (): Promise => { // Streaming TTS catalog is operator-controlled via configKV - // (`STREAMING_TTS_MODELS`). The wire `model` field uses the - // `/` shape unspeech expects (see - // `unspeech/docs/wire-protocols/audio-speech-stream-v1.md`); the server - // returns whatever the operator put there, no client-side defaults. + // (`UNSPEECH_UPSTREAM.streaming`). Wire shape uses `/` + // (see `unspeech/docs/wire-protocols/audio-speech-stream-v1.md`); the + // server returns whatever the operator put there, no client-side + // defaults. `default` (when set) seeds initial model selection via + // {@link getDefaultStreamingModel}. + // Reset the operator-driven signals up front so a failed/aborted probe + // leaves the provider hidden rather than stuck on a stale "available". + streamingTtsAvailable.value = false + defaultStreamingModelId = null + const res = await globalThis.fetch(`${SERVER_URL}/api/v1/audio/models/streaming`, { headers: authHeaders() }) if (!res.ok) throw new Error(`streaming models upstream ${res.status}: ${await res.text().catch(() => '')}`.slice(0, 256)) - const data = await res.json() as { models?: { id: string, name?: string, description?: string }[] } + const data = await res.json() as { available?: boolean, models: { id: string, name?: string, description?: string }[], default?: string | null } if (!Array.isArray(data.models)) - throw new Error('streaming models upstream returned malformed body') + throw new Error('streaming models upstream missing models[]') + + streamingTtsAvailable.value = data.available === true + defaultStreamingModelId = typeof data.default === 'string' && data.default.length > 0 ? data.default : null return data.models.map(m => ({ id: m.id, @@ -235,8 +265,8 @@ export const providerOfficialSpeechStreaming = defineProvider({ }, listVoices: async (_config, _provider, model): Promise => { // Streaming voices live behind a dedicated endpoint - // (`/audio/voices/streaming`) because they come from a separate - // configKV entry (`STREAMING_TTS_UPSTREAM`) than the HTTP TTS + // (`/audio/voices/streaming`) because they come from the + // `UNSPEECH_UPSTREAM.streaming` configKV subtree rather than the HTTP TTS // `?model=...` lookup. The server proxies to unspeech's // `/api/voices?provider=volcengine`, which ships an embed-time // catalogue without requiring credentials. @@ -264,7 +294,14 @@ export const providerOfficialSpeechStreaming = defineProvider({ languages?: { code: string, title: string }[] preview_audio_url?: string }[] + recommended?: Record } + + // Mirror the HTTP provider: stash the server's per-locale recommendations + // so setupOfficialSpeechAutoPick can seed a curated default voice when + // the streaming provider becomes active. + recommendedVoicesByProvider[OFFICIAL_SPEECH_STREAMING_PROVIDER_ID] = (data.recommended && typeof data.recommended === 'object') ? data.recommended : {} + if (!Array.isArray(data.voices)) throw new Error('streaming voices upstream returned malformed body') @@ -318,10 +355,13 @@ function lookupRecommendedVoiceId(locale: string, map: Record): return undefined } -// NOTICE: Only the official speech provider auto-configures a default voice -// after login. Third-party providers leave voice selection to the user. The -// target locale is derived from the UI locale on each run — we don't persist -// it, since that was the root of the cross-provider filter drift bug. +const AUTO_PICK_PROVIDER_IDS = new Set([OFFICIAL_SPEECH_PROVIDER_ID, OFFICIAL_SPEECH_STREAMING_PROVIDER_ID]) + +// NOTICE: Only the official speech providers (HTTP + streaming) auto-configure +// a default voice after login. Third-party providers leave voice selection to +// the user. The target locale is derived from the UI locale on each run — we +// don't persist it, since that was the root of the cross-provider filter +// drift bug. export function setupOfficialSpeechAutoPick(ctx: { activeSpeechProvider: Ref activeSpeechVoiceId: Ref @@ -329,12 +369,12 @@ export function setupOfficialSpeechAutoPick(ctx: { uiLocale: WatchSource | Ref }) { watch([ctx.availableVoices, ctx.activeSpeechProvider], ([voices, provider]) => { - if (provider !== OFFICIAL_SPEECH_PROVIDER_ID) + if (!AUTO_PICK_PROVIDER_IDS.has(provider)) return if (ctx.activeSpeechVoiceId.value) return - const providerVoices = voices[OFFICIAL_SPEECH_PROVIDER_ID] + const providerVoices = voices[provider] if (!providerVoices?.length) return @@ -356,7 +396,7 @@ export function setupOfficialSpeechAutoPick(ctx: { // 3) any English voice (en-US, then en-*) — broadest comprehensible // fallback when the user's locale has no coverage at all // 4) alphabetical first voice, as a last resort - const recommendedId = lookupRecommendedVoiceId(targetLocale, recommendedVoicesByLocale) + const recommendedId = lookupRecommendedVoiceId(targetLocale, recommendedVoicesByProvider[provider] ?? {}) const speaksLocale = (v: VoiceInfo, code: string) => (v.languages || []).some(l => l.code === code) const match = (recommendedId && providerVoices.find(v => v.id === recommendedId)) || providerVoices.find(v => speaksLocale(v, targetLocale)) diff --git a/packages/stage-ui/src/stores/modules/speech.test.ts b/packages/stage-ui/src/stores/modules/speech.test.ts index 23f94c0d1..ce894cf57 100644 --- a/packages/stage-ui/src/stores/modules/speech.test.ts +++ b/packages/stage-ui/src/stores/modules/speech.test.ts @@ -1,8 +1,22 @@ -import { describe, expect, it } from 'vitest' +import { createPinia, setActivePinia } from 'pinia' +import { beforeEach, describe, expect, it, vi } from 'vitest' -import { toSignedPercent } from './speech' +import { OFFICIAL_SPEECH_PROVIDER_ID, OFFICIAL_SPEECH_STREAMING_PROVIDER_ID } from '../../libs/providers/providers/official' +import { useProvidersStore } from '../providers' +import { toSignedPercent, useSpeechStore } from './speech' + +vi.mock('vue-i18n', () => ({ + useI18n: () => ({ + locale: { value: 'en-US' }, + t: (_key: string, fallback?: string) => fallback ?? _key, + }), +})) describe('speech store helpers', () => { + beforeEach(() => { + setActivePinia(createPinia()) + }) + it('formats positive percentages with a plus sign', () => { expect(toSignedPercent(25)).toBe('+25%') }) @@ -15,4 +29,53 @@ describe('speech store helpers', () => { it('formats zero as 0%', () => { expect(toSignedPercent(0)).toBe('0%') }) + + /** + * @example + * await speechStore.loadVoicesForProvider(OFFICIAL_SPEECH_STREAMING_PROVIDER_ID, 'volcengine/seed-tts-2.0') + */ + it('does not load streaming voices before server availability is confirmed', async () => { + const providersStore = useProvidersStore() + const speechStore = useSpeechStore() + const listVoices = vi.fn(async () => []) + const metadata = providersStore.providerMetadata[OFFICIAL_SPEECH_STREAMING_PROVIDER_ID] + metadata.capabilities.listVoices = listVoices + providersStore.providerRuntimeState[OFFICIAL_SPEECH_STREAMING_PROVIDER_ID].isConfigured = false + + const voices = await speechStore.loadVoicesForProvider( + OFFICIAL_SPEECH_STREAMING_PROVIDER_ID, + 'volcengine/seed-tts-2.0', + ) + + expect(voices).toEqual([]) + expect(listVoices).not.toHaveBeenCalled() + }) + + /** + * @example + * speechStore.ensureActiveSpeechModel() + */ + it('resets stale streaming model when the regular official speech provider is active', () => { + const providersStore = useProvidersStore() + const speechStore = useSpeechStore() + speechStore.activeSpeechProvider = OFFICIAL_SPEECH_PROVIDER_ID + speechStore.activeSpeechModel = 'volcengine/seed-tts-2.0' + speechStore.activeSpeechVoiceId = 'zh_female_x' + speechStore.activeSpeechVoice = { + id: 'zh_female_x', + name: 'X', + provider: OFFICIAL_SPEECH_STREAMING_PROVIDER_ID, + languages: [], + } + providersStore.providerRuntimeState[OFFICIAL_SPEECH_PROVIDER_ID].models = [ + { id: 'microsoft/v1', name: 'microsoft/v1', provider: OFFICIAL_SPEECH_PROVIDER_ID }, + { id: 'alibaba/cosyvoice-v2', name: 'alibaba/cosyvoice-v2', provider: OFFICIAL_SPEECH_PROVIDER_ID }, + ] + + speechStore.ensureActiveSpeechModel() + + expect(speechStore.activeSpeechModel).toBe('microsoft/v1') + expect(speechStore.activeSpeechVoiceId).toBe('') + expect(speechStore.activeSpeechVoice).toBeUndefined() + }) }) diff --git a/packages/stage-ui/src/stores/modules/speech.ts b/packages/stage-ui/src/stores/modules/speech.ts index e65538211..839903a59 100644 --- a/packages/stage-ui/src/stores/modules/speech.ts +++ b/packages/stage-ui/src/stores/modules/speech.ts @@ -2,6 +2,7 @@ import type { SpeechProviderWithExtraOptions } from '@xsai-ext/providers/utils' import type { VoiceInfo } from '../providers' +import { errorMessageFrom } from '@moeru/std' import { useLocalStorageManualReset } from '@proj-airi/stage-shared/composables' import { refManualReset } from '@vueuse/core' import { generateSpeech } from '@xsai/generate-speech' @@ -11,7 +12,7 @@ import { useI18n } from 'vue-i18n' import { toXml } from 'xast-util-to-xml' import { x } from 'xastscript' -import { setupOfficialSpeechAutoPick } from '../../libs/providers/providers/official' +import { getDefaultStreamingModel, OFFICIAL_SPEECH_PROVIDER_ID, OFFICIAL_SPEECH_STREAMING_PROVIDER_ID, setupOfficialSpeechAutoPick } from '../../libs/providers/providers/official' import { useProvidersStore } from '../providers' export function toSignedPercent(value: number): string { @@ -88,6 +89,13 @@ export const useSpeechStore = defineStore('speech', () => { return [] } + // Streaming provider visibility is server-driven and only confirmed after + // the auth probe force-configures it. Keep the gate at the public loader so + // pages cannot bypass it and issue `/voices/streaming` while unavailable. + if (provider === OFFICIAL_SPEECH_STREAMING_PROVIDER_ID && !providersStore.configuredProviders[provider]) { + return [] + } + isLoadingSpeechProviderVoices.value = true speechProviderError.value = null @@ -102,7 +110,7 @@ export const useSpeechStore = defineStore('speech', () => { } catch (error) { console.error(`Error fetching voices for ${provider}:`, error) - speechProviderError.value = error instanceof Error ? error.message : 'Unknown error' + speechProviderError.value = errorMessageFrom(error) ?? 'Unknown error' return [] } finally { @@ -115,12 +123,64 @@ export const useSpeechStore = defineStore('speech', () => { return availableVoices.value[provider] || [] } + function clearVoiceSelection() { + activeSpeechVoiceId.value = '' + activeSpeechVoice.value = undefined + } + + // Streaming TTS voices are model-scoped: the server only returns recommended + // voices for an explicit `?model=`. Ensure the active model is a valid + // streaming model id so voice loading gets the right recommendations (parity + // with the HTTP provider's auto-pick). Reseeds the server-curated default + // both when no model is selected AND when `activeSpeechModel` still holds a + // stale id from a previously-active provider (the global model ref is shared + // across providers, and the per-surface reset may not have run yet). No-op + // for non-streaming providers. + function ensureStreamingDefaultModel() { + if (activeSpeechProvider.value !== OFFICIAL_SPEECH_STREAMING_PROVIDER_ID) + return + const streamingModels = providersStore.getModelsForProvider(OFFICIAL_SPEECH_STREAMING_PROVIDER_ID) + const hasValidSelection = !!activeSpeechModel.value && streamingModels.some(m => m.id === activeSpeechModel.value) + if (hasValidSelection) + return + // Replace an empty/stale (non-streaming) selection with the server default. + // When no default can be resolved yet (catalog not loaded), clear it to '' + // so callers pass `undefined` (server returns the full streaming catalog) + // rather than forwarding a stale non-streaming model id as `?model=`. + const nextModel = getDefaultStreamingModel() ?? streamingModels[0]?.id ?? '' + if (activeSpeechModel.value === nextModel) + return + activeSpeechModel.value = nextModel + // The previously-selected voice belonged to the stale/empty model context, + // so drop it; auto-pick re-picks a recommended voice for the new model. + clearVoiceSelection() + } + + function ensureActiveSpeechModel() { + ensureStreamingDefaultModel() + + if (activeSpeechProvider.value !== OFFICIAL_SPEECH_PROVIDER_ID) + return + + const models = providersStore.getModelsForProvider(OFFICIAL_SPEECH_PROVIDER_ID) + if (!models.length) + return + + const hasValidSelection = !!activeSpeechModel.value && models.some(m => m.id === activeSpeechModel.value) + if (hasValidSelection) + return + + activeSpeechModel.value = models[0]?.id ?? '' + clearVoiceSelection() + } + // Watch for provider changes and load voices watch(activeSpeechProvider, async (newProvider) => { - if (newProvider) { - await loadVoicesForProvider(newProvider) - // Don't reset voice settings when changing providers to allow for persistence - } + if (!newProvider) + return + ensureActiveSpeechModel() + await loadVoicesForProvider(newProvider, activeSpeechModel.value || undefined) + // Don't reset voice settings when changing providers to allow for persistence }, { // REVIEW: should we always load voices on init? What will happen when network is not available? immediate: true, @@ -159,7 +219,8 @@ export const useSpeechStore = defineStore('speech', () => { ) onMounted(() => { - loadVoicesForProvider(activeSpeechProvider.value).then(() => { + ensureActiveSpeechModel() + loadVoicesForProvider(activeSpeechProvider.value, activeSpeechModel.value || undefined).then(() => { if (activeSpeechVoiceId.value) { activeSpeechVoice.value = availableVoices.value[activeSpeechProvider.value]?.find(voice => voice.id === activeSpeechVoiceId.value) } @@ -173,6 +234,10 @@ export const useSpeechStore = defineStore('speech', () => { uiLocale: locale, }) + watch(providerModels, () => { + ensureActiveSpeechModel() + }) + watch([activeSpeechVoiceId, availableVoices], ([voiceId, voices]) => { if (voiceId) { // For OpenAI Compatible, create a custom voice object (no voices available from API) @@ -334,6 +399,8 @@ export const useSpeechStore = defineStore('speech', () => { speech, loadVoicesForProvider, getVoicesForProvider, + ensureStreamingDefaultModel, + ensureActiveSpeechModel, generateSSML, resetState, } diff --git a/packages/stage-ui/src/stores/providers.ts b/packages/stage-ui/src/stores/providers.ts index 157cdfbca..8534da216 100644 --- a/packages/stage-ui/src/stores/providers.ts +++ b/packages/stage-ui/src/stores/providers.ts @@ -2275,6 +2275,19 @@ export const useProvidersStore = defineStore('providers', () => { const providerValidationInFlight = new Map>() const providerRevalidationLoops = new Map void }>() + // Server-driven availability overrides for providers whose visibility can + // only be decided at runtime from the backend (e.g. the streaming TTS + // provider, which exists only when `UNSPEECH_UPSTREAM.streaming` is + // configured server-side). A `false` entry hides the provider from the + // available lists regardless of its static `isAvailableBy`; an absent entry + // means no override. Written by the auth-sync glue after it probes the + // server. Reactive so the available/configured provider lists re-derive. + const providerAvailabilityOverrides = ref>({}) + + function setProviderAvailabilityOverride(providerId: string, available: boolean) { + providerAvailabilityOverrides.value = { ...providerAvailabilityOverrides.value, [providerId]: available } + } + const configuredProviders = computed(() => { const result: Record = {} for (const [key, state] of Object.entries(providerRuntimeState.value)) { @@ -2686,9 +2699,17 @@ export const useProvidersStore = defineStore('providers', () => { } const availableProvidersMetadata = computedAsync(async () => { + // Spread-read the overrides synchronously so this re-runs when a + // server-driven availability flips: computedAsync uses watchEffect, which + // only tracks reactive reads before the first `await` — the per-provider + // `isAvailableBy()` below runs after one, so reads inside it aren't tracked. + const overrides = { ...providerAvailabilityOverrides.value } const providers: ProviderMetadata[] = [] for (const provider of allProvidersMetadata.value) { + if (overrides[provider.id] === false) + continue + const p = getProviderMetadata(provider.id) const isAvailableBy = p.isAvailableBy || (() => true) @@ -2786,6 +2807,7 @@ export const useProvidersStore = defineStore('providers', () => { resetProviderSettings, forceProviderConfigured, setProviderUnconfigured, + setProviderAvailabilityOverride, availableProvidersMetadata, allChatProvidersMetadata, allAudioSpeechProvidersMetadata,