nanakura/moeka-project
1
0
Fork 0
Code Issues Pull Requests 1 Actions 1 Packages Projects Releases Wiki Activity

feat(server): server-controlled default streaming TTS model

Browse Source 
The streaming speech provider settings page was seeding its model
picker with a hardcoded `volcengine/seed-tts-2.0`, which contradicts
the just-landed "streaming model catalog comes from the server"
contract. Now the default also comes from configKV.

- `UNSPEECH_UPSTREAM.streaming` gains `defaultModel?: string`.
- `GET /api/v1/audio/models/streaming` response gains `default: string
  | null` reading that field.
- admin slice apply: `streaming.defaultModel` survives key/URL
  rotation alongside `streaming.models`.
- `providerOfficialSpeechStreaming.listModels` stashes the value in a
  module-scope ref, exposed via `getDefaultStreamingModel()`.
- Streaming speech settings page reads the helper instead of the
  hardcoded id; falls back to the first server-returned model if the
  operator hasn't curated a default. `handleGenerateSpeech` no longer
  has a `seed-tts-2.0` backstop — a missing backend prefix in the
  selected model id now throws instead of silently defaulting.
This commit is contained in:
RainbowBird
2026-05-24 21:13:52 +08:00
parent 18e6380e47
commit 6f63ce96e9
22 changed files with 569 additions and 142 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/server/src/routes/admin/config/router/index.ts
+6
View File
@@ -91,6 +91,12 @@ const UnspeechSliceSchema = object({
),
plaintextKey: pipe(string(), nonEmpty('streaming.plaintextKey is required'), maxLength(MAX_KEY_LENGTH)),
keyEntryId: optional(pipe(string(), nonEmpty(), maxLength(200), NO_PIPE)),
models: optional(array(object({
id: pipe(string(), nonEmpty('streaming.models[].id is required'), maxLength(200)),
name: optional(pipe(string(), nonEmpty(), maxLength(200))),
description: optional(pipe(string(), nonEmpty(), maxLength(500))),
}))),
defaultModel: optional(pipe(string(), nonEmpty('streaming.defaultModel must not be empty'), maxLength(200))),
})),
})
apps/server/src/routes/openai/v1/index.ts
+11 -14
View File
@@ -290,7 +290,6 @@ export function createV1Routes(
logger.withError(err).warn('Failed to close stream writer')
}
// Extract usage from final SSE data lines
let usage: UsageInfo = {}
try {
const lines = tailBuffer.split('\n').filter(l => l.startsWith('data: ') && !l.includes('[DONE]'))
@@ -702,14 +701,9 @@ export function createV1Routes(
}
async function handleListTTSModels(_c: Context<HonoEnv>) {
// Surface the concrete TTS models the operator has configured plus the
// `auto` alias. Clients need real model ids to pass `?model=<id>` to
// `/audio/voices`, otherwise the voice catalog endpoint can never resolve
// anything beyond the DEFAULT_TTS_MODEL catalog — which is the bug that
// hid the Azure voices from the UI.
//
// `auto` is kept on top as an explicit "use the operator default" knob
// for clients that don't care which concrete model handles them.
// Surface the concrete TTS models the operator has configured. The UI
// should select an explicit model id so voice catalog requests stay
// model-scoped instead of hiding behind DEFAULT_TTS_MODEL.
const config = await configKV.getOrThrow('LLM_ROUTER_CONFIG')
// `LLM_ROUTER_CONFIG` is `optional()` at the schema, so its inferred type
// tolerates `undefined`. `getOrThrow` already throws on missing entries,
@@ -717,29 +711,32 @@ export function createV1Routes(
// a TS narrowing aid.
const modelIds = Object.keys(config?.tts?.models ?? {}).sort()
return Response.json({
models: [
{ id: 'auto', name: 'Auto' },
...modelIds.map(id => ({ id, name: id })),
],
models: modelIds.map(id => ({ id, name: id })),
})
}
async function handleListStreamingTTSModels(_c: Context<HonoEnv>) {
const unspeech = await configKV.getOptional('UNSPEECH_UPSTREAM')
const models = unspeech?.streaming?.models ?? []
// `available` is the operator-controlled visibility switch the client gates
// the streaming provider on. It tracks whether `UNSPEECH_UPSTREAM.streaming`
// is configured at all — not whether `models[]` happens to be empty — so an
// operator who has wired the upstream but not yet curated models still
// surfaces the provider rather than silently hiding it.
return Response.json({
available: !!unspeech?.streaming?.baseURL,
models: models.map(m => ({
id: m.id,
name: m.name ?? m.id,
description: m.description,
})),
default: unspeech?.streaming?.defaultModel ?? null,
})
}
const chatGuard = configGuard(configKV, ['FLUX_PER_REQUEST'], 'Service is not available yet')
const ttsGuard = configGuard(configKV, ['FLUX_PER_1K_CHARS_TTS'], 'TTS service is not available yet')
// 60 requests per minute per user for LLM completions
const completionsRateLimit = rateLimiter({ max: 60, windowSec: 60, metrics: rateLimitMetrics, routeLabel: 'openai.completions' })
// OpenAI-compatible surface (mounted at /api/v1/openai). Only routes that
apps/server/src/routes/openai/v1/route.test.ts
+38 -18
View File
@@ -11,8 +11,6 @@ import { afterAll, describe, expect, it, vi } from 'vitest'
import { createV1Routes } from '.'
import { ApiError } from '../../../utils/error'
// --- Mock helpers ---
function createMockFluxService(flux = 100): FluxService {
return {
getFlux: vi.fn(async () => ({ userId: 'user-1', flux })),
@@ -165,8 +163,6 @@ function createTestApp(
const testUser = { id: 'user-1', name: 'Test User', email: 'test@example.com' }
// --- Tests ---
describe('v1CompletionsRoutes', () => {
const originalFetch = globalThis.fetch
@@ -319,12 +315,10 @@ describe('v1CompletionsRoutes', () => {
const data = await res.json() as { id: string }
expect(data.id).toBe('chatcmpl-1')
// Verify flux was debited via billingService
expect(billingService.consumeFluxForLLM).toHaveBeenCalledWith(
expect.objectContaining({ userId: 'user-1', amount: 1 }),
)
// Verify upstream was called with correct URL and resolved model
expect(globalThis.fetch).toHaveBeenCalledWith(
'http://mock-gateway/chat/completions',
expect.objectContaining({
@@ -406,13 +400,11 @@ describe('v1CompletionsRoutes', () => {
)
expect(res.status).toBe(500)
// Post-billing: no charge on failed requests
expect(billingService.consumeFluxForLLM).not.toHaveBeenCalled()
})
it('should return 503 when config keys are missing', async () => {
const configKV = createMockConfigKV()
// Override getOptional to return null for required keys
configKV.getOptional = vi.fn(async () => null)
const app = createTestApp(createMockFluxService(), configKV)
@@ -748,7 +740,7 @@ describe('v1CompletionsRoutes', () => {
})
describe('gET /api/v1/audio/models', () => {
it('exposes auto alias plus every configured tts model id', async () => {
it('exposes every configured tts model id', async () => {
const app = createTestApp(
createMockFluxService(),
createMockConfigKV({
@@ -771,14 +763,13 @@ describe('v1CompletionsRoutes', () => {
expect(res.status).toBe(200)
const data = await res.json() as { models: { id: string, name: string }[] }
expect(data.models[0]).toEqual({ id: 'auto', name: 'Auto' })
expect(data.models.slice(1).map(m => m.id)).toEqual([
expect(data.models.map(m => m.id)).toEqual([
'alibaba/cosyvoice-v2',
'microsoft/v1',
])
})
it('returns only the auto alias when no tts models are configured', async () => {
it('returns an empty list when no tts models are configured', async () => {
const app = createTestApp(
createMockFluxService(),
createMockConfigKV({
@@ -793,7 +784,7 @@ describe('v1CompletionsRoutes', () => {
expect(res.status).toBe(200)
const data = await res.json() as { models: { id: string, name: string }[] }
expect(data.models).toEqual([{ id: 'auto', name: 'Auto' }])
expect(data.models).toEqual([])
})
it('should return 401 when unauthenticated', async () => {
@@ -805,7 +796,7 @@ describe('v1CompletionsRoutes', () => {
})
describe('gET /api/v1/audio/models/streaming', () => {
it('returns the operator-configured streaming model catalog', async () => {
it('returns the operator-configured streaming model catalog + default', async () => {
const app = createTestApp(
createMockFluxService(),
createMockConfigKV({
@@ -818,6 +809,7 @@ describe('v1CompletionsRoutes', () => {
{ id: 'volcengine/seed-tts-2.0', name: 'Volcengine Seed-TTS 2.0', description: 'TTS 2.0' },
{ id: 'volcengine/seed-tts-1.0' },
],
defaultModel: 'volcengine/seed-tts-2.0',
},
},
}),
@@ -829,11 +821,37 @@ describe('v1CompletionsRoutes', () => {
)
expect(res.status).toBe(200)
const data = await res.json() as { models: { id: string, name: string, description?: string }[] }
const data = await res.json() as { available: boolean, models: { id: string, name: string, description?: string }[], default: string | null }
expect(data.available).toBe(true)
expect(data.models).toEqual([
{ id: 'volcengine/seed-tts-2.0', name: 'Volcengine Seed-TTS 2.0', description: 'TTS 2.0' },
{ id: 'volcengine/seed-tts-1.0', name: 'volcengine/seed-tts-1.0' },
])
expect(data.default).toBe('volcengine/seed-tts-2.0')
})
it('returns default: null when operator has not set a streaming default', async () => {
const app = createTestApp(
createMockFluxService(),
createMockConfigKV({
UNSPEECH_UPSTREAM: {
restBaseURL: 'http://unspeech.local:5933',
streaming: {
baseURL: 'wss://unspeech.local',
keys: [{ id: 'k1', ciphertext: 'enc' }],
models: [{ id: 'volcengine/seed-tts-2.0', name: 'Vol' }],
},
},
}),
)
const res = await app.fetch(
new Request('http://localhost/api/v1/audio/models/streaming', { method: 'GET' }),
{ user: testUser } as any,
)
const data = await res.json() as { default: string | null }
expect(data.default).toBeNull()
})
it('returns an empty list when UNSPEECH_UPSTREAM is unset', async () => {
@@ -845,11 +863,12 @@ describe('v1CompletionsRoutes', () => {
)
expect(res.status).toBe(200)
const data = await res.json() as { models: unknown[] }
const data = await res.json() as { available: boolean, models: unknown[] }
expect(data.available).toBe(false)
expect(data.models).toEqual([])
})
it('returns an empty list when streaming subtree has no models', async () => {
it('reports available: true with empty models when streaming subtree has no models', async () => {
const app = createTestApp(
createMockFluxService(),
createMockConfigKV({
@@ -869,7 +888,8 @@ describe('v1CompletionsRoutes', () => {
)
expect(res.status).toBe(200)
const data = await res.json() as { models: unknown[] }
const data = await res.json() as { available: boolean, models: unknown[] }
expect(data.available).toBe(true)
expect(data.models).toEqual([])
})
apps/server/src/services/adapters/config-kv.test.ts
+29 -5
View File
@@ -21,8 +21,6 @@ describe('configKVService', () => {
service = createConfigKVService(redis as any)
})
// --- get ---
it('get should throw 503 when key is not set', async () => {
await expect(service.getOrThrow('FLUX_PER_1K_CHARS_TTS'))
.rejects
@@ -43,8 +41,6 @@ describe('configKVService', () => {
expect(redis.get).toHaveBeenCalledWith(configRedisKey('FLUX_PER_REQUEST'))
})
// --- getOptional ---
it('getOptional should return schema default when key has one', async () => {
const value = await service.getOptional('FLUX_PER_REQUEST')
expect(value).toBe(5)
@@ -62,7 +58,35 @@ describe('configKVService', () => {
expect(value).toBe(200)
})
// --- set ---
it('getOptional should throw CONFIG_INVALID when Redis contains malformed JSON', async () => {
// ROOT CAUSE:
//
// If an operator edits config:LLM_ROUTER_CONFIG directly with invalid JSON,
// JSON.parse used to throw SyntaxError through the request handler and log
// it as an unhandled 500.
//
// We fixed this by translating stored config parse/validation failures into
// a stable API error at the configKV boundary.
redis._store.set(configRedisKey('LLM_ROUTER_CONFIG'), '{"llm":{}')
await expect(service.getOptional('LLM_ROUTER_CONFIG'))
.rejects
.toMatchObject({
statusCode: 503,
errorCode: 'CONFIG_INVALID',
})
})
it('getOptional should throw CONFIG_INVALID when Redis contains schema-invalid JSON', async () => {
redis._store.set(configRedisKey('FLUX_PER_REQUEST'), JSON.stringify('5'))
await expect(service.getOptional('FLUX_PER_REQUEST'))
.rejects
.toMatchObject({
statusCode: 503,
errorCode: 'CONFIG_INVALID',
})
})
it('set should write value to Redis with prefix', async () => {
await service.set('FLUX_PER_REQUEST', 10)
apps/server/src/services/adapters/config-kv.ts
+15 -1
View File
@@ -1,6 +1,7 @@
import type Redis from 'ioredis'
import type { InferOutput } from 'valibot'
import { errorMessageFrom } from '@moeru/std'
import { any, array, boolean, check, nonEmpty, number, object, optional, parse, picklist, pipe, record, regex, string } from 'valibot'
import { createServiceUnavailableError } from '../../utils/error'
@@ -71,6 +72,7 @@ export const streamingTtsUpstreamSchema = object({
})),
[],
),
defaultModel: optional(string()),
})
export const unspeechUpstreamSchema = object({
@@ -158,7 +160,19 @@ type ConfigDefinitions = {
type ConfigKey = keyof ConfigDefinitions
function parseValue<K extends ConfigKey>(key: K, raw: string): ConfigDefinitions[K] {
return parse(ConfigEntrySchemas[key], JSON.parse(raw)) as ConfigDefinitions[K]
try {
return parse(ConfigEntrySchemas[key], JSON.parse(raw)) as ConfigDefinitions[K]
}
catch (error) {
throw createServiceUnavailableError(
'Service configuration is invalid',
'CONFIG_INVALID',
{
key,
message: errorMessageFrom(error) ?? 'Unknown config parse error',
},
)
}
}
function serializeValue<K extends ConfigKey>(key: K, value: ConfigDefinitions[K]): string {
apps/server/src/services/adapters/tts/azure.ts
+1 -1
View File
@@ -228,7 +228,7 @@ export const azureAdapter: TtsAdapter = {
if (!ctx.keyPlaintext)
throw createServiceUnavailableError('azure tts key not configured', 'AZURE_TTS_NOT_CONFIGURED')
const url = `${ctx.unspeechBaseURL.replace(/\/+$/, '')}/api/voices?backend=microsoft&region=${encodeURIComponent(ctx.region)}`
const url = `${ctx.unspeechBaseURL.replace(/\/+$/, '')}/api/voices?provider=microsoft&region=${encodeURIComponent(ctx.region)}`
let response: Response
try {
apps/server/src/services/adapters/tts/dashscope-cosyvoice.ts
+1 -1
View File
@@ -109,7 +109,7 @@ export const dashscopeCosyvoiceAdapter: TtsAdapter = {
// (unspeech/pkg/backend/alibaba/voices.go `//go:embed voices.json`),
// so this call is in-memory on unspeech's side and only crosses a TCP
// hop. No upstream credential is required.
const url = `${ctx.unspeechBaseURL.replace(/\/+$/, '')}/api/voices?backend=alibaba`
const url = `${ctx.unspeechBaseURL.replace(/\/+$/, '')}/api/voices?provider=alibaba`
let response: Response
try {
apps/server/src/services/adapters/tts/index.test.ts
+6 -6
View File
@@ -61,7 +61,7 @@ describe('getAdapter', () => {
})
describe('dashscopeCosyvoiceAdapter.getVoiceCatalog', () => {
it('calls unspeech with backend=alibaba (no Bearer)', async () => {
it('calls unspeech with provider=alibaba (no Bearer)', async () => {
const adapter = getAdapter('dashscope-cosyvoice')
const fetchImpl = vi.fn(async () => new Response(JSON.stringify({
voices: [{ id: 'longxiaochun_v2', name: 'Longxiaochun v2' }],
@@ -75,7 +75,7 @@ describe('dashscopeCosyvoiceAdapter.getVoiceCatalog', () => {
expect(voices).toEqual([{ id: 'longxiaochun_v2', name: 'Longxiaochun v2' }])
const [calledUrl, init] = (fetchImpl as unknown as { mock: { calls: [string, RequestInit][] } }).mock.calls[0]
expect(calledUrl).toBe('http://unspeech.local/api/voices?backend=alibaba')
expect(calledUrl).toBe('http://unspeech.local/api/voices?provider=alibaba')
const headers = (init.headers ?? {}) as Record<string, string>
expect(headers.Authorization).toBeUndefined()
})
@@ -92,7 +92,7 @@ describe('dashscopeCosyvoiceAdapter.getVoiceCatalog', () => {
})
describe('volcengineAdapter.getVoiceCatalog', () => {
it('calls unspeech with backend=volcengine and forwards adapterParams.model as ?model=', async () => {
it('calls unspeech with provider=volcengine and forwards adapterParams.model as ?model=', async () => {
const adapter = getAdapter('volcengine')
const fetchImpl = vi.fn(async () => new Response(JSON.stringify({
voices: [{ id: 'zh_female_x', name: 'X' }],
@@ -106,7 +106,7 @@ describe('volcengineAdapter.getVoiceCatalog', () => {
expect(voices).toEqual([{ id: 'zh_female_x', name: 'X' }])
const [calledUrl] = (fetchImpl as unknown as { mock: { calls: [string, RequestInit][] } }).mock.calls[0]
expect(calledUrl).toBe('http://unspeech.local/api/voices?backend=volcengine&model=seed-tts-2.0')
expect(calledUrl).toBe('http://unspeech.local/api/voices?provider=volcengine&model=seed-tts-2.0')
})
it('omits ?model= when adapterParams.model is not set', async () => {
@@ -118,7 +118,7 @@ describe('volcengineAdapter.getVoiceCatalog', () => {
fetchImpl,
})
const [calledUrl] = (fetchImpl as unknown as { mock: { calls: [string, RequestInit][] } }).mock.calls[0]
expect(calledUrl).toBe('http://unspeech.local/api/voices?backend=volcengine')
expect(calledUrl).toBe('http://unspeech.local/api/voices?provider=volcengine')
})
})
@@ -140,7 +140,7 @@ describe('azureAdapter.getVoiceCatalog', () => {
expect(voices).toEqual([{ id: 'en-US-AvaMultilingualNeural', name: 'Ava' }])
expect(fetchImpl).toHaveBeenCalledTimes(1)
const [calledUrl, init] = (fetchImpl as unknown as { mock: { calls: [string, RequestInit][] } }).mock.calls[0]
expect(calledUrl).toBe('http://unspeech.local:5933/api/voices?backend=microsoft&region=eastasia')
expect(calledUrl).toBe('http://unspeech.local:5933/api/voices?provider=microsoft&region=eastasia')
const headers = init.headers as Record<string, string>
expect(headers.Authorization).toBe('Bearer subscription-key-XYZ')
})
apps/server/src/services/adapters/tts/volcengine.ts
+1 -1
View File
@@ -120,7 +120,7 @@ export const volcengineAdapter: TtsAdapter = {
// further by `compatible_models` — adapterParams.model is the operator-
// configured resource id (e.g. `seed-tts-2.0`).
const url = new URL(`${ctx.unspeechBaseURL.replace(/\/+$/, '')}/api/voices`)
url.searchParams.set('backend', 'volcengine')
url.searchParams.set('provider', 'volcengine')
const apiResourceId = typeof ctx.adapterParams?.model === 'string'
? ctx.adapterParams.model
: undefined
apps/server/src/services/domain/admin/router-config/index.ts
+15 -6
View File
@@ -96,6 +96,10 @@ export interface UnspeechSliceInput {
plaintextKey: string
/** @default 'volcengine-prod-1' */
keyEntryId?: string
/** Operator-curated streaming models exposed to the frontend picker. */
models?: Array<{ id: string, name?: string, description?: string }>
/** Server-curated default streaming model id. */
defaultModel?: string
}
}
@@ -262,7 +266,8 @@ export function buildUnspeechSlice(input: UnspeechSliceInput, envelope: Envelope
baseURL: input.streaming.upstreamURL,
keys: [{ id: keyEntryId, ciphertext }],
adapterParams: {},
models: [],
models: input.streaming.models ?? [],
defaultModel: input.streaming.defaultModel,
},
},
}
@@ -451,18 +456,22 @@ export function createAdminRouterConfigService(deps: AdminRouterConfigDeps) {
nextRouterConfig = buildNextRouterConfig(input.mode, existing, llmTtsSlices)
}
// Step 3: build the next UNSPEECH_UPSTREAM. Streaming `models` carry the
// operator-curated catalog and must survive key/URL rotation, so we read
// existing and graft them onto the new value when the slice's streaming
// Step 3: build the next UNSPEECH_UPSTREAM. Streaming `models` +
// `defaultModel` are operator-curated and must survive key/URL rotation,
// so we graft them from the existing entry when the slice's streaming
// subtree is set (otherwise there's nothing to merge into).
let nextUnspeech: UnspeechUpstream | undefined
if (unspeechSlice) {
const existing = await deps.configKV.getOptional('UNSPEECH_UPSTREAM')
const newValue = unspeechSlice.value
if (newValue.streaming && existing?.streaming?.models?.length) {
if (newValue.streaming && existing?.streaming) {
nextUnspeech = {
...newValue,
streaming: { ...newValue.streaming, models: existing.streaming.models },
streaming: {
...newValue.streaming,
models: existing.streaming.models?.length ? existing.streaming.models : newValue.streaming.models,
defaultModel: existing.streaming.defaultModel ?? newValue.streaming.defaultModel,
},
}
}
else {
apps/server/src/services/domain/admin/router-config/tests/admin-router-config.test.ts
+12
View File
@@ -356,11 +356,23 @@ describe('createAdminRouterConfigService', () => {
streaming: {
upstreamURL: 'wss://unspeech.example/v1/audio/speech/stream',
plaintextKey: 'volc',
models: [
{ id: 'volcengine/seed-tts-2.0', name: 'Seed-TTS 2.0', description: 'Low-latency streaming TTS' },
],
defaultModel: 'volcengine/seed-tts-2.0',
},
}],
})
expect(kv.store.has('UNSPEECH_UPSTREAM')).toBe(true)
expect(kv.store.get('UNSPEECH_UPSTREAM')).toMatchObject({
streaming: {
models: [
{ id: 'volcengine/seed-tts-2.0', name: 'Seed-TTS 2.0', description: 'Low-latency streaming TTS' },
],
defaultModel: 'volcengine/seed-tts-2.0',
},
})
expect(kv.store.has('LLM_ROUTER_CONFIG')).toBe(false)
expect(result.invalidatedKeys).toEqual(['UNSPEECH_UPSTREAM'])
expect(captured.map(p => JSON.parse(p.payload).key)).toEqual(['UNSPEECH_UPSTREAM'])
apps/server/src/services/domain/llm-router/router.ts
+45 -32
View File
@@ -1,6 +1,7 @@
import type { Buffer } from 'node:buffer'
import type Redis from 'ioredis'
import type { Voice } from 'unspeech'
import type { GatewayMetrics } from '../../../otel'
import type { EnvelopeCrypto } from '../../../utils/envelope-crypto'
@@ -176,6 +177,7 @@ export function createLlmRouterService(options: CreateLlmRouterServiceOptions) {
const logger = useLogger('llm-router').useGlobalConfig()
const fetchImpl = options.fetchImpl ?? globalThis.fetch
const configLoader = createConfigLoader({ configKV: options.configKV, ttlMs: options.configCacheTtlMs })
const ttsVoiceCatalogLoads = new Map<string, Promise<Voice[]>>()
/**
* Run one upstream's key list in order, returning either:
@@ -652,44 +654,55 @@ export function createLlmRouterService(options: CreateLlmRouterServiceOptions) {
}
}
const unspeechBaseURL = (await options.configKV.getOrThrow('UNSPEECH_UPSTREAM')).restBaseURL
const existingLoad = ttsVoiceCatalogLoads.get(cacheKey)
if (existingLoad != null)
return existingLoad
// Live providers (Azure) need the decrypted Azure subscription key + region;
// static-catalog providers (alibaba, volcengine) ignore both. The router
// decrypts unconditionally so the adapter doesn't have to know which
// category it's in — adapters that don't need creds just won't read them.
const region = typeof upstream.adapterParams?.region === 'string'
? upstream.adapterParams.region
: undefined
const load = (async () => {
const unspeechBaseURL = (await options.configKV.getOrThrow('UNSPEECH_UPSTREAM')).restBaseURL
const keyEntry = upstream.keys[0]
const plaintext = slice.model.provider === 'azure'
? options.envelopeCrypto.decryptKey(keyEntry.ciphertext, { modelName, keyEntryId: keyEntry.id })
: undefined
// Live providers (Azure) need the decrypted Azure subscription key + region;
// static-catalog providers (alibaba, volcengine) ignore both. The router
// decrypts unconditionally so the adapter doesn't have to know which
// category it's in — adapters that don't need creds just won't read them.
const region = typeof upstream.adapterParams?.region === 'string'
? upstream.adapterParams.region
: undefined
try {
const voices = await adapter.getVoiceCatalog({
keyPlaintext: plaintext,
region,
adapterParams: upstream.adapterParams ?? {},
unspeechBaseURL,
fetchImpl,
})
const keyEntry = upstream.keys[0]
const plaintext = slice.model.provider === 'azure'
? options.envelopeCrypto.decryptKey(keyEntry.ciphertext, { modelName, keyEntryId: keyEntry.id })
: undefined
// Cache only on success — failure responses must NOT be persisted or
// the next admin reconfigure would have to wait out the TTL even after
// fixing credentials.
const ttl = options.ttsVoiceCacheTtlSeconds ?? ttsVoicesCacheTtl(slice.model.provider)
await options.redis.set(cacheKey, JSON.stringify(voices), 'EX', ttl)
.catch((err) => {
logger.withError(err).withFields({ cacheKey }).warn('failed to write tts voices cache')
try {
const voices = await adapter.getVoiceCatalog({
keyPlaintext: plaintext,
region,
adapterParams: upstream.adapterParams ?? {},
unspeechBaseURL,
fetchImpl,
})
return voices
}
finally {
plaintext?.fill(0)
}
// Cache only on success — failure responses must NOT be persisted or
// the next admin reconfigure would have to wait out the TTL even after
// fixing credentials.
const ttl = options.ttsVoiceCacheTtlSeconds ?? ttsVoicesCacheTtl(slice.model.provider)
await options.redis.set(cacheKey, JSON.stringify(voices), 'EX', ttl)
.catch((err) => {
logger.withError(err).withFields({ cacheKey }).warn('failed to write tts voices cache')
})
return voices
}
finally {
plaintext?.fill(0)
}
})().finally(() => {
ttsVoiceCatalogLoads.delete(cacheKey)
})
ttsVoiceCatalogLoads.set(cacheKey, load)
return load
}
/**
apps/server/src/services/domain/llm-router/tests/router.test.ts
+43
View File
@@ -754,5 +754,48 @@ describe('createLlmRouterService', () => {
// adapter's `Error & { status }` was read as undefined.
expect(fallbackCalls[0][1]).toMatchObject({ reason: '401' })
})
it('listTtsVoices deduplicates concurrent cold-cache upstream fetches per model', async () => {
// ROOT CAUSE:
//
// Azure voice catalogs are cached after a successful fetch, but concurrent
// cold-cache requests used to miss Redis together and each hit unspeech's
// microsoft voices endpoint. That can amplify one settings-page open into
// several Azure voices/list calls and trigger upstream 429.
//
// We fixed this by sharing the in-flight catalog load for the same
// provider/model cache key. Failures are still returned to every caller and
// are not cached.
const { config, crypto } = makeTtsConfig({
upstreams: [{ baseURL: 'https://az.example', keyIds: ['kA1'], adapterParams: { region: 'eastasia' } }],
})
let resolveFetch!: () => void
const fetchImpl = vi.fn(() => new Promise<Response>((resolve) => {
resolveFetch = () => resolve(happyResponse({
voices: [{ id: 'en-US-AvaMultilingualNeural', name: 'Ava' }],
}))
}))
const router = createLlmRouterService({
configKV: makeConfigKV(config),
envelopeCrypto: crypto,
gatewayMetrics: null,
fetchImpl,
redis: makeRedisStub(),
})
const first = router.listTtsVoices('tts-test')
const second = router.listTtsVoices('tts-test')
await vi.waitFor(() => {
expect(fetchImpl).toHaveBeenCalledTimes(1)
})
resolveFetch()
const [firstVoices, secondVoices] = await Promise.all([first, second])
expect(firstVoices.map(voice => voice.id)).toEqual(['en-US-AvaMultilingualNeural'])
expect(secondVoices.map(voice => voice.id)).toEqual(['en-US-AvaMultilingualNeural'])
})
})
})