feat(server): finalize in-process LLM/TTS router cutover

End-state of the multi-step KTD-5 / KTD-6 / U8 work. The knoway sidecar
is no longer reachable from server code; the router is required at boot
and now owns chat completions, TTS synthesis, and voice catalog listing.

Highlights:
- LLM_ROUTER_MASTER_KEY becomes required; app.ts drops the graceful-
  skip branch and the chat fallback fetch path is gone.
- /audio/speech and /audio/voices route through new routeTts /
  listTtsVoices entries that reuse the chat key-rotator + per-attempt
  timeout + abort propagation.
- DEFAULT_CHAT_MODEL / DEFAULT_TTS_MODEL move from env to configKV so
  default-model swaps are hot-reloadable via Pub/Sub.
- GATEWAY_BASE_URL removed from env schema, .env, .env.local, smoke,
  verification harness. Redis upstream-voices cache deleted — catalogs
  come from in-process adapter JSON.
- routeTts splits adapter error contract by ApiError statusCode:
  4xx propagates without fallback; 5xx folds into the network-failure
  fallback path. handleTTS wraps billing + span attribute in try/finally
  to plug a span leak when ttsMeter.accumulate() throws.
- seed-router-config.ts rewritten with --merge (default) / --reset /
  --dry-run modes and env-var key handoff (OPENROUTER_KEY / AZURE_KEY /
  DASHSCOPE_KEY) so prod seed flows never put plaintext on the CLI.
  Adds DashScope CosyVoice seeding.

Docs (CLAUDE.md, architecture-overview.md, transport-and-routes.md)
reflect the new boundary. verifications/llm-router.md replaces the
overstated "U1-U9 shipped" line with an evidence-vs-pending table.

Tests: full 40-file / 343-case server suite green. New regressions pin
ApiError 4xx → no-fallback, ApiError 5xx → fallback, TTS billing
failure → span closed and error propagated.
This commit is contained in:
RainbowBird
2026-05-18 23:32:33 +08:00
parent 5d256e4951
commit 4da4a72703
17 changed files with 957 additions and 408 deletions
+24 -32
View File
@@ -93,7 +93,7 @@ interface AppDeps {
env: Env
otel: OtelInstance | null
userDeletionService: UserDeletionService
llmRouter: LlmRouterService | null
llmRouter: LlmRouterService
posthog: PostHog | null
}
@@ -169,29 +169,27 @@ export async function buildApp(deps: AppDeps) {
// (R16 / KTD-4). Falls back to the in-memory cache TTL on missed messages.
// Uses a dedicated ioredis subscriber connection (subscribe mode requires
// a separate connection per ioredis docs).
if (deps.llmRouter) {
const configSub = deps.redis.duplicate()
configSub.on('message', (channel, message) => {
if (channel !== 'configkv:invalidate')
const configSub = deps.redis.duplicate()
configSub.on('message', (channel, message) => {
if (channel !== 'configkv:invalidate')
return
try {
const payload = JSON.parse(message) as { key?: unknown }
if (payload?.key !== 'LLM_ROUTER_CONFIG')
return
try {
const payload = JSON.parse(message) as { key?: unknown }
if (payload?.key !== 'LLM_ROUTER_CONFIG')
return
deps.llmRouter?.invalidateConfig()
deps.otel?.gateway?.configReload.add(1, {
source: 'pubsub',
service_instance_id: deps.env.OTEL_SERVICE_NAME,
})
}
catch (err) {
logger.withError(err).warn('Failed to parse configkv:invalidate payload')
}
})
configSub.subscribe('configkv:invalidate').catch((err: unknown) => {
logger.withError(err).warn('Failed to subscribe to configkv:invalidate channel')
})
}
deps.llmRouter.invalidateConfig()
deps.otel?.gateway?.configReload.add(1, {
source: 'pubsub',
service_instance_id: deps.env.OTEL_SERVICE_NAME,
})
}
catch (err) {
logger.withError(err).warn('Failed to parse configkv:invalidate payload')
}
})
configSub.subscribe('configkv:invalidate').catch((err: unknown) => {
logger.withError(err).warn('Failed to subscribe to configkv:invalidate channel')
})
const builtApp = app
.use('*', sessionMiddleware(deps.auth, deps.env))
@@ -296,7 +294,7 @@ export async function buildApp(deps: AppDeps) {
/**
* V1 routes for official provider.
*/
.route('/api/v1/openai', createV1CompletionsRoutes(deps.fluxService, deps.billingService, deps.configKV, deps.requestLogService, deps.ttsMeter, deps.redis, deps.env, deps.llmRouter, deps.otel?.genAi, deps.otel?.revenue, deps.otel?.rateLimit))
.route('/api/v1/openai', createV1CompletionsRoutes(deps.fluxService, deps.billingService, deps.configKV, deps.requestLogService, deps.ttsMeter, deps.llmRouter, deps.otel?.genAi, deps.otel?.revenue, deps.otel?.rateLimit))
/**
* Flux routes.
@@ -572,17 +570,11 @@ export async function createApp() {
})
// LLM router (KTD-5 in-process replacement for the knoway sidecar).
// Graceful skip when LLM_ROUTER_MASTER_KEY is unset: the chat-completions
// route falls back to the legacy GATEWAY_BASE_URL fetch path. This branch
// exists only for the U4→U8 transition window — U8 makes the master key
// (and the router) non-optional and deletes the legacy fallback.
// LLM_ROUTER_MASTER_KEY is required at env-parse time, so this provider
// always builds a real router — the legacy `null` fallback path is gone.
const llmRouter = injeca.provide('services:llmRouter', {
dependsOn: { configKV, env: parsedEnv, otel },
build: ({ dependsOn }) => {
if (dependsOn.env.LLM_ROUTER_MASTER_KEY == null) {
logger.warn('LLM_ROUTER_MASTER_KEY is not set; LLM router is disabled and chat completions will use the legacy GATEWAY_BASE_URL path (U4 transition)')
return null
}
const envelopeCrypto = createEnvelopeCrypto({
masterKey: dependsOn.env.LLM_ROUTER_MASTER_KEY,
previousMasterKey: dependsOn.env.LLM_ROUTER_MASTER_KEY_PREVIOUS,