feat(server): finalize in-process LLM/TTS router cutover
End-state of the multi-step KTD-5 / KTD-6 / U8 work. The knoway sidecar is no longer reachable from server code; the router is required at boot and now owns chat completions, TTS synthesis, and voice catalog listing. Highlights: - LLM_ROUTER_MASTER_KEY becomes required; app.ts drops the graceful- skip branch and the chat fallback fetch path is gone. - /audio/speech and /audio/voices route through new routeTts / listTtsVoices entries that reuse the chat key-rotator + per-attempt timeout + abort propagation. - DEFAULT_CHAT_MODEL / DEFAULT_TTS_MODEL move from env to configKV so default-model swaps are hot-reloadable via Pub/Sub. - GATEWAY_BASE_URL removed from env schema, .env, .env.local, smoke, verification harness. Redis upstream-voices cache deleted — catalogs come from in-process adapter JSON. - routeTts splits adapter error contract by ApiError statusCode: 4xx propagates without fallback; 5xx folds into the network-failure fallback path. handleTTS wraps billing + span attribute in try/finally to plug a span leak when ttsMeter.accumulate() throws. - seed-router-config.ts rewritten with --merge (default) / --reset / --dry-run modes and env-var key handoff (OPENROUTER_KEY / AZURE_KEY / DASHSCOPE_KEY) so prod seed flows never put plaintext on the CLI. Adds DashScope CosyVoice seeding. Docs (CLAUDE.md, architecture-overview.md, transport-and-routes.md) reflect the new boundary. verifications/llm-router.md replaces the overstated "U1-U9 shipped" line with an evidence-vs-pending table. Tests: full 40-file / 343-case server suite green. New regressions pin ApiError 4xx → no-fallback, ApiError 5xx → fallback, TTS billing failure → span closed and error propagated.
This commit is contained in:
+24
-32
@@ -93,7 +93,7 @@ interface AppDeps {
|
||||
env: Env
|
||||
otel: OtelInstance | null
|
||||
userDeletionService: UserDeletionService
|
||||
llmRouter: LlmRouterService | null
|
||||
llmRouter: LlmRouterService
|
||||
posthog: PostHog | null
|
||||
}
|
||||
|
||||
@@ -169,29 +169,27 @@ export async function buildApp(deps: AppDeps) {
|
||||
// (R16 / KTD-4). Falls back to the in-memory cache TTL on missed messages.
|
||||
// Uses a dedicated ioredis subscriber connection (subscribe mode requires
|
||||
// a separate connection per ioredis docs).
|
||||
if (deps.llmRouter) {
|
||||
const configSub = deps.redis.duplicate()
|
||||
configSub.on('message', (channel, message) => {
|
||||
if (channel !== 'configkv:invalidate')
|
||||
const configSub = deps.redis.duplicate()
|
||||
configSub.on('message', (channel, message) => {
|
||||
if (channel !== 'configkv:invalidate')
|
||||
return
|
||||
try {
|
||||
const payload = JSON.parse(message) as { key?: unknown }
|
||||
if (payload?.key !== 'LLM_ROUTER_CONFIG')
|
||||
return
|
||||
try {
|
||||
const payload = JSON.parse(message) as { key?: unknown }
|
||||
if (payload?.key !== 'LLM_ROUTER_CONFIG')
|
||||
return
|
||||
deps.llmRouter?.invalidateConfig()
|
||||
deps.otel?.gateway?.configReload.add(1, {
|
||||
source: 'pubsub',
|
||||
service_instance_id: deps.env.OTEL_SERVICE_NAME,
|
||||
})
|
||||
}
|
||||
catch (err) {
|
||||
logger.withError(err).warn('Failed to parse configkv:invalidate payload')
|
||||
}
|
||||
})
|
||||
configSub.subscribe('configkv:invalidate').catch((err: unknown) => {
|
||||
logger.withError(err).warn('Failed to subscribe to configkv:invalidate channel')
|
||||
})
|
||||
}
|
||||
deps.llmRouter.invalidateConfig()
|
||||
deps.otel?.gateway?.configReload.add(1, {
|
||||
source: 'pubsub',
|
||||
service_instance_id: deps.env.OTEL_SERVICE_NAME,
|
||||
})
|
||||
}
|
||||
catch (err) {
|
||||
logger.withError(err).warn('Failed to parse configkv:invalidate payload')
|
||||
}
|
||||
})
|
||||
configSub.subscribe('configkv:invalidate').catch((err: unknown) => {
|
||||
logger.withError(err).warn('Failed to subscribe to configkv:invalidate channel')
|
||||
})
|
||||
|
||||
const builtApp = app
|
||||
.use('*', sessionMiddleware(deps.auth, deps.env))
|
||||
@@ -296,7 +294,7 @@ export async function buildApp(deps: AppDeps) {
|
||||
/**
|
||||
* V1 routes for official provider.
|
||||
*/
|
||||
.route('/api/v1/openai', createV1CompletionsRoutes(deps.fluxService, deps.billingService, deps.configKV, deps.requestLogService, deps.ttsMeter, deps.redis, deps.env, deps.llmRouter, deps.otel?.genAi, deps.otel?.revenue, deps.otel?.rateLimit))
|
||||
.route('/api/v1/openai', createV1CompletionsRoutes(deps.fluxService, deps.billingService, deps.configKV, deps.requestLogService, deps.ttsMeter, deps.llmRouter, deps.otel?.genAi, deps.otel?.revenue, deps.otel?.rateLimit))
|
||||
|
||||
/**
|
||||
* Flux routes.
|
||||
@@ -572,17 +570,11 @@ export async function createApp() {
|
||||
})
|
||||
|
||||
// LLM router (KTD-5 in-process replacement for the knoway sidecar).
|
||||
// Graceful skip when LLM_ROUTER_MASTER_KEY is unset: the chat-completions
|
||||
// route falls back to the legacy GATEWAY_BASE_URL fetch path. This branch
|
||||
// exists only for the U4→U8 transition window — U8 makes the master key
|
||||
// (and the router) non-optional and deletes the legacy fallback.
|
||||
// LLM_ROUTER_MASTER_KEY is required at env-parse time, so this provider
|
||||
// always builds a real router — the legacy `null` fallback path is gone.
|
||||
const llmRouter = injeca.provide('services:llmRouter', {
|
||||
dependsOn: { configKV, env: parsedEnv, otel },
|
||||
build: ({ dependsOn }) => {
|
||||
if (dependsOn.env.LLM_ROUTER_MASTER_KEY == null) {
|
||||
logger.warn('LLM_ROUTER_MASTER_KEY is not set; LLM router is disabled and chat completions will use the legacy GATEWAY_BASE_URL path (U4 transition)')
|
||||
return null
|
||||
}
|
||||
const envelopeCrypto = createEnvelopeCrypto({
|
||||
masterKey: dependsOn.env.LLM_ROUTER_MASTER_KEY,
|
||||
previousMasterKey: dependsOn.env.LLM_ROUTER_MASTER_KEY_PREVIOUS,
|
||||
|
||||
Reference in New Issue
Block a user