feat(server): apply official llm alias routing policy

2026-07-01 22:34:57 +08:00
parent 14fb183723
commit ed8c93c77b
2 changed files with 287 additions and 7 deletions
@@ -1,3 +1,4 @@
+import type { OfficialProviderAliasRoute } from '../../../../../schemas/official-catalog'
 import type { UsageInfo } from '../../../../../services/domain/billing/billing'
 import type { GatewayCallback } from '../../gateway'
 import type { V1RouteDeps } from '../../types'
@@ -5,6 +6,7 @@ import type { V1RouteDeps } from '../../types'
 import { useLogger } from '@guiiai/logg'

 import { extractUsageFromBody } from '../../../../../services/domain/billing/billing'
+import { createBadRequestError } from '../../../../../utils/error'
 import { nanoid } from '../../../../../utils/id'
 import { buildSafeResponseHeaders } from '../../http/response'
 import { createOpenAiRouteBilling } from '../../middlewares/billing'
@@ -41,7 +43,8 @@ export function chatCompletions(deps: V1RouteDeps): GatewayCallback<'chat.comple

    const body = input.body
    const requestedAlias = typeof body.model === 'string' && body.model.length > 0 ? body.model : 'auto'
-    const requestModel = await resolveChatModelAlias(deps, requestedAlias)
+    const aliasPlan = await resolveChatModelAliasPlan(deps, requestedAlias)
+    let requestModel = aliasPlan.modelIds[0]

    const stream = !!body.stream
    logger.withFields({
@@ -81,11 +84,19 @@ export function chatCompletions(deps: V1RouteDeps): GatewayCallback<'chat.comple
    // fluxConsumed: 0 while real cost was incurred — a silent revenue leak.
    // Source: codex review 2026-05-15 HIGH #1.
    const clientAbort = input.abortSignal
-    const routeCtx = newRouteContext()
+    let routeCtx = newRouteContext()
    let response: Response
    try {
-      response = await telemetry.runWithSpan(span, () =>
-        deps.llmRouter.route({ modelName: requestModel, body, headers: {}, abortSignal: clientAbort }, routeCtx))
+      const routed = await telemetry.runWithSpan(span, () =>
+        routeChatAliasCandidates({
+          deps,
+          body,
+          modelIds: aliasPlan.modelIds,
+          abortSignal: clientAbort,
+        }))
+      response = routed.response
+      routeCtx = routed.routeCtx
+      requestModel = routed.modelId
    }
    catch (err) {
      telemetry.failSpan(span, 'Router exhausted or unknown model')
@@ -198,7 +209,11 @@ export function chatCompletions(deps: V1RouteDeps): GatewayCallback<'chat.comple
  }
 }

-async function resolveChatModelAlias(deps: V1RouteDeps, aliasId: string): Promise<string> {
+interface ChatModelAliasPlan {
+  modelIds: string[]
+}
+
+async function resolveChatModelAliasPlan(deps: V1RouteDeps, aliasId: string): Promise<ChatModelAliasPlan> {
  const config = await deps.configKV.getOrThrow('LLM_ROUTER_CONFIG')
  const defaultModel = await deps.configKV.getOrThrow('DEFAULT_CHAT_MODEL')
  const modelIds = [
@@ -211,8 +226,82 @@ async function resolveChatModelAlias(deps: V1RouteDeps, aliasId: string): Promis
  })

  const alias = await deps.officialCatalogService.resolveEnabledAlias('llm', aliasId)
-  const primary = alias.routes.find(route => route.pool === 'primary')
-  return (primary ?? alias.routes[0]).routerModelId
+  const primaryRoutes = alias.routes.filter(route => route.pool === 'primary')
+  const fallbackRoutes = alias.fallbackEnabled
+    ? alias.routes.filter(route => route.pool === 'fallback')
+    : []
+  const orderedPrimaryRoutes = alias.loadBalancingEnabled
+    ? weightedRouteOrder(primaryRoutes)
+    : primaryRoutes
+  const routedModelIds = uniqueModelIds([...orderedPrimaryRoutes, ...fallbackRoutes])
+
+  if (routedModelIds.length === 0) {
+    throw createBadRequestError('Official provider alias has no enabled route', 'OFFICIAL_ALIAS_ROUTE_NOT_FOUND', {
+      surface: 'llm',
+      aliasId,
+    })
+  }
+
+  return { modelIds: routedModelIds }
+}
+
+async function routeChatAliasCandidates(input: {
+  deps: V1RouteDeps
+  body: Record<string, unknown>
+  modelIds: string[]
+  abortSignal?: AbortSignal
+}): Promise<{
+  modelId: string
+  response: Response
+  routeCtx: ReturnType<typeof newRouteContext>
+}> {
+  let lastError: unknown
+  for (const modelId of input.modelIds) {
+    const routeCtx = newRouteContext()
+    try {
+      const response = await input.deps.llmRouter.route({
+        modelName: modelId,
+        body: input.body,
+        headers: {},
+        abortSignal: input.abortSignal,
+      }, routeCtx)
+      return { modelId, response, routeCtx }
+    }
+    catch (err) {
+      if (input.abortSignal?.aborted)
+        throw err
+      lastError = err
+    }
+  }
+
+  throw lastError
+}
+
+function weightedRouteOrder(routes: OfficialProviderAliasRoute[]): OfficialProviderAliasRoute[] {
+  if (routes.length <= 1)
+    return routes
+
+  const totalWeight = routes.reduce((sum, route) => sum + Math.max(route.weight, 0), 0)
+  if (totalWeight <= 0)
+    return routes
+
+  let cursor = Math.random() * totalWeight
+  const selectedIndex = routes.findIndex((route) => {
+    cursor -= Math.max(route.weight, 0)
+    return cursor < 0
+  })
+  if (selectedIndex < 0)
+    return routes
+
+  const selected = routes[selectedIndex]
+  return [
+    selected,
+    ...routes.filter((_, index) => index !== selectedIndex),
+  ]
+}
+
+function uniqueModelIds(routes: OfficialProviderAliasRoute[]): string[] {
+  return Array.from(new Set(routes.map(route => route.routerModelId)))
 }

 function streamChatCompletion(input: {
@@ -694,6 +694,197 @@ describe('v1CompletionsRoutes', () => {
      expect(route).not.toHaveBeenCalled()
    })

+    it('falls back to the alias fallback pool when every primary route is exhausted', async () => {
+      const route = vi.fn(async ({ modelName }, ctx) => {
+        if (modelName === 'openai/primary')
+          throw new ApiError(502, 'BAD_GATEWAY', 'primary exhausted')
+        if (ctx) {
+          ctx.provider = 'openrouter'
+          ctx.upstreamModel = modelName
+        }
+        return new Response(JSON.stringify({ choices: [], usage: { prompt_tokens: 1, completion_tokens: 1 } }), {
+          status: 200,
+          headers: { 'Content-Type': 'application/json' },
+        })
+      })
+      const now = new Date()
+      const officialCatalogService = createMockOfficialCatalogService({
+        resolveEnabledAlias: vi.fn(async () => ({
+          id: 'alias-auto',
+          surface: 'llm' as const,
+          aliasId: 'auto',
+          displayName: 'Auto',
+          enabled: true,
+          displayOrder: 0,
+          fallbackEnabled: true,
+          loadBalancingEnabled: false,
+          createdAt: now,
+          updatedAt: now,
+          routes: [
+            { id: 'route-primary', aliasId: 'alias-auto', routerModelId: 'openai/primary', pool: 'primary' as const, enabled: true, weight: 1, displayOrder: 0, createdAt: now, updatedAt: now },
+            { id: 'route-fallback', aliasId: 'alias-auto', routerModelId: 'openai/fallback', pool: 'fallback' as const, enabled: true, weight: 1, displayOrder: 1, createdAt: now, updatedAt: now },
+          ],
+        })),
+      })
+      const app = createTestApp(
+        createMockFluxService(),
+        createMockConfigKV({
+          DEFAULT_CHAT_MODEL: 'openai/primary',
+          LLM_ROUTER_CONFIG: {
+            llm: { models: { 'openai/primary': { upstreams: [] }, 'openai/fallback': { upstreams: [] } } },
+            tts: { models: {} },
+          },
+        }),
+        undefined,
+        undefined,
+        undefined,
+        createMockLlmRouter({ route }),
+        createMockLlmTracing(),
+        createMockProductEventService(),
+        createMockVoicePackService(),
+        officialCatalogService,
+      )
+
+      const res = await app.fetch(
+        new Request('http://localhost/api/v1/openai/chat/completions', {
+          method: 'POST',
+          headers: { 'Content-Type': 'application/json' },
+          body: JSON.stringify({ model: 'auto', messages: [] }),
+        }),
+        { user: testUser } as any,
+      )
+
+      expect(res.status).toBe(200)
+      expect(route).toHaveBeenCalledTimes(2)
+      expect(route).toHaveBeenNthCalledWith(1, expect.objectContaining({ modelName: 'openai/primary' }), expect.any(Object))
+      expect(route).toHaveBeenNthCalledWith(2, expect.objectContaining({ modelName: 'openai/fallback' }), expect.any(Object))
+    })
+
+    it('does not use the alias fallback pool when fallback is disabled', async () => {
+      const route = vi.fn(async () => {
+        throw new ApiError(502, 'BAD_GATEWAY', 'primary exhausted')
+      })
+      const now = new Date()
+      const officialCatalogService = createMockOfficialCatalogService({
+        resolveEnabledAlias: vi.fn(async () => ({
+          id: 'alias-auto',
+          surface: 'llm' as const,
+          aliasId: 'auto',
+          displayName: 'Auto',
+          enabled: true,
+          displayOrder: 0,
+          fallbackEnabled: false,
+          loadBalancingEnabled: false,
+          createdAt: now,
+          updatedAt: now,
+          routes: [
+            { id: 'route-primary', aliasId: 'alias-auto', routerModelId: 'openai/primary', pool: 'primary' as const, enabled: true, weight: 1, displayOrder: 0, createdAt: now, updatedAt: now },
+            { id: 'route-fallback', aliasId: 'alias-auto', routerModelId: 'openai/fallback', pool: 'fallback' as const, enabled: true, weight: 1, displayOrder: 1, createdAt: now, updatedAt: now },
+          ],
+        })),
+      })
+      const app = createTestApp(
+        createMockFluxService(),
+        createMockConfigKV({
+          DEFAULT_CHAT_MODEL: 'openai/primary',
+          LLM_ROUTER_CONFIG: {
+            llm: { models: { 'openai/primary': { upstreams: [] }, 'openai/fallback': { upstreams: [] } } },
+            tts: { models: {} },
+          },
+        }),
+        undefined,
+        undefined,
+        undefined,
+        createMockLlmRouter({ route }),
+        createMockLlmTracing(),
+        createMockProductEventService(),
+        createMockVoicePackService(),
+        officialCatalogService,
+      )
+
+      const res = await app.fetch(
+        new Request('http://localhost/api/v1/openai/chat/completions', {
+          method: 'POST',
+          headers: { 'Content-Type': 'application/json' },
+          body: JSON.stringify({ model: 'auto', messages: [] }),
+        }),
+        { user: testUser } as any,
+      )
+
+      expect(res.status).toBe(502)
+      expect(route).toHaveBeenCalledTimes(1)
+      expect(route).toHaveBeenCalledWith(expect.objectContaining({ modelName: 'openai/primary' }), expect.any(Object))
+    })
+
+    it('uses weighted primary routing when alias load balancing is enabled', async () => {
+      const randomSpy = vi.spyOn(Math, 'random').mockReturnValue(0.95)
+      const route = vi.fn(async ({ modelName }, ctx) => {
+        if (ctx) {
+          ctx.provider = 'openrouter'
+          ctx.upstreamModel = modelName
+        }
+        return new Response(JSON.stringify({ choices: [], usage: { prompt_tokens: 1, completion_tokens: 1 } }), {
+          status: 200,
+          headers: { 'Content-Type': 'application/json' },
+        })
+      })
+      const now = new Date()
+      const officialCatalogService = createMockOfficialCatalogService({
+        resolveEnabledAlias: vi.fn(async () => ({
+          id: 'alias-auto',
+          surface: 'llm' as const,
+          aliasId: 'auto',
+          displayName: 'Auto',
+          enabled: true,
+          displayOrder: 0,
+          fallbackEnabled: false,
+          loadBalancingEnabled: true,
+          createdAt: now,
+          updatedAt: now,
+          routes: [
+            { id: 'route-a', aliasId: 'alias-auto', routerModelId: 'openai/light', pool: 'primary' as const, enabled: true, weight: 1, displayOrder: 0, createdAt: now, updatedAt: now },
+            { id: 'route-b', aliasId: 'alias-auto', routerModelId: 'openai/heavy', pool: 'primary' as const, enabled: true, weight: 9, displayOrder: 1, createdAt: now, updatedAt: now },
+          ],
+        })),
+      })
+      const app = createTestApp(
+        createMockFluxService(),
+        createMockConfigKV({
+          DEFAULT_CHAT_MODEL: 'openai/light',
+          LLM_ROUTER_CONFIG: {
+            llm: { models: { 'openai/light': { upstreams: [] }, 'openai/heavy': { upstreams: [] } } },
+            tts: { models: {} },
+          },
+        }),
+        undefined,
+        undefined,
+        undefined,
+        createMockLlmRouter({ route }),
+        createMockLlmTracing(),
+        createMockProductEventService(),
+        createMockVoicePackService(),
+        officialCatalogService,
+      )
+
+      try {
+        const res = await app.fetch(
+          new Request('http://localhost/api/v1/openai/chat/completions', {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/json' },
+            body: JSON.stringify({ model: 'auto', messages: [] }),
+          }),
+          { user: testUser } as any,
+        )
+
+        expect(res.status).toBe(200)
+        expect(route).toHaveBeenCalledTimes(1)
+        expect(route).toHaveBeenCalledWith(expect.objectContaining({ modelName: 'openai/heavy' }), expect.any(Object))
+      }
+      finally {
+        randomSpy.mockRestore()
+      }
+    })
+
    it('records Langfuse chat generation with the router-resolved upstream model', async () => {
      const llmRouter = createMockLlmRouter({
        route: vi.fn(async (_req, ctx) => {