From ed8c93c77baf94eda37b6537e85091898557b259 Mon Sep 17 00:00:00 2001 From: RainbowBird Date: Wed, 1 Jul 2026 22:32:40 +0800 Subject: [PATCH] feat(server): apply official llm alias routing policy --- .../v1/operations/chat-completions/index.ts | 103 +++++++++- .../server/src/routes/openai/v1/route.test.ts | 191 ++++++++++++++++++ 2 files changed, 287 insertions(+), 7 deletions(-) diff --git a/apps/server/src/routes/openai/v1/operations/chat-completions/index.ts b/apps/server/src/routes/openai/v1/operations/chat-completions/index.ts index 84036997e..7285bf726 100644 --- a/apps/server/src/routes/openai/v1/operations/chat-completions/index.ts +++ b/apps/server/src/routes/openai/v1/operations/chat-completions/index.ts @@ -1,3 +1,4 @@ +import type { OfficialProviderAliasRoute } from '../../../../../schemas/official-catalog' import type { UsageInfo } from '../../../../../services/domain/billing/billing' import type { GatewayCallback } from '../../gateway' import type { V1RouteDeps } from '../../types' @@ -5,6 +6,7 @@ import type { V1RouteDeps } from '../../types' import { useLogger } from '@guiiai/logg' import { extractUsageFromBody } from '../../../../../services/domain/billing/billing' +import { createBadRequestError } from '../../../../../utils/error' import { nanoid } from '../../../../../utils/id' import { buildSafeResponseHeaders } from '../../http/response' import { createOpenAiRouteBilling } from '../../middlewares/billing' @@ -41,7 +43,8 @@ export function chatCompletions(deps: V1RouteDeps): GatewayCallback<'chat.comple const body = input.body const requestedAlias = typeof body.model === 'string' && body.model.length > 0 ? body.model : 'auto' - const requestModel = await resolveChatModelAlias(deps, requestedAlias) + const aliasPlan = await resolveChatModelAliasPlan(deps, requestedAlias) + let requestModel = aliasPlan.modelIds[0] const stream = !!body.stream logger.withFields({ @@ -81,11 +84,19 @@ export function chatCompletions(deps: V1RouteDeps): GatewayCallback<'chat.comple // fluxConsumed: 0 while real cost was incurred — a silent revenue leak. // Source: codex review 2026-05-15 HIGH #1. const clientAbort = input.abortSignal - const routeCtx = newRouteContext() + let routeCtx = newRouteContext() let response: Response try { - response = await telemetry.runWithSpan(span, () => - deps.llmRouter.route({ modelName: requestModel, body, headers: {}, abortSignal: clientAbort }, routeCtx)) + const routed = await telemetry.runWithSpan(span, () => + routeChatAliasCandidates({ + deps, + body, + modelIds: aliasPlan.modelIds, + abortSignal: clientAbort, + })) + response = routed.response + routeCtx = routed.routeCtx + requestModel = routed.modelId } catch (err) { telemetry.failSpan(span, 'Router exhausted or unknown model') @@ -198,7 +209,11 @@ export function chatCompletions(deps: V1RouteDeps): GatewayCallback<'chat.comple } } -async function resolveChatModelAlias(deps: V1RouteDeps, aliasId: string): Promise { +interface ChatModelAliasPlan { + modelIds: string[] +} + +async function resolveChatModelAliasPlan(deps: V1RouteDeps, aliasId: string): Promise { const config = await deps.configKV.getOrThrow('LLM_ROUTER_CONFIG') const defaultModel = await deps.configKV.getOrThrow('DEFAULT_CHAT_MODEL') const modelIds = [ @@ -211,8 +226,82 @@ async function resolveChatModelAlias(deps: V1RouteDeps, aliasId: string): Promis }) const alias = await deps.officialCatalogService.resolveEnabledAlias('llm', aliasId) - const primary = alias.routes.find(route => route.pool === 'primary') - return (primary ?? alias.routes[0]).routerModelId + const primaryRoutes = alias.routes.filter(route => route.pool === 'primary') + const fallbackRoutes = alias.fallbackEnabled + ? alias.routes.filter(route => route.pool === 'fallback') + : [] + const orderedPrimaryRoutes = alias.loadBalancingEnabled + ? weightedRouteOrder(primaryRoutes) + : primaryRoutes + const routedModelIds = uniqueModelIds([...orderedPrimaryRoutes, ...fallbackRoutes]) + + if (routedModelIds.length === 0) { + throw createBadRequestError('Official provider alias has no enabled route', 'OFFICIAL_ALIAS_ROUTE_NOT_FOUND', { + surface: 'llm', + aliasId, + }) + } + + return { modelIds: routedModelIds } +} + +async function routeChatAliasCandidates(input: { + deps: V1RouteDeps + body: Record + modelIds: string[] + abortSignal?: AbortSignal +}): Promise<{ + modelId: string + response: Response + routeCtx: ReturnType +}> { + let lastError: unknown + for (const modelId of input.modelIds) { + const routeCtx = newRouteContext() + try { + const response = await input.deps.llmRouter.route({ + modelName: modelId, + body: input.body, + headers: {}, + abortSignal: input.abortSignal, + }, routeCtx) + return { modelId, response, routeCtx } + } + catch (err) { + if (input.abortSignal?.aborted) + throw err + lastError = err + } + } + + throw lastError +} + +function weightedRouteOrder(routes: OfficialProviderAliasRoute[]): OfficialProviderAliasRoute[] { + if (routes.length <= 1) + return routes + + const totalWeight = routes.reduce((sum, route) => sum + Math.max(route.weight, 0), 0) + if (totalWeight <= 0) + return routes + + let cursor = Math.random() * totalWeight + const selectedIndex = routes.findIndex((route) => { + cursor -= Math.max(route.weight, 0) + return cursor < 0 + }) + if (selectedIndex < 0) + return routes + + const selected = routes[selectedIndex] + return [ + selected, + ...routes.filter((_, index) => index !== selectedIndex), + ] +} + +function uniqueModelIds(routes: OfficialProviderAliasRoute[]): string[] { + return Array.from(new Set(routes.map(route => route.routerModelId))) } function streamChatCompletion(input: { diff --git a/apps/server/src/routes/openai/v1/route.test.ts b/apps/server/src/routes/openai/v1/route.test.ts index 7a2949d7d..359b6741f 100644 --- a/apps/server/src/routes/openai/v1/route.test.ts +++ b/apps/server/src/routes/openai/v1/route.test.ts @@ -694,6 +694,197 @@ describe('v1CompletionsRoutes', () => { expect(route).not.toHaveBeenCalled() }) + it('falls back to the alias fallback pool when every primary route is exhausted', async () => { + const route = vi.fn(async ({ modelName }, ctx) => { + if (modelName === 'openai/primary') + throw new ApiError(502, 'BAD_GATEWAY', 'primary exhausted') + if (ctx) { + ctx.provider = 'openrouter' + ctx.upstreamModel = modelName + } + return new Response(JSON.stringify({ choices: [], usage: { prompt_tokens: 1, completion_tokens: 1 } }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }) + }) + const now = new Date() + const officialCatalogService = createMockOfficialCatalogService({ + resolveEnabledAlias: vi.fn(async () => ({ + id: 'alias-auto', + surface: 'llm' as const, + aliasId: 'auto', + displayName: 'Auto', + enabled: true, + displayOrder: 0, + fallbackEnabled: true, + loadBalancingEnabled: false, + createdAt: now, + updatedAt: now, + routes: [ + { id: 'route-primary', aliasId: 'alias-auto', routerModelId: 'openai/primary', pool: 'primary' as const, enabled: true, weight: 1, displayOrder: 0, createdAt: now, updatedAt: now }, + { id: 'route-fallback', aliasId: 'alias-auto', routerModelId: 'openai/fallback', pool: 'fallback' as const, enabled: true, weight: 1, displayOrder: 1, createdAt: now, updatedAt: now }, + ], + })), + }) + const app = createTestApp( + createMockFluxService(), + createMockConfigKV({ + DEFAULT_CHAT_MODEL: 'openai/primary', + LLM_ROUTER_CONFIG: { + llm: { models: { 'openai/primary': { upstreams: [] }, 'openai/fallback': { upstreams: [] } } }, + tts: { models: {} }, + }, + }), + undefined, + undefined, + undefined, + createMockLlmRouter({ route }), + createMockLlmTracing(), + createMockProductEventService(), + createMockVoicePackService(), + officialCatalogService, + ) + + const res = await app.fetch( + new Request('http://localhost/api/v1/openai/chat/completions', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ model: 'auto', messages: [] }), + }), + { user: testUser } as any, + ) + + expect(res.status).toBe(200) + expect(route).toHaveBeenCalledTimes(2) + expect(route).toHaveBeenNthCalledWith(1, expect.objectContaining({ modelName: 'openai/primary' }), expect.any(Object)) + expect(route).toHaveBeenNthCalledWith(2, expect.objectContaining({ modelName: 'openai/fallback' }), expect.any(Object)) + }) + + it('does not use the alias fallback pool when fallback is disabled', async () => { + const route = vi.fn(async () => { + throw new ApiError(502, 'BAD_GATEWAY', 'primary exhausted') + }) + const now = new Date() + const officialCatalogService = createMockOfficialCatalogService({ + resolveEnabledAlias: vi.fn(async () => ({ + id: 'alias-auto', + surface: 'llm' as const, + aliasId: 'auto', + displayName: 'Auto', + enabled: true, + displayOrder: 0, + fallbackEnabled: false, + loadBalancingEnabled: false, + createdAt: now, + updatedAt: now, + routes: [ + { id: 'route-primary', aliasId: 'alias-auto', routerModelId: 'openai/primary', pool: 'primary' as const, enabled: true, weight: 1, displayOrder: 0, createdAt: now, updatedAt: now }, + { id: 'route-fallback', aliasId: 'alias-auto', routerModelId: 'openai/fallback', pool: 'fallback' as const, enabled: true, weight: 1, displayOrder: 1, createdAt: now, updatedAt: now }, + ], + })), + }) + const app = createTestApp( + createMockFluxService(), + createMockConfigKV({ + DEFAULT_CHAT_MODEL: 'openai/primary', + LLM_ROUTER_CONFIG: { + llm: { models: { 'openai/primary': { upstreams: [] }, 'openai/fallback': { upstreams: [] } } }, + tts: { models: {} }, + }, + }), + undefined, + undefined, + undefined, + createMockLlmRouter({ route }), + createMockLlmTracing(), + createMockProductEventService(), + createMockVoicePackService(), + officialCatalogService, + ) + + const res = await app.fetch( + new Request('http://localhost/api/v1/openai/chat/completions', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ model: 'auto', messages: [] }), + }), + { user: testUser } as any, + ) + + expect(res.status).toBe(502) + expect(route).toHaveBeenCalledTimes(1) + expect(route).toHaveBeenCalledWith(expect.objectContaining({ modelName: 'openai/primary' }), expect.any(Object)) + }) + + it('uses weighted primary routing when alias load balancing is enabled', async () => { + const randomSpy = vi.spyOn(Math, 'random').mockReturnValue(0.95) + const route = vi.fn(async ({ modelName }, ctx) => { + if (ctx) { + ctx.provider = 'openrouter' + ctx.upstreamModel = modelName + } + return new Response(JSON.stringify({ choices: [], usage: { prompt_tokens: 1, completion_tokens: 1 } }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }) + }) + const now = new Date() + const officialCatalogService = createMockOfficialCatalogService({ + resolveEnabledAlias: vi.fn(async () => ({ + id: 'alias-auto', + surface: 'llm' as const, + aliasId: 'auto', + displayName: 'Auto', + enabled: true, + displayOrder: 0, + fallbackEnabled: false, + loadBalancingEnabled: true, + createdAt: now, + updatedAt: now, + routes: [ + { id: 'route-a', aliasId: 'alias-auto', routerModelId: 'openai/light', pool: 'primary' as const, enabled: true, weight: 1, displayOrder: 0, createdAt: now, updatedAt: now }, + { id: 'route-b', aliasId: 'alias-auto', routerModelId: 'openai/heavy', pool: 'primary' as const, enabled: true, weight: 9, displayOrder: 1, createdAt: now, updatedAt: now }, + ], + })), + }) + const app = createTestApp( + createMockFluxService(), + createMockConfigKV({ + DEFAULT_CHAT_MODEL: 'openai/light', + LLM_ROUTER_CONFIG: { + llm: { models: { 'openai/light': { upstreams: [] }, 'openai/heavy': { upstreams: [] } } }, + tts: { models: {} }, + }, + }), + undefined, + undefined, + undefined, + createMockLlmRouter({ route }), + createMockLlmTracing(), + createMockProductEventService(), + createMockVoicePackService(), + officialCatalogService, + ) + + try { + const res = await app.fetch( + new Request('http://localhost/api/v1/openai/chat/completions', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ model: 'auto', messages: [] }), + }), + { user: testUser } as any, + ) + + expect(res.status).toBe(200) + expect(route).toHaveBeenCalledTimes(1) + expect(route).toHaveBeenCalledWith(expect.objectContaining({ modelName: 'openai/heavy' }), expect.any(Object)) + } + finally { + randomSpy.mockRestore() + } + }) + it('records Langfuse chat generation with the router-resolved upstream model', async () => { const llmRouter = createMockLlmRouter({ route: vi.fn(async (_req, ctx) => {