nanakura/moeka-project
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

feat(server): apply official llm alias routing policy

Browse Source
This commit is contained in:
RainbowBird
2026-07-01 22:34:57 +08:00
parent 14fb183723
commit ed8c93c77b
2 changed files with 287 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/server/src/routes/openai/v1/operations/chat-completions/index.ts
+96 -7
View File
@@ -1,3 +1,4 @@
import type { OfficialProviderAliasRoute } from '../../../../../schemas/official-catalog'
import type { UsageInfo } from '../../../../../services/domain/billing/billing'
import type { GatewayCallback } from '../../gateway'
import type { V1RouteDeps } from '../../types'
@@ -5,6 +6,7 @@ import type { V1RouteDeps } from '../../types'
import { useLogger } from '@guiiai/logg'
import { extractUsageFromBody } from '../../../../../services/domain/billing/billing'
import { createBadRequestError } from '../../../../../utils/error'
import { nanoid } from '../../../../../utils/id'
import { buildSafeResponseHeaders } from '../../http/response'
import { createOpenAiRouteBilling } from '../../middlewares/billing'
@@ -41,7 +43,8 @@ export function chatCompletions(deps: V1RouteDeps): GatewayCallback<'chat.comple
const body = input.body
const requestedAlias = typeof body.model === 'string' && body.model.length > 0 ? body.model : 'auto'
const requestModel = await resolveChatModelAlias(deps, requestedAlias)
const aliasPlan = await resolveChatModelAliasPlan(deps, requestedAlias)
let requestModel = aliasPlan.modelIds[0]
const stream = !!body.stream
logger.withFields({
@@ -81,11 +84,19 @@ export function chatCompletions(deps: V1RouteDeps): GatewayCallback<'chat.comple
// fluxConsumed: 0 while real cost was incurred — a silent revenue leak.
// Source: codex review 2026-05-15 HIGH #1.
const clientAbort = input.abortSignal
const routeCtx = newRouteContext()
let routeCtx = newRouteContext()
let response: Response
try {
response = await telemetry.runWithSpan(span, () =>
deps.llmRouter.route({ modelName: requestModel, body, headers: {}, abortSignal: clientAbort }, routeCtx))
const routed = await telemetry.runWithSpan(span, () =>
routeChatAliasCandidates({
deps,
body,
modelIds: aliasPlan.modelIds,
abortSignal: clientAbort,
}))
response = routed.response
routeCtx = routed.routeCtx
requestModel = routed.modelId
}
catch (err) {
telemetry.failSpan(span, 'Router exhausted or unknown model')
@@ -198,7 +209,11 @@ export function chatCompletions(deps: V1RouteDeps): GatewayCallback<'chat.comple
}
}
async function resolveChatModelAlias(deps: V1RouteDeps, aliasId: string): Promise<string> {
interface ChatModelAliasPlan {
modelIds: string[]
}
async function resolveChatModelAliasPlan(deps: V1RouteDeps, aliasId: string): Promise<ChatModelAliasPlan> {
const config = await deps.configKV.getOrThrow('LLM_ROUTER_CONFIG')
const defaultModel = await deps.configKV.getOrThrow('DEFAULT_CHAT_MODEL')
const modelIds = [
@@ -211,8 +226,82 @@ async function resolveChatModelAlias(deps: V1RouteDeps, aliasId: string): Promis
})
const alias = await deps.officialCatalogService.resolveEnabledAlias('llm', aliasId)
const primary = alias.routes.find(route => route.pool === 'primary')
return (primary ?? alias.routes[0]).routerModelId
const primaryRoutes = alias.routes.filter(route => route.pool === 'primary')
const fallbackRoutes = alias.fallbackEnabled
? alias.routes.filter(route => route.pool === 'fallback')
: []
const orderedPrimaryRoutes = alias.loadBalancingEnabled
? weightedRouteOrder(primaryRoutes)
: primaryRoutes
const routedModelIds = uniqueModelIds([...orderedPrimaryRoutes, ...fallbackRoutes])
if (routedModelIds.length === 0) {
throw createBadRequestError('Official provider alias has no enabled route', 'OFFICIAL_ALIAS_ROUTE_NOT_FOUND', {
surface: 'llm',
aliasId,
})
}
return { modelIds: routedModelIds }
}
async function routeChatAliasCandidates(input: {
deps: V1RouteDeps
body: Record<string, unknown>
modelIds: string[]
abortSignal?: AbortSignal
}): Promise<{
modelId: string
response: Response
routeCtx: ReturnType<typeof newRouteContext>
}> {
let lastError: unknown
for (const modelId of input.modelIds) {
const routeCtx = newRouteContext()
try {
const response = await input.deps.llmRouter.route({
modelName: modelId,
body: input.body,
headers: {},
abortSignal: input.abortSignal,
}, routeCtx)
return { modelId, response, routeCtx }
}
catch (err) {
if (input.abortSignal?.aborted)
throw err
lastError = err
}
}
throw lastError
}
function weightedRouteOrder(routes: OfficialProviderAliasRoute[]): OfficialProviderAliasRoute[] {
if (routes.length <= 1)
return routes
const totalWeight = routes.reduce((sum, route) => sum + Math.max(route.weight, 0), 0)
if (totalWeight <= 0)
return routes
let cursor = Math.random() * totalWeight
const selectedIndex = routes.findIndex((route) => {
cursor -= Math.max(route.weight, 0)
return cursor < 0
})
if (selectedIndex < 0)
return routes
const selected = routes[selectedIndex]
return [
selected,
...routes.filter((_, index) => index !== selectedIndex),
]
}
function uniqueModelIds(routes: OfficialProviderAliasRoute[]): string[] {
return Array.from(new Set(routes.map(route => route.routerModelId)))
}
function streamChatCompletion(input: {
apps/server/src/routes/openai/v1/route.test.ts
+191
View File
@@ -694,6 +694,197 @@ describe('v1CompletionsRoutes', () => {
expect(route).not.toHaveBeenCalled()
})
it('falls back to the alias fallback pool when every primary route is exhausted', async () => {
const route = vi.fn(async ({ modelName }, ctx) => {
if (modelName === 'openai/primary')
throw new ApiError(502, 'BAD_GATEWAY', 'primary exhausted')
if (ctx) {
ctx.provider = 'openrouter'
ctx.upstreamModel = modelName
}
return new Response(JSON.stringify({ choices: [], usage: { prompt_tokens: 1, completion_tokens: 1 } }), {
status: 200,
headers: { 'Content-Type': 'application/json' },
})
})
const now = new Date()
const officialCatalogService = createMockOfficialCatalogService({
resolveEnabledAlias: vi.fn(async () => ({
id: 'alias-auto',
surface: 'llm' as const,
aliasId: 'auto',
displayName: 'Auto',
enabled: true,
displayOrder: 0,
fallbackEnabled: true,
loadBalancingEnabled: false,
createdAt: now,
updatedAt: now,
routes: [
{ id: 'route-primary', aliasId: 'alias-auto', routerModelId: 'openai/primary', pool: 'primary' as const, enabled: true, weight: 1, displayOrder: 0, createdAt: now, updatedAt: now },
{ id: 'route-fallback', aliasId: 'alias-auto', routerModelId: 'openai/fallback', pool: 'fallback' as const, enabled: true, weight: 1, displayOrder: 1, createdAt: now, updatedAt: now },
],
})),
})
const app = createTestApp(
createMockFluxService(),
createMockConfigKV({
DEFAULT_CHAT_MODEL: 'openai/primary',
LLM_ROUTER_CONFIG: {
llm: { models: { 'openai/primary': { upstreams: [] }, 'openai/fallback': { upstreams: [] } } },
tts: { models: {} },
},
}),
undefined,
undefined,
undefined,
createMockLlmRouter({ route }),
createMockLlmTracing(),
createMockProductEventService(),
createMockVoicePackService(),
officialCatalogService,
)
const res = await app.fetch(
new Request('http://localhost/api/v1/openai/chat/completions', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ model: 'auto', messages: [] }),
}),
{ user: testUser } as any,
)
expect(res.status).toBe(200)
expect(route).toHaveBeenCalledTimes(2)
expect(route).toHaveBeenNthCalledWith(1, expect.objectContaining({ modelName: 'openai/primary' }), expect.any(Object))
expect(route).toHaveBeenNthCalledWith(2, expect.objectContaining({ modelName: 'openai/fallback' }), expect.any(Object))
})
it('does not use the alias fallback pool when fallback is disabled', async () => {
const route = vi.fn(async () => {
throw new ApiError(502, 'BAD_GATEWAY', 'primary exhausted')
})
const now = new Date()
const officialCatalogService = createMockOfficialCatalogService({
resolveEnabledAlias: vi.fn(async () => ({
id: 'alias-auto',
surface: 'llm' as const,
aliasId: 'auto',
displayName: 'Auto',
enabled: true,
displayOrder: 0,
fallbackEnabled: false,
loadBalancingEnabled: false,
createdAt: now,
updatedAt: now,
routes: [
{ id: 'route-primary', aliasId: 'alias-auto', routerModelId: 'openai/primary', pool: 'primary' as const, enabled: true, weight: 1, displayOrder: 0, createdAt: now, updatedAt: now },
{ id: 'route-fallback', aliasId: 'alias-auto', routerModelId: 'openai/fallback', pool: 'fallback' as const, enabled: true, weight: 1, displayOrder: 1, createdAt: now, updatedAt: now },
],
})),
})
const app = createTestApp(
createMockFluxService(),
createMockConfigKV({
DEFAULT_CHAT_MODEL: 'openai/primary',
LLM_ROUTER_CONFIG: {
llm: { models: { 'openai/primary': { upstreams: [] }, 'openai/fallback': { upstreams: [] } } },
tts: { models: {} },
},
}),
undefined,
undefined,
undefined,
createMockLlmRouter({ route }),
createMockLlmTracing(),
createMockProductEventService(),
createMockVoicePackService(),
officialCatalogService,
)
const res = await app.fetch(
new Request('http://localhost/api/v1/openai/chat/completions', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ model: 'auto', messages: [] }),
}),
{ user: testUser } as any,
)
expect(res.status).toBe(502)
expect(route).toHaveBeenCalledTimes(1)
expect(route).toHaveBeenCalledWith(expect.objectContaining({ modelName: 'openai/primary' }), expect.any(Object))
})
it('uses weighted primary routing when alias load balancing is enabled', async () => {
const randomSpy = vi.spyOn(Math, 'random').mockReturnValue(0.95)
const route = vi.fn(async ({ modelName }, ctx) => {
if (ctx) {
ctx.provider = 'openrouter'
ctx.upstreamModel = modelName
}
return new Response(JSON.stringify({ choices: [], usage: { prompt_tokens: 1, completion_tokens: 1 } }), {
status: 200,
headers: { 'Content-Type': 'application/json' },
})
})
const now = new Date()
const officialCatalogService = createMockOfficialCatalogService({
resolveEnabledAlias: vi.fn(async () => ({
id: 'alias-auto',
surface: 'llm' as const,
aliasId: 'auto',
displayName: 'Auto',
enabled: true,
displayOrder: 0,
fallbackEnabled: false,
loadBalancingEnabled: true,
createdAt: now,
updatedAt: now,
routes: [
{ id: 'route-a', aliasId: 'alias-auto', routerModelId: 'openai/light', pool: 'primary' as const, enabled: true, weight: 1, displayOrder: 0, createdAt: now, updatedAt: now },
{ id: 'route-b', aliasId: 'alias-auto', routerModelId: 'openai/heavy', pool: 'primary' as const, enabled: true, weight: 9, displayOrder: 1, createdAt: now, updatedAt: now },
],
})),
})
const app = createTestApp(
createMockFluxService(),
createMockConfigKV({
DEFAULT_CHAT_MODEL: 'openai/light',
LLM_ROUTER_CONFIG: {
llm: { models: { 'openai/light': { upstreams: [] }, 'openai/heavy': { upstreams: [] } } },
tts: { models: {} },
},
}),
undefined,
undefined,
undefined,
createMockLlmRouter({ route }),
createMockLlmTracing(),
createMockProductEventService(),
createMockVoicePackService(),
officialCatalogService,
)
try {
const res = await app.fetch(
new Request('http://localhost/api/v1/openai/chat/completions', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ model: 'auto', messages: [] }),
}),
{ user: testUser } as any,
)
expect(res.status).toBe(200)
expect(route).toHaveBeenCalledTimes(1)
expect(route).toHaveBeenCalledWith(expect.objectContaining({ modelName: 'openai/heavy' }), expect.any(Object))
}
finally {
randomSpy.mockRestore()
}
})
it('records Langfuse chat generation with the router-resolved upstream model', async () => {
const llmRouter = createMockLlmRouter({
route: vi.fn(async (_req, ctx) => {