nanakura/moeka-project
1
0
Fork 0
Code Issues Pull Requests 1 Actions 3 Packages Projects Releases Wiki Activity

feat(server): add gateway and model configuration to environment variables

Browse Source
This commit is contained in:
RainbowBird
2026-04-17 02:59:14 +08:00
committed by RainbowBird
parent a11727020a
commit 57533044f7
8 changed files with 94 additions and 107 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/server/.env
+4
View File
@@ -16,3 +16,7 @@ CLIENT_URL=""
API_SERVER_URL=""
# OTEL_EXPORTER_OTLP_ENDPOINT="http://localhost:4318"
GATEWAY_BASE_URL="http://localhost:18080"
DEFAULT_CHAT_MODEL="openai/gpt-5-mini"
DEFAULT_TTS_MODEL="microsoft/v1"
apps/server/src/app.ts
+1 -1
View File
@@ -191,7 +191,7 @@ export async function buildApp(deps: AppDeps) {
/**
* V1 routes for official provider.
*/
.route('/api/v1/openai', createV1CompletionsRoutes(deps.fluxService, deps.billingService, deps.configKV, deps.billingMq, deps.ttsMeter, deps.otel?.genAi))
.route('/api/v1/openai', createV1CompletionsRoutes(deps.fluxService, deps.billingService, deps.configKV, deps.billingMq, deps.ttsMeter, deps.env, deps.otel?.genAi))
/**
* Flux routes.
apps/server/src/libs/env.ts
+5
View File
@@ -56,6 +56,11 @@ const EnvSchema = object({
STRIPE_SECRET_KEY: optional(string()),
STRIPE_WEBHOOK_SECRET: optional(string()),
// LLM gateway (infrastructure config — baked per deployment)
GATEWAY_BASE_URL: pipe(string(), nonEmpty('GATEWAY_BASE_URL is required')),
DEFAULT_CHAT_MODEL: pipe(string(), nonEmpty('DEFAULT_CHAT_MODEL is required')),
DEFAULT_TTS_MODEL: pipe(string(), nonEmpty('DEFAULT_TTS_MODEL is required')),
BILLING_EVENTS_STREAM: optional(string(), DEFAULT_BILLING_EVENTS_STREAM),
BILLING_EVENTS_CONSUMER_NAME: optional(string()),
BILLING_EVENTS_BATCH_SIZE: optionalIntegerFromString(10, 'BILLING_EVENTS_BATCH_SIZE', 1),
apps/server/src/routes/openai/v1/index.ts
+25 -66
View File
@@ -1,5 +1,6 @@
import type { Context } from 'hono'
import type { Env } from '../../../libs/env'
import type { MqService } from '../../../libs/mq'
import type { GenAiMetrics } from '../../../libs/otel'
import type { UsageInfo } from '../../../services/billing/billing'
@@ -70,7 +71,7 @@ function getLlmMetricAttributes(opts: { model: string, type: string, status: num
}
}
export function createV1CompletionsRoutes(fluxService: FluxService, billingService: BillingService, configKV: ConfigKVService, billingMq: MqService<BillingEvent>, ttsMeter: FluxMeter, genAi?: GenAiMetrics | null) {
export function createV1CompletionsRoutes(fluxService: FluxService, billingService: BillingService, configKV: ConfigKVService, billingMq: MqService<BillingEvent>, ttsMeter: FluxMeter, env: Env, genAi?: GenAiMetrics | null) {
const logger = useLogger('v1-completions').useGlobalConfig()
// TODO: Extract this compat route into smaller facades/modules.
// It currently mixes auth, rate limiting, proxying, billing, telemetry, and event publishing in one transport layer entrypoint.
@@ -121,13 +122,12 @@ export function createV1CompletionsRoutes(fluxService: FluxService, billingServi
}
const body = await c.req.json()
const gatewayBaseUrl = await configKV.getOrThrow('GATEWAY_BASE_URL')
const baseUrl = normalizeBaseUrl(gatewayBaseUrl)
const baseUrl = normalizeBaseUrl(env.GATEWAY_BASE_URL)
const serverAttributes = getServerConnectionAttributes(baseUrl)
let requestModel = body.model || 'auto'
if (requestModel === 'auto') {
requestModel = await configKV.getOrThrow('DEFAULT_CHAT_MODEL')
requestModel = env.DEFAULT_CHAT_MODEL
}
const span = tracer.startSpan('llm.gateway.chat', {
@@ -324,19 +324,15 @@ export function createV1CompletionsRoutes(fluxService: FluxService, billingServi
// throw createPaymentRequiredError('Insufficient flux')
// }
// const body = await c.req.json()
// const gatewayBaseUrl = await configKV.getOrThrow('GATEWAY_BASE_URL')
// const baseUrl = normalizeBaseUrl(gatewayBaseUrl)
// const serverAttributes = getServerConnectionAttributes(baseUrl)
// const requestModel = body.model || 'auto'
const body = await c.req.json()
const baseUrl = normalizeBaseUrl(env.GATEWAY_BASE_URL)
const serverAttributes = getServerConnectionAttributes(baseUrl)
let requestModel = body.model || 'auto'
const inputText: string = body.input ?? ''
// const span = tracer.startSpan('llm.gateway.tts', {
// attributes: {
// [GEN_AI_ATTR_REQUEST_MODEL]: requestModel,
// [AIRI_ATTR_GEN_AI_OPERATION_KIND]: 'text_to_speech',
// ...serverAttributes,
// },
// })
if (requestModel === 'auto') {
requestModel = env.DEFAULT_TTS_MODEL
}
// Pre-flight: refuse before hitting upstream if this segment would push the
// user past their balance. Cheap-path requests below the Flux threshold
@@ -407,13 +403,8 @@ export function createV1CompletionsRoutes(fluxService: FluxService, billingServi
// const baseUrl = normalizeBaseUrl(gatewayBaseUrl)
// const serverAttributes = getServerConnectionAttributes(baseUrl)
// const span = tracer.startSpan('llm.gateway.asr', {
// attributes: {
// [GEN_AI_ATTR_REQUEST_MODEL]: 'auto',
// [AIRI_ATTR_GEN_AI_OPERATION_KIND]: 'speech_to_text',
// ...serverAttributes,
// },
// })
async function handleListVoices(_c: Context<HonoEnv>) {
const baseUrl = normalizeBaseUrl(env.GATEWAY_BASE_URL)
// const startedAt = Date.now()
@@ -427,48 +418,15 @@ export function createV1CompletionsRoutes(fluxService: FluxService, billingServi
// body: rawBody,
// }))
// const durationMs = Date.now() - startedAt
// span.setAttribute('http.response.status_code', response.status)
async function handleListTTSModels(_c: Context<HonoEnv>) {
const model = env.DEFAULT_TTS_MODEL
return Response.json({
models: [{ id: model, name: model }],
})
}
// if (!response.ok) {
// span.setStatus({ code: SpanStatusCode.ERROR, message: `Gateway ${response.status}` })
// span.end()
// recordMetrics({ model: 'auto', status: response.status, type: 'asr', durationMs, fluxConsumed: 0 })
// return new Response(response.body, {
// status: response.status,
// headers: buildSafeResponseHeaders(response),
// })
// }
// const fluxPerRequest = await configKV.getOrThrow('FLUX_PER_REQUEST_ASR')
// await billingService.consumeFluxForLLM({
// userId: user.id,
// amount: fluxPerRequest,
// requestId: nanoid(),
// description: `asr:auto`,
// })
// span.setAttribute(AIRI_ATTR_BILLING_FLUX_CONSUMED, fluxPerRequest)
// span.end()
// recordMetrics({ model: 'auto', status: response.status, type: 'asr', durationMs, fluxConsumed: fluxPerRequest })
// publishRequestLog({
// userId: user.id,
// model: 'auto',
// status: response.status,
// durationMs,
// fluxConsumed: fluxPerRequest,
// })
// return new Response(response.body, {
// status: response.status,
// headers: buildSafeResponseHeaders(response),
// })
// }
const chatGuard = configGuard(configKV, ['FLUX_PER_REQUEST', 'GATEWAY_BASE_URL', 'DEFAULT_CHAT_MODEL'], 'Service is not available yet')
// const ttsGuard = configGuard(configKV, ['FLUX_PER_REQUEST_TTS', 'GATEWAY_BASE_URL'], 'TTS service is not available yet')
// const asrGuard = configGuard(configKV, ['FLUX_PER_REQUEST_ASR', 'GATEWAY_BASE_URL'], 'ASR service is not available yet')
const chatGuard = configGuard(configKV, ['FLUX_PER_REQUEST'], 'Service is not available yet')
const ttsGuard = configGuard(configKV, ['FLUX_PER_1K_CHARS_TTS'], 'TTS service is not available yet')
// 60 requests per minute per user for LLM completions
const completionsRateLimit = rateLimiter({ max: 60, windowSec: 60 })
@@ -477,6 +435,7 @@ export function createV1CompletionsRoutes(fluxService: FluxService, billingServi
.use('*', authGuard)
.post('/chat/completions', completionsRateLimit, chatGuard, handleCompletion)
.post('/chat/completion', completionsRateLimit, chatGuard, handleCompletion)
// .post('/audio/speech', ttsGuard, handleTTS)
// .post('/audio/transcriptions', bodyLimit({ maxSize: 25 * 1024 * 1024 }), asrGuard, handleTranscription)
.post('/audio/speech', ttsGuard, handleTTS)
.get('/audio/voices', handleListVoices)
.get('/audio/models', handleListTTSModels)
}
apps/server/src/routes/openai/v1/route.test.ts
+53 -32
View File
@@ -1,3 +1,4 @@
import type { Env } from '../../../libs/env'
import type { MqService } from '../../../libs/mq'
import type { BillingEvent } from '../../../services/billing/billing-events'
import type { BillingService } from '../../../services/billing/billing-service'
@@ -34,13 +35,20 @@ function createMockBillingService(flux = 100): BillingService {
} as any
}
function createMockEnv(overrides: Partial<Env> = {}): Env {
return {
GATEWAY_BASE_URL: 'http://mock-gateway/',
DEFAULT_CHAT_MODEL: 'openai/gpt-5-mini',
DEFAULT_TTS_MODEL: 'tts-1',
...overrides,
} as Env
}
function createMockConfigKV(overrides: Record<string, any> = {}): ConfigKVService {
const defaults: Record<string, any> = {
FLUX_PER_REQUEST: 1,
FLUX_PER_1K_CHARS_TTS: 2,
TTS_DEBT_TTL_SECONDS: 86400,
GATEWAY_BASE_URL: 'http://mock-gateway/',
DEFAULT_CHAT_MODEL: 'openai/gpt-5-mini',
...overrides,
}
return {
@@ -87,6 +95,7 @@ function createTestApp(
billingService?: BillingService,
billingMq?: MqService<BillingEvent>,
ttsMeter?: ReturnType<typeof createMockTtsMeter>,
env?: Env,
) {
const routes = createV1CompletionsRoutes(
fluxService,
@@ -94,6 +103,7 @@ function createTestApp(
configKV,
billingMq ?? createMockBillingMq(),
ttsMeter ?? createMockTtsMeter(),
env ?? createMockEnv(),
null,
)
const app = new Hono<HonoEnv>()
@@ -174,7 +184,7 @@ describe('v1CompletionsRoutes', () => {
const fluxService = createMockFluxService(100)
const billingService = createMockBillingService(100)
const configKV = createMockConfigKV({ GATEWAY_BASE_URL: 'http://mock-gateway/' })
const configKV = createMockConfigKV()
const app = createTestApp(fluxService, configKV, billingService)
const res = await app.fetch(
@@ -213,7 +223,11 @@ describe('v1CompletionsRoutes', () => {
const app = createTestApp(
createMockFluxService(),
createMockConfigKV({ DEFAULT_CHAT_MODEL: 'anthropic/claude-sonnet' }),
createMockConfigKV(),
undefined,
undefined,
undefined,
createMockEnv({ DEFAULT_CHAT_MODEL: 'anthropic/claude-sonnet' }),
)
await app.fetch(
@@ -409,7 +423,11 @@ describe('v1CompletionsRoutes', () => {
const app = createTestApp(
createMockFluxService(),
createMockConfigKV({ DEFAULT_TTS_MODEL: 'tts-1-hd' }),
createMockConfigKV(),
undefined,
undefined,
undefined,
createMockEnv({ DEFAULT_TTS_MODEL: 'tts-1-hd' }),
)
await app.fetch(
@@ -543,25 +561,15 @@ describe('v1CompletionsRoutes', () => {
describe('gET /api/v1/openai/audio/models', () => {
it('should return configured TTS model from config', async () => {
const app = createTestApp(createMockFluxService(), createMockConfigKV({ DEFAULT_TTS_MODEL: 'microsoft/v1' }))
const res = await app.fetch(
new Request('http://localhost/api/v1/openai/audio/models', { method: 'GET' }),
{ user: testUser } as any,
const app = createTestApp(
createMockFluxService(),
createMockConfigKV(),
undefined,
undefined,
undefined,
createMockEnv({ DEFAULT_TTS_MODEL: 'microsoft/v1' }),
)
expect(res.status).toBe(200)
expect(globalThis.fetch).toHaveBeenCalledWith(
'http://mock-gateway/audio/transcriptions',
expect.objectContaining({ method: 'POST' }),
)
})
})
describe('gET /api/v1/openai/audio/models', () => {
it('should return configured TTS model from config', async () => {
const app = createTestApp(createMockFluxService(), createMockConfigKV({ DEFAULT_TTS_MODEL: 'microsoft/v1' }))
const res = await app.fetch(
new Request('http://localhost/api/v1/openai/audio/models', { method: 'GET' }),
{ user: testUser } as any,
@@ -579,22 +587,35 @@ describe('v1CompletionsRoutes', () => {
const res = await app.request('/api/v1/openai/audio/models', { method: 'GET' })
expect(res.status).toBe(401)
})
})
it('should return 503 when DEFAULT_TTS_MODEL is not configured', async () => {
const configKV = createMockConfigKV()
configKV.getOptional = vi.fn(async (key: string) => {
if (key === 'DEFAULT_TTS_MODEL')
return null
return (configKV as any).__defaults?.[key] ?? null
})
describe('gET /api/v1/openai/audio/voices', () => {
it('should proxy voice list from gateway', async () => {
const voicesResponse = { voices: [
{ id: 'en-US-JennyNeural', name: 'Jenny', provider: 'MICROSOFT_SPEECH_SERVICE_V1', locale: 'en-US', gender: 'Female' },
{ id: 'alloy', name: 'Alloy', provider: 'OPEN_AI', locale: '', gender: '' },
] }
globalThis.fetch = vi.fn(async () => new Response(JSON.stringify(voicesResponse), {
status: 200,
headers: { 'Content-Type': 'application/json' },
}))
const app = createTestApp(createMockFluxService(), configKV)
const app = createTestApp(createMockFluxService(), createMockConfigKV())
const res = await app.fetch(
new Request('http://localhost/api/v1/openai/audio/models', { method: 'GET' }),
new Request('http://localhost/api/v1/openai/audio/voices', { method: 'GET' }),
{ user: testUser } as any,
)
expect(res.status).toBe(503)
expect(res.status).toBe(200)
const data = await res.json() as typeof voicesResponse
expect(data.voices).toHaveLength(2)
expect(data.voices[0].id).toBe('en-US-JennyNeural')
expect(globalThis.fetch).toHaveBeenCalledWith(
'http://mock-gateway/audio/transcriptions',
expect.objectContaining({ method: 'POST' }),
)
})
})
apps/server/src/services/config-kv.ts
-2
View File
@@ -22,8 +22,6 @@ const ConfigEntrySchemas = {
// Debt-ledger TTL: residual TTS chars below 1 Flux are forgiven on expiry.
// 24h gives users a long-enough window for accumulated dust to settle naturally.
TTS_DEBT_TTL_SECONDS: optional(number(), 86400),
GATEWAY_BASE_URL: string(),
DEFAULT_CHAT_MODEL: string(),
AUTH_RATE_LIMIT_MAX: optional(number(), 20),
AUTH_RATE_LIMIT_WINDOW_SEC: optional(number(), 60),
// No default — absent means top-up is not available yet
apps/server/src/services/tests/config-kv.test.ts
+5 -5
View File
@@ -24,7 +24,7 @@ describe('configKVService', () => {
// --- get ---
it('get should throw 503 when key is not set', async () => {
await expect(service.getOrThrow('GATEWAY_BASE_URL'))
await expect(service.getOrThrow('FLUX_PER_1K_CHARS_TTS'))
.rejects
.toThrow('Service configuration is incomplete')
})
@@ -51,7 +51,7 @@ describe('configKVService', () => {
})
it('getOptional should return null when required key is not set', async () => {
const value = await service.getOptional('GATEWAY_BASE_URL')
const value = await service.getOptional('FLUX_PER_1K_CHARS_TTS')
expect(value).toBeNull()
})
@@ -72,7 +72,7 @@ describe('configKVService', () => {
})
it('set should reject invalid values for string config keys', async () => {
await expect(service.set('GATEWAY_BASE_URL', { url: 'https://example.com' } as any))
await expect(service.set('STRIPE_FLUX_PRODUCT_ID', { id: 'prod_123' } as any))
.rejects
.toThrow()
})
@@ -85,8 +85,8 @@ describe('configKVService', () => {
})
it('set should store string values as JSON strings', async () => {
await service.set('GATEWAY_BASE_URL', 'https://gateway.example.com')
await service.set('STRIPE_FLUX_PRODUCT_ID', 'prod_abc123')
expect(redis._store.get(configRedisKey('GATEWAY_BASE_URL'))).toBe(JSON.stringify('https://gateway.example.com'))
expect(redis._store.get(configRedisKey('STRIPE_FLUX_PRODUCT_ID'))).toBe(JSON.stringify('prod_abc123'))
})
})
apps/server/src/utils/tests/redis-keys.test.ts
+1 -1
View File
@@ -22,7 +22,7 @@ describe('redis key utils', () => {
it('exposes stable helpers for config, user, and lock namespaces', () => {
expect(DEFAULT_BILLING_EVENTS_STREAM).toBe('billing:events')
expect(configRedisKey('GATEWAY_BASE_URL')).toBe('config:GATEWAY_BASE_URL')
expect(configRedisKey('FLUX_PER_REQUEST')).toBe('config:FLUX_PER_REQUEST')
expect(userFluxRedisKey('user-1')).toBe('user:user-1:flux')
expect(userChatBroadcastRedisKey('user-1')).toBe('user:user-1:chat:broadcast')
expect(lockRedisKey('user', 'user-1', 'flux')).toBe('lock:user:user-1:flux')