diff --git a/apps/server/.env b/apps/server/.env index 716d12b75..2895eb0a0 100644 --- a/apps/server/.env +++ b/apps/server/.env @@ -16,3 +16,7 @@ CLIENT_URL="" API_SERVER_URL="" # OTEL_EXPORTER_OTLP_ENDPOINT="http://localhost:4318" + +GATEWAY_BASE_URL="http://localhost:18080" +DEFAULT_CHAT_MODEL="openai/gpt-5-mini" +DEFAULT_TTS_MODEL="microsoft/v1" diff --git a/apps/server/src/app.ts b/apps/server/src/app.ts index 3675f0bd9..32854d8a6 100644 --- a/apps/server/src/app.ts +++ b/apps/server/src/app.ts @@ -191,7 +191,7 @@ export async function buildApp(deps: AppDeps) { /** * V1 routes for official provider. */ - .route('/api/v1/openai', createV1CompletionsRoutes(deps.fluxService, deps.billingService, deps.configKV, deps.billingMq, deps.ttsMeter, deps.otel?.genAi)) + .route('/api/v1/openai', createV1CompletionsRoutes(deps.fluxService, deps.billingService, deps.configKV, deps.billingMq, deps.ttsMeter, deps.env, deps.otel?.genAi)) /** * Flux routes. diff --git a/apps/server/src/libs/env.ts b/apps/server/src/libs/env.ts index 5749fbff0..b948ed158 100644 --- a/apps/server/src/libs/env.ts +++ b/apps/server/src/libs/env.ts @@ -56,6 +56,11 @@ const EnvSchema = object({ STRIPE_SECRET_KEY: optional(string()), STRIPE_WEBHOOK_SECRET: optional(string()), + // LLM gateway (infrastructure config — baked per deployment) + GATEWAY_BASE_URL: pipe(string(), nonEmpty('GATEWAY_BASE_URL is required')), + DEFAULT_CHAT_MODEL: pipe(string(), nonEmpty('DEFAULT_CHAT_MODEL is required')), + DEFAULT_TTS_MODEL: pipe(string(), nonEmpty('DEFAULT_TTS_MODEL is required')), + BILLING_EVENTS_STREAM: optional(string(), DEFAULT_BILLING_EVENTS_STREAM), BILLING_EVENTS_CONSUMER_NAME: optional(string()), BILLING_EVENTS_BATCH_SIZE: optionalIntegerFromString(10, 'BILLING_EVENTS_BATCH_SIZE', 1), diff --git a/apps/server/src/routes/openai/v1/index.ts b/apps/server/src/routes/openai/v1/index.ts index ad6165a9c..d3f22a60e 100644 --- a/apps/server/src/routes/openai/v1/index.ts +++ b/apps/server/src/routes/openai/v1/index.ts @@ -1,5 +1,6 @@ import type { Context } from 'hono' +import type { Env } from '../../../libs/env' import type { MqService } from '../../../libs/mq' import type { GenAiMetrics } from '../../../libs/otel' import type { UsageInfo } from '../../../services/billing/billing' @@ -70,7 +71,7 @@ function getLlmMetricAttributes(opts: { model: string, type: string, status: num } } -export function createV1CompletionsRoutes(fluxService: FluxService, billingService: BillingService, configKV: ConfigKVService, billingMq: MqService, ttsMeter: FluxMeter, genAi?: GenAiMetrics | null) { +export function createV1CompletionsRoutes(fluxService: FluxService, billingService: BillingService, configKV: ConfigKVService, billingMq: MqService, ttsMeter: FluxMeter, env: Env, genAi?: GenAiMetrics | null) { const logger = useLogger('v1-completions').useGlobalConfig() // TODO: Extract this compat route into smaller facades/modules. // It currently mixes auth, rate limiting, proxying, billing, telemetry, and event publishing in one transport layer entrypoint. @@ -121,13 +122,12 @@ export function createV1CompletionsRoutes(fluxService: FluxService, billingServi } const body = await c.req.json() - const gatewayBaseUrl = await configKV.getOrThrow('GATEWAY_BASE_URL') - const baseUrl = normalizeBaseUrl(gatewayBaseUrl) + const baseUrl = normalizeBaseUrl(env.GATEWAY_BASE_URL) const serverAttributes = getServerConnectionAttributes(baseUrl) let requestModel = body.model || 'auto' if (requestModel === 'auto') { - requestModel = await configKV.getOrThrow('DEFAULT_CHAT_MODEL') + requestModel = env.DEFAULT_CHAT_MODEL } const span = tracer.startSpan('llm.gateway.chat', { @@ -324,19 +324,15 @@ export function createV1CompletionsRoutes(fluxService: FluxService, billingServi // throw createPaymentRequiredError('Insufficient flux') // } - // const body = await c.req.json() - // const gatewayBaseUrl = await configKV.getOrThrow('GATEWAY_BASE_URL') - // const baseUrl = normalizeBaseUrl(gatewayBaseUrl) - // const serverAttributes = getServerConnectionAttributes(baseUrl) - // const requestModel = body.model || 'auto' + const body = await c.req.json() + const baseUrl = normalizeBaseUrl(env.GATEWAY_BASE_URL) + const serverAttributes = getServerConnectionAttributes(baseUrl) + let requestModel = body.model || 'auto' + const inputText: string = body.input ?? '' - // const span = tracer.startSpan('llm.gateway.tts', { - // attributes: { - // [GEN_AI_ATTR_REQUEST_MODEL]: requestModel, - // [AIRI_ATTR_GEN_AI_OPERATION_KIND]: 'text_to_speech', - // ...serverAttributes, - // }, - // }) + if (requestModel === 'auto') { + requestModel = env.DEFAULT_TTS_MODEL + } // Pre-flight: refuse before hitting upstream if this segment would push the // user past their balance. Cheap-path requests below the Flux threshold @@ -407,13 +403,8 @@ export function createV1CompletionsRoutes(fluxService: FluxService, billingServi // const baseUrl = normalizeBaseUrl(gatewayBaseUrl) // const serverAttributes = getServerConnectionAttributes(baseUrl) - // const span = tracer.startSpan('llm.gateway.asr', { - // attributes: { - // [GEN_AI_ATTR_REQUEST_MODEL]: 'auto', - // [AIRI_ATTR_GEN_AI_OPERATION_KIND]: 'speech_to_text', - // ...serverAttributes, - // }, - // }) + async function handleListVoices(_c: Context) { + const baseUrl = normalizeBaseUrl(env.GATEWAY_BASE_URL) // const startedAt = Date.now() @@ -427,48 +418,15 @@ export function createV1CompletionsRoutes(fluxService: FluxService, billingServi // body: rawBody, // })) - // const durationMs = Date.now() - startedAt - // span.setAttribute('http.response.status_code', response.status) + async function handleListTTSModels(_c: Context) { + const model = env.DEFAULT_TTS_MODEL + return Response.json({ + models: [{ id: model, name: model }], + }) + } - // if (!response.ok) { - // span.setStatus({ code: SpanStatusCode.ERROR, message: `Gateway ${response.status}` }) - // span.end() - // recordMetrics({ model: 'auto', status: response.status, type: 'asr', durationMs, fluxConsumed: 0 }) - // return new Response(response.body, { - // status: response.status, - // headers: buildSafeResponseHeaders(response), - // }) - // } - - // const fluxPerRequest = await configKV.getOrThrow('FLUX_PER_REQUEST_ASR') - // await billingService.consumeFluxForLLM({ - // userId: user.id, - // amount: fluxPerRequest, - // requestId: nanoid(), - // description: `asr:auto`, - // }) - - // span.setAttribute(AIRI_ATTR_BILLING_FLUX_CONSUMED, fluxPerRequest) - // span.end() - // recordMetrics({ model: 'auto', status: response.status, type: 'asr', durationMs, fluxConsumed: fluxPerRequest }) - - // publishRequestLog({ - // userId: user.id, - // model: 'auto', - // status: response.status, - // durationMs, - // fluxConsumed: fluxPerRequest, - // }) - - // return new Response(response.body, { - // status: response.status, - // headers: buildSafeResponseHeaders(response), - // }) - // } - - const chatGuard = configGuard(configKV, ['FLUX_PER_REQUEST', 'GATEWAY_BASE_URL', 'DEFAULT_CHAT_MODEL'], 'Service is not available yet') - // const ttsGuard = configGuard(configKV, ['FLUX_PER_REQUEST_TTS', 'GATEWAY_BASE_URL'], 'TTS service is not available yet') - // const asrGuard = configGuard(configKV, ['FLUX_PER_REQUEST_ASR', 'GATEWAY_BASE_URL'], 'ASR service is not available yet') + const chatGuard = configGuard(configKV, ['FLUX_PER_REQUEST'], 'Service is not available yet') + const ttsGuard = configGuard(configKV, ['FLUX_PER_1K_CHARS_TTS'], 'TTS service is not available yet') // 60 requests per minute per user for LLM completions const completionsRateLimit = rateLimiter({ max: 60, windowSec: 60 }) @@ -477,6 +435,7 @@ export function createV1CompletionsRoutes(fluxService: FluxService, billingServi .use('*', authGuard) .post('/chat/completions', completionsRateLimit, chatGuard, handleCompletion) .post('/chat/completion', completionsRateLimit, chatGuard, handleCompletion) - // .post('/audio/speech', ttsGuard, handleTTS) - // .post('/audio/transcriptions', bodyLimit({ maxSize: 25 * 1024 * 1024 }), asrGuard, handleTranscription) + .post('/audio/speech', ttsGuard, handleTTS) + .get('/audio/voices', handleListVoices) + .get('/audio/models', handleListTTSModels) } diff --git a/apps/server/src/routes/openai/v1/route.test.ts b/apps/server/src/routes/openai/v1/route.test.ts index ceba731df..02e07526b 100644 --- a/apps/server/src/routes/openai/v1/route.test.ts +++ b/apps/server/src/routes/openai/v1/route.test.ts @@ -1,3 +1,4 @@ +import type { Env } from '../../../libs/env' import type { MqService } from '../../../libs/mq' import type { BillingEvent } from '../../../services/billing/billing-events' import type { BillingService } from '../../../services/billing/billing-service' @@ -34,13 +35,20 @@ function createMockBillingService(flux = 100): BillingService { } as any } +function createMockEnv(overrides: Partial = {}): Env { + return { + GATEWAY_BASE_URL: 'http://mock-gateway/', + DEFAULT_CHAT_MODEL: 'openai/gpt-5-mini', + DEFAULT_TTS_MODEL: 'tts-1', + ...overrides, + } as Env +} + function createMockConfigKV(overrides: Record = {}): ConfigKVService { const defaults: Record = { FLUX_PER_REQUEST: 1, FLUX_PER_1K_CHARS_TTS: 2, TTS_DEBT_TTL_SECONDS: 86400, - GATEWAY_BASE_URL: 'http://mock-gateway/', - DEFAULT_CHAT_MODEL: 'openai/gpt-5-mini', ...overrides, } return { @@ -87,6 +95,7 @@ function createTestApp( billingService?: BillingService, billingMq?: MqService, ttsMeter?: ReturnType, + env?: Env, ) { const routes = createV1CompletionsRoutes( fluxService, @@ -94,6 +103,7 @@ function createTestApp( configKV, billingMq ?? createMockBillingMq(), ttsMeter ?? createMockTtsMeter(), + env ?? createMockEnv(), null, ) const app = new Hono() @@ -174,7 +184,7 @@ describe('v1CompletionsRoutes', () => { const fluxService = createMockFluxService(100) const billingService = createMockBillingService(100) - const configKV = createMockConfigKV({ GATEWAY_BASE_URL: 'http://mock-gateway/' }) + const configKV = createMockConfigKV() const app = createTestApp(fluxService, configKV, billingService) const res = await app.fetch( @@ -213,7 +223,11 @@ describe('v1CompletionsRoutes', () => { const app = createTestApp( createMockFluxService(), - createMockConfigKV({ DEFAULT_CHAT_MODEL: 'anthropic/claude-sonnet' }), + createMockConfigKV(), + undefined, + undefined, + undefined, + createMockEnv({ DEFAULT_CHAT_MODEL: 'anthropic/claude-sonnet' }), ) await app.fetch( @@ -409,7 +423,11 @@ describe('v1CompletionsRoutes', () => { const app = createTestApp( createMockFluxService(), - createMockConfigKV({ DEFAULT_TTS_MODEL: 'tts-1-hd' }), + createMockConfigKV(), + undefined, + undefined, + undefined, + createMockEnv({ DEFAULT_TTS_MODEL: 'tts-1-hd' }), ) await app.fetch( @@ -543,25 +561,15 @@ describe('v1CompletionsRoutes', () => { describe('gET /api/v1/openai/audio/models', () => { it('should return configured TTS model from config', async () => { - const app = createTestApp(createMockFluxService(), createMockConfigKV({ DEFAULT_TTS_MODEL: 'microsoft/v1' })) - - const res = await app.fetch( - new Request('http://localhost/api/v1/openai/audio/models', { method: 'GET' }), - { user: testUser } as any, + const app = createTestApp( + createMockFluxService(), + createMockConfigKV(), + undefined, + undefined, + undefined, + createMockEnv({ DEFAULT_TTS_MODEL: 'microsoft/v1' }), ) - expect(res.status).toBe(200) - expect(globalThis.fetch).toHaveBeenCalledWith( - 'http://mock-gateway/audio/transcriptions', - expect.objectContaining({ method: 'POST' }), - ) - }) - }) - - describe('gET /api/v1/openai/audio/models', () => { - it('should return configured TTS model from config', async () => { - const app = createTestApp(createMockFluxService(), createMockConfigKV({ DEFAULT_TTS_MODEL: 'microsoft/v1' })) - const res = await app.fetch( new Request('http://localhost/api/v1/openai/audio/models', { method: 'GET' }), { user: testUser } as any, @@ -579,22 +587,35 @@ describe('v1CompletionsRoutes', () => { const res = await app.request('/api/v1/openai/audio/models', { method: 'GET' }) expect(res.status).toBe(401) }) + }) - it('should return 503 when DEFAULT_TTS_MODEL is not configured', async () => { - const configKV = createMockConfigKV() - configKV.getOptional = vi.fn(async (key: string) => { - if (key === 'DEFAULT_TTS_MODEL') - return null - return (configKV as any).__defaults?.[key] ?? null - }) + describe('gET /api/v1/openai/audio/voices', () => { + it('should proxy voice list from gateway', async () => { + const voicesResponse = { voices: [ + { id: 'en-US-JennyNeural', name: 'Jenny', provider: 'MICROSOFT_SPEECH_SERVICE_V1', locale: 'en-US', gender: 'Female' }, + { id: 'alloy', name: 'Alloy', provider: 'OPEN_AI', locale: '', gender: '' }, + ] } + globalThis.fetch = vi.fn(async () => new Response(JSON.stringify(voicesResponse), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + })) - const app = createTestApp(createMockFluxService(), configKV) + const app = createTestApp(createMockFluxService(), createMockConfigKV()) const res = await app.fetch( - new Request('http://localhost/api/v1/openai/audio/models', { method: 'GET' }), + new Request('http://localhost/api/v1/openai/audio/voices', { method: 'GET' }), { user: testUser } as any, ) - expect(res.status).toBe(503) + + expect(res.status).toBe(200) + const data = await res.json() as typeof voicesResponse + expect(data.voices).toHaveLength(2) + expect(data.voices[0].id).toBe('en-US-JennyNeural') + + expect(globalThis.fetch).toHaveBeenCalledWith( + 'http://mock-gateway/audio/transcriptions', + expect.objectContaining({ method: 'POST' }), + ) }) }) diff --git a/apps/server/src/services/config-kv.ts b/apps/server/src/services/config-kv.ts index 9ee08f3e1..8c9f5012e 100644 --- a/apps/server/src/services/config-kv.ts +++ b/apps/server/src/services/config-kv.ts @@ -22,8 +22,6 @@ const ConfigEntrySchemas = { // Debt-ledger TTL: residual TTS chars below 1 Flux are forgiven on expiry. // 24h gives users a long-enough window for accumulated dust to settle naturally. TTS_DEBT_TTL_SECONDS: optional(number(), 86400), - GATEWAY_BASE_URL: string(), - DEFAULT_CHAT_MODEL: string(), AUTH_RATE_LIMIT_MAX: optional(number(), 20), AUTH_RATE_LIMIT_WINDOW_SEC: optional(number(), 60), // No default — absent means top-up is not available yet diff --git a/apps/server/src/services/tests/config-kv.test.ts b/apps/server/src/services/tests/config-kv.test.ts index 1ef8f00c8..ace74eb08 100644 --- a/apps/server/src/services/tests/config-kv.test.ts +++ b/apps/server/src/services/tests/config-kv.test.ts @@ -24,7 +24,7 @@ describe('configKVService', () => { // --- get --- it('get should throw 503 when key is not set', async () => { - await expect(service.getOrThrow('GATEWAY_BASE_URL')) + await expect(service.getOrThrow('FLUX_PER_1K_CHARS_TTS')) .rejects .toThrow('Service configuration is incomplete') }) @@ -51,7 +51,7 @@ describe('configKVService', () => { }) it('getOptional should return null when required key is not set', async () => { - const value = await service.getOptional('GATEWAY_BASE_URL') + const value = await service.getOptional('FLUX_PER_1K_CHARS_TTS') expect(value).toBeNull() }) @@ -72,7 +72,7 @@ describe('configKVService', () => { }) it('set should reject invalid values for string config keys', async () => { - await expect(service.set('GATEWAY_BASE_URL', { url: 'https://example.com' } as any)) + await expect(service.set('STRIPE_FLUX_PRODUCT_ID', { id: 'prod_123' } as any)) .rejects .toThrow() }) @@ -85,8 +85,8 @@ describe('configKVService', () => { }) it('set should store string values as JSON strings', async () => { - await service.set('GATEWAY_BASE_URL', 'https://gateway.example.com') + await service.set('STRIPE_FLUX_PRODUCT_ID', 'prod_abc123') - expect(redis._store.get(configRedisKey('GATEWAY_BASE_URL'))).toBe(JSON.stringify('https://gateway.example.com')) + expect(redis._store.get(configRedisKey('STRIPE_FLUX_PRODUCT_ID'))).toBe(JSON.stringify('prod_abc123')) }) }) diff --git a/apps/server/src/utils/tests/redis-keys.test.ts b/apps/server/src/utils/tests/redis-keys.test.ts index 092c56706..bc4f14a18 100644 --- a/apps/server/src/utils/tests/redis-keys.test.ts +++ b/apps/server/src/utils/tests/redis-keys.test.ts @@ -22,7 +22,7 @@ describe('redis key utils', () => { it('exposes stable helpers for config, user, and lock namespaces', () => { expect(DEFAULT_BILLING_EVENTS_STREAM).toBe('billing:events') - expect(configRedisKey('GATEWAY_BASE_URL')).toBe('config:GATEWAY_BASE_URL') + expect(configRedisKey('FLUX_PER_REQUEST')).toBe('config:FLUX_PER_REQUEST') expect(userFluxRedisKey('user-1')).toBe('user:user-1:flux') expect(userChatBroadcastRedisKey('user-1')).toBe('user:user-1:chat:broadcast') expect(lockRedisKey('user', 'user-1', 'flux')).toBe('lock:user:user-1:flux')