From 0062070d7dc2a0abbb5f62f5de75f95514abfd60 Mon Sep 17 00:00:00 2001 From: RainbowBird Date: Wed, 19 Aug 2026 16:30:55 +0800 Subject: [PATCH] fix(api): correct gateway and database pool alerts (#2324) --- pnpm-lock.yaml | 1 + server/apps/api/src/app.ts | 5 +- .../apps/api/src/otel/gauges/db-pool.test.ts | 47 ++++++++++++++++++ server/apps/api/src/otel/gauges/db-pool.ts | 33 +++++++++++++ server/apps/api/src/otel/index.ts | 27 +++++++++-- .../src/services/domain/llm-router/router.ts | 14 +++++- .../domain/llm-router/tests/router.test.ts | 48 +++++++++++++++++-- server/apps/api/src/utils/observability.ts | 3 ++ 8 files changed, 167 insertions(+), 11 deletions(-) create mode 100644 server/apps/api/src/otel/gauges/db-pool.test.ts create mode 100644 server/apps/api/src/otel/gauges/db-pool.ts diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 48d698dbb..77eddafcf 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -6482,6 +6482,7 @@ packages: '@better-auth/cli@1.4.22': resolution: {integrity: sha512-7azgrNiP1zJXMLqoLgCVj3KsZeYWLHeaGMapYlLblS6yU/o5n//sn1HBasRD2z4HBy2Etz/C7V483/mYvRHI2g==} + deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info. hasBin: true '@better-auth/core@1.4.22': diff --git a/server/apps/api/src/app.ts b/server/apps/api/src/app.ts index 463eb4a11..b132240ba 100644 --- a/server/apps/api/src/app.ts +++ b/server/apps/api/src/app.ts @@ -41,6 +41,7 @@ import { resolveRequestAuth } from './libs/request-auth' import { createUnauthorizedWsEvents } from './libs/ws-auth' import { sessionMiddleware } from './middlewares/auth' import { emitOtelLog, initOtel } from './otel' +import { registerDbPoolGauge } from './otel/gauges/db-pool' import { registerTtsPoolGauge } from './otel/gauges/tts-pool' import { registerWsOnlineUsersGauge } from './otel/gauges/ws-online-users' import { createAudioSpeechWsHandlers } from './routes/audio-speech-ws' @@ -427,7 +428,7 @@ export async function createApp() { }) const db = injeca.provide('datastore:db', { - dependsOn: { env: parsedEnv, lifecycle }, + dependsOn: { env: parsedEnv, lifecycle, otel }, build: async ({ dependsOn }) => { const { db: dbInstance, pool } = await initializeExternalDependency( 'Database', @@ -449,6 +450,8 @@ export async function createApp() { }, ) + if (dependsOn.otel) + registerDbPoolGauge(dependsOn.otel.database.poolConnections, pool) dependsOn.lifecycle.appHooks.onStop(() => pool.end()) return dbInstance }, diff --git a/server/apps/api/src/otel/gauges/db-pool.test.ts b/server/apps/api/src/otel/gauges/db-pool.test.ts new file mode 100644 index 000000000..9403098d0 --- /dev/null +++ b/server/apps/api/src/otel/gauges/db-pool.test.ts @@ -0,0 +1,47 @@ +import type { DatabaseMetrics } from '..' + +import { describe, expect, it, vi } from 'vitest' + +import { registerDbPoolGauge } from './db-pool' + +function makeGauge() { + let callback: ((result: { observe: (value: number, attributes: Record) => void }) => void) | null = null + const observe = vi.fn() + const gauge = { + addCallback: vi.fn((registeredCallback: typeof callback) => { + callback = registeredCallback + }), + } as unknown as DatabaseMetrics['poolConnections'] + + return { + gauge, + observe, + run() { + if (!callback) + throw new Error('no callback registered') + callback({ observe }) + }, + } +} + +describe('registerDbPoolGauge', () => { + it('reports the configured capacity and the live pool counts', () => { + const pool = { + options: { max: 20 }, + totalCount: 7, + idleCount: 2, + waitingCount: 3, + } + const { gauge, observe, run } = makeGauge() + + registerDbPoolGauge(gauge, pool) + run() + + expect(observe).toHaveBeenCalledTimes(5) + expect(observe).toHaveBeenCalledWith(20, { pool_state: 'max' }) + expect(observe).toHaveBeenCalledWith(7, { pool_state: 'total' }) + expect(observe).toHaveBeenCalledWith(5, { pool_state: 'used' }) + expect(observe).toHaveBeenCalledWith(2, { pool_state: 'idle' }) + expect(observe).toHaveBeenCalledWith(3, { pool_state: 'waiting' }) + }) +}) diff --git a/server/apps/api/src/otel/gauges/db-pool.ts b/server/apps/api/src/otel/gauges/db-pool.ts new file mode 100644 index 000000000..6daf1d08d --- /dev/null +++ b/server/apps/api/src/otel/gauges/db-pool.ts @@ -0,0 +1,33 @@ +import type { DatabaseMetrics } from '..' + +interface PoolStats { + idleCount: number + options: { max?: number } + totalCount: number + waitingCount: number +} + +/** + * Observe the local pg pool. The standard pg metric has no configured limit, + * so it cannot show how close this process is to its own pool capacity. + */ +export function registerDbPoolGauge( + gauge: DatabaseMetrics['poolConnections'], + pool: PoolStats, +) { + gauge.addCallback((result) => { + const total = pool.totalCount + const idle = pool.idleCount + const used = total - idle + const counts = { + max: pool.options.max ?? 10, + total, + used, + idle, + waiting: pool.waitingCount, + } + + for (const [pool_state, value] of Object.entries(counts)) + result.observe(value, { pool_state }) + }) +} diff --git a/server/apps/api/src/otel/index.ts b/server/apps/api/src/otel/index.ts index d03702db4..be82ae888 100644 --- a/server/apps/api/src/otel/index.ts +++ b/server/apps/api/src/otel/index.ts @@ -14,6 +14,7 @@ import { metrics, trace } from '@opentelemetry/api' import { logs, SeverityNumber } from '@opentelemetry/api-logs' import { + METRIC_AIRI_DB_POOL_CONNECTIONS, METRIC_AIRI_EMAIL_DURATION, METRIC_AIRI_EMAIL_FAILURES, METRIC_AIRI_EMAIL_SEND, @@ -168,12 +169,12 @@ export interface GatewayMetrics { */ upstreamErrors: Counter /** - * All keys (across all upstreams) failed in a single request — the user gets - * a 5xx. Primary alert source for user-facing degradation. - * Recommended label: `provider`. + * The configured route exhausted every allowed key and upstream in one + * request. The user gets a 5xx. Primary alert source for user-facing + * degradation. Recommended labels: `provider`, `status_code`, `surface`. * * Recommended alert: - * `increase(airi_gen_ai_gateway_key_exhausted_total[5m]) > 0` → page on-call. + * Filter to operational status codes before paging on this metric. */ keyExhaustedCount: Counter /** @@ -255,12 +256,22 @@ export interface ObservabilityMetrics { metricReadErrors: Counter } +export interface DatabaseMetrics { + /** + * Per-process pg pool counts. Labels: `pool_state` (`max`, `total`, `used`, + * `idle`, `waiting`). Use `used / max` for capacity and `waiting > 0` for + * saturation. Do not use `used / (used + idle)` as a capacity ratio. + */ + poolConnections: ObservableGauge +} + export interface OtelInstance { auth: AuthMetrics engagement: EngagementMetrics revenue: RevenueMetrics genAi: GenAiMetrics gateway: GatewayMetrics + database: DatabaseMetrics email: EmailMetrics rateLimit: RateLimitMetrics observability: ObservabilityMetrics @@ -464,6 +475,12 @@ export function initOtel(env: Env): OtelInstance | null { }), } + const database: DatabaseMetrics = { + poolConnections: meter.createObservableGauge(METRIC_AIRI_DB_POOL_CONNECTIONS, { + description: 'Local pg pool connections by capacity, use, idle, and waiting state', + }), + } + // NOTICE: // OTel SDK only emits a Counter time series after .add() runs the first time. // Without this priming step, low-traffic counters (auth_failures_total, @@ -515,7 +532,7 @@ export function initOtel(env: Env): OtelInstance | null { ] for (const counter of counters) counter.add(0) - return { auth, engagement, revenue, genAi, gateway, email, rateLimit, observability } + return { auth, engagement, revenue, genAi, gateway, database, email, rateLimit, observability } } const severityMap: Record = { diff --git a/server/apps/api/src/services/domain/llm-router/router.ts b/server/apps/api/src/services/domain/llm-router/router.ts index 4d9ebfc4a..96461015b 100644 --- a/server/apps/api/src/services/domain/llm-router/router.ts +++ b/server/apps/api/src/services/domain/llm-router/router.ts @@ -421,7 +421,6 @@ export function createLlmRouterService(options: CreateLlmRouterServiceOptions) { return { kind: 'ok' as const, response: result.response } } - options.gatewayMetrics?.keyExhaustedCount.add(1, { provider }) return { kind: 'exhausted' as const, statuses: result.failures.map(failure => failure.status), @@ -489,6 +488,12 @@ export function createLlmRouterService(options: CreateLlmRouterServiceOptions) { throw new Error(`Router exhausted with no recorded failures for model ${req.modelName}`) } + options.gatewayMetrics?.keyExhaustedCount.add(1, { + provider: lastFailure.provider, + status_code: typeof lastFailure.status === 'number' ? lastFailure.status : 'timeout', + surface: 'chat', + }) + // Same-status exhaustion: every recorded failure shares one status (or // timeout). This is a strong signal of a shared upstream constraint that // ordinary candidate fallback cannot recover from. @@ -840,7 +845,6 @@ export function createLlmRouterService(options: CreateLlmRouterServiceOptions) { } } - options.gatewayMetrics?.keyExhaustedCount.add(1, { provider: providerTag }) return { kind: 'exhausted', sawTooManyRequests: result.failures.some(f => f.status === 429), @@ -935,6 +939,12 @@ export function createLlmRouterService(options: CreateLlmRouterServiceOptions) { throw new Error(`Router exhausted with no recorded failures for tts model ${req.modelName}`) } + options.gatewayMetrics?.keyExhaustedCount.add(1, { + provider: lastFailure.provider, + status_code: typeof lastFailure.status === 'number' ? lastFailure.status : 'timeout', + surface: 'tts', + }) + const distinctStatuses = new Set(allFailures.map(f => f.status)) if (distinctStatuses.size === 1) { const status = allFailures[0].status diff --git a/server/apps/api/src/services/domain/llm-router/tests/router.test.ts b/server/apps/api/src/services/domain/llm-router/tests/router.test.ts index 99694dfcd..68eca3909 100644 --- a/server/apps/api/src/services/domain/llm-router/tests/router.test.ts +++ b/server/apps/api/src/services/domain/llm-router/tests/router.test.ts @@ -310,7 +310,7 @@ describe('createLlmRouterService', () => { expect((metrics.keyExhaustedCount.add as ReturnType).mock.calls.length).toBe(0) }) - it('cross-upstream fallback: upstream A keys all 401, upstream B[0] = 200 → returns 200, A exhaustion counted', async () => { + it('cross-upstream fallback: upstream A keys all 401, upstream B[0] = 200 → returns 200 without terminal exhaustion', async () => { const { config, crypto } = makeConfig({ upstreams: [ { baseURL: 'https://up-a.example/v1', keyIds: ['kA1', 'kA2'] }, @@ -336,7 +336,7 @@ describe('createLlmRouterService', () => { expect(res.status).toBe(200) expect(fetchImpl.mock.calls.length).toBe(3) - expect((metrics.keyExhaustedCount.add as ReturnType).mock.calls.length).toBe(1) + expect((metrics.keyExhaustedCount.add as ReturnType).mock.calls.length).toBe(0) expect((metrics.fallbackCount.add as ReturnType).mock.calls.length).toBe(2) }) @@ -370,7 +370,14 @@ describe('createLlmRouterService', () => { expect((err as ApiError).details).toMatchObject({ triedKeys: 2, triedUpstreams: 2, lastStatusCode: 401 }) } - expect((metrics.keyExhaustedCount.add as ReturnType).mock.calls.length).toBe(2) + const exhaustionCalls = (metrics.keyExhaustedCount.add as ReturnType).mock.calls + expect(exhaustionCalls.length).toBe(1) + expect(exhaustionCalls[0][0]).toBe(1) + expect(exhaustionCalls[0][1]).toMatchObject({ + provider: 'up-b.example', + status_code: 401, + surface: 'chat', + }) }) it('full exhaustion attaches per-attempt cause (bodySnippet for HTTP, errorMessage for network) so operators can debug 502s', async () => { @@ -968,6 +975,41 @@ describe('createLlmRouterService', () => { expect(fallbackCalls[0][1]).toMatchObject({ reason: '401' }) }) + it('records a terminal TTS exhaustion with its final status', async () => { + const { config, crypto } = makeTtsConfig({ + upstreams: [{ + baseURL: 'https://az.example', + keyIds: ['kA1'], + adapterParams: { region: 'eastasia' }, + }], + }) + const fetchImpl = vi.fn(async () => failResponse(451)) + const metrics = makeMetrics() + + const router = createLlmRouterService({ + configKV: makeConfigKV(config), + envelopeCrypto: crypto, + gatewayMetrics: metrics, + fetchImpl, + redis: makeRedisStub(), + concurrencyLedger: makeLedger(), + }) + + await expect(router.routeTts({ + modelName: 'tts-test', + input: { text: 'hi', voice: 'en-US-AvaMultilingualNeural' }, + })).rejects.toMatchObject({ statusCode: 502 }) + + const exhaustionCalls = (metrics.keyExhaustedCount.add as ReturnType).mock.calls + expect(exhaustionCalls.length).toBe(1) + expect(exhaustionCalls[0][0]).toBe(1) + expect(exhaustionCalls[0][1]).toMatchObject({ + provider: 'az.example', + status_code: 451, + surface: 'tts', + }) + }) + it('listTtsVoices deduplicates concurrent cold-cache upstream fetches per model', async () => { // ROOT CAUSE: // diff --git a/server/apps/api/src/utils/observability.ts b/server/apps/api/src/utils/observability.ts index 9d263b845..96f4e3500 100644 --- a/server/apps/api/src/utils/observability.ts +++ b/server/apps/api/src/utils/observability.ts @@ -85,6 +85,9 @@ export const METRIC_AIRI_TTS_PREFLIGHT_REJECTIONS = 'airi.billing.tts.preflight_ // AIRI observability — self-monitoring for the metric pipeline export const METRIC_AIRI_OBSERVABILITY_READ_ERRORS = 'airi.observability.read_errors' +// AIRI database — local pg pool capacity and queue state +export const METRIC_AIRI_DB_POOL_CONNECTIONS = 'airi.db.pool.connections' + // AIRI revenue — actual money in (smallest currency unit, e.g. cents) export const METRIC_AIRI_STRIPE_REVENUE = 'airi.stripe.revenue'