fix(api): correct gateway and database pool alerts (#2324)

server/apps/api/src/app.ts

@@ -41,6 +41,7 @@ import { resolveRequestAuth } from './libs/request-auth'
 import { createUnauthorizedWsEvents } from './libs/ws-auth'
 import { sessionMiddleware } from './middlewares/auth'
 import { emitOtelLog, initOtel } from './otel'
+import { registerDbPoolGauge } from './otel/gauges/db-pool'
 import { registerTtsPoolGauge } from './otel/gauges/tts-pool'
 import { registerWsOnlineUsersGauge } from './otel/gauges/ws-online-users'
 import { createAudioSpeechWsHandlers } from './routes/audio-speech-ws'
@@ -427,7 +428,7 @@ export async function createApp() {
   })
 
   const db = injeca.provide('datastore:db', {
-    dependsOn: { env: parsedEnv, lifecycle },
+    dependsOn: { env: parsedEnv, lifecycle, otel },
     build: async ({ dependsOn }) => {
       const { db: dbInstance, pool } = await initializeExternalDependency(
         'Database',
@@ -449,6 +450,8 @@ export async function createApp() {
         },
       )
 
+      if (dependsOn.otel)
+        registerDbPoolGauge(dependsOn.otel.database.poolConnections, pool)
       dependsOn.lifecycle.appHooks.onStop(() => pool.end())
       return dbInstance
     },

server/apps/api/src/otel/gauges/db-pool.test.ts

@@ -0,0 +1,47 @@
+import type { DatabaseMetrics } from '..'
+
+import { describe, expect, it, vi } from 'vitest'
+
+import { registerDbPoolGauge } from './db-pool'
+
+function makeGauge() {
+  let callback: ((result: { observe: (value: number, attributes: Record<string, string>) => void }) => void) | null = null
+  const observe = vi.fn()
+  const gauge = {
+    addCallback: vi.fn((registeredCallback: typeof callback) => {
+      callback = registeredCallback
+    }),
+  } as unknown as DatabaseMetrics['poolConnections']
+
+  return {
+    gauge,
+    observe,
+    run() {
+      if (!callback)
+        throw new Error('no callback registered')
+      callback({ observe })
+    },
+  }
+}
+
+describe('registerDbPoolGauge', () => {
+  it('reports the configured capacity and the live pool counts', () => {
+    const pool = {
+      options: { max: 20 },
+      totalCount: 7,
+      idleCount: 2,
+      waitingCount: 3,
+    }
+    const { gauge, observe, run } = makeGauge()
+
+    registerDbPoolGauge(gauge, pool)
+    run()
+
+    expect(observe).toHaveBeenCalledTimes(5)
+    expect(observe).toHaveBeenCalledWith(20, { pool_state: 'max' })
+    expect(observe).toHaveBeenCalledWith(7, { pool_state: 'total' })
+    expect(observe).toHaveBeenCalledWith(5, { pool_state: 'used' })
+    expect(observe).toHaveBeenCalledWith(2, { pool_state: 'idle' })
+    expect(observe).toHaveBeenCalledWith(3, { pool_state: 'waiting' })
+  })
+})

server/apps/api/src/otel/gauges/db-pool.ts

@@ -0,0 +1,33 @@
+import type { DatabaseMetrics } from '..'
+
+interface PoolStats {
+  idleCount: number
+  options: { max?: number }
+  totalCount: number
+  waitingCount: number
+}
+
+/**
+ * Observe the local pg pool. The standard pg metric has no configured limit,
+ * so it cannot show how close this process is to its own pool capacity.
+ */
+export function registerDbPoolGauge(
+  gauge: DatabaseMetrics['poolConnections'],
+  pool: PoolStats,
+) {
+  gauge.addCallback((result) => {
+    const total = pool.totalCount
+    const idle = pool.idleCount
+    const used = total - idle
+    const counts = {
+      max: pool.options.max ?? 10,
+      total,
+      used,
+      idle,
+      waiting: pool.waitingCount,
+    }
+
+    for (const [pool_state, value] of Object.entries(counts))
+      result.observe(value, { pool_state })
+  })
+}

server/apps/api/src/otel/index.ts

@@ -14,6 +14,7 @@ import { metrics, trace } from '@opentelemetry/api'
 import { logs, SeverityNumber } from '@opentelemetry/api-logs'
 
 import {
+  METRIC_AIRI_DB_POOL_CONNECTIONS,
   METRIC_AIRI_EMAIL_DURATION,
   METRIC_AIRI_EMAIL_FAILURES,
   METRIC_AIRI_EMAIL_SEND,
@@ -168,12 +169,12 @@ export interface GatewayMetrics {
    */
   upstreamErrors: Counter
   /**
-   * All keys (across all upstreams) failed in a single request — the user gets
-   * a 5xx. Primary alert source for user-facing degradation.
-   * Recommended label: `provider`.
+   * The configured route exhausted every allowed key and upstream in one
+   * request. The user gets a 5xx. Primary alert source for user-facing
+   * degradation. Recommended labels: `provider`, `status_code`, `surface`.
    *
    * Recommended alert:
-   *   `increase(airi_gen_ai_gateway_key_exhausted_total[5m]) > 0` → page on-call.
+   *   Filter to operational status codes before paging on this metric.
    */
   keyExhaustedCount: Counter
   /**
@@ -255,12 +256,22 @@ export interface ObservabilityMetrics {
   metricReadErrors: Counter
 }
 
+export interface DatabaseMetrics {
+  /**
+   * Per-process pg pool counts. Labels: `pool_state` (`max`, `total`, `used`,
+   * `idle`, `waiting`). Use `used / max` for capacity and `waiting > 0` for
+   * saturation. Do not use `used / (used + idle)` as a capacity ratio.
+   */
+  poolConnections: ObservableGauge
+}
+
 export interface OtelInstance {
   auth: AuthMetrics
   engagement: EngagementMetrics
   revenue: RevenueMetrics
   genAi: GenAiMetrics
   gateway: GatewayMetrics
+  database: DatabaseMetrics
   email: EmailMetrics
   rateLimit: RateLimitMetrics
   observability: ObservabilityMetrics
@@ -464,6 +475,12 @@ export function initOtel(env: Env): OtelInstance | null {
     }),
   }
 
+  const database: DatabaseMetrics = {
+    poolConnections: meter.createObservableGauge(METRIC_AIRI_DB_POOL_CONNECTIONS, {
+      description: 'Local pg pool connections by capacity, use, idle, and waiting state',
+    }),
+  }
+
   // NOTICE:
   // OTel SDK only emits a Counter time series after .add() runs the first time.
   // Without this priming step, low-traffic counters (auth_failures_total,
@@ -515,7 +532,7 @@ export function initOtel(env: Env): OtelInstance | null {
   ]
   for (const counter of counters) counter.add(0)
 
-  return { auth, engagement, revenue, genAi, gateway, email, rateLimit, observability }
+  return { auth, engagement, revenue, genAi, gateway, database, email, rateLimit, observability }
 }
 
 const severityMap: Record<string, SeverityNumber> = {

server/apps/api/src/services/domain/llm-router/router.ts

@@ -421,7 +421,6 @@ export function createLlmRouterService(options: CreateLlmRouterServiceOptions) {
         return { kind: 'ok' as const, response: result.response }
       }
 
-      options.gatewayMetrics?.keyExhaustedCount.add(1, { provider })
       return {
         kind: 'exhausted' as const,
         statuses: result.failures.map(failure => failure.status),
@@ -489,6 +488,12 @@ export function createLlmRouterService(options: CreateLlmRouterServiceOptions) {
       throw new Error(`Router exhausted with no recorded failures for model ${req.modelName}`)
     }
 
+    options.gatewayMetrics?.keyExhaustedCount.add(1, {
+      provider: lastFailure.provider,
+      status_code: typeof lastFailure.status === 'number' ? lastFailure.status : 'timeout',
+      surface: 'chat',
+    })
+
     // Same-status exhaustion: every recorded failure shares one status (or
     // timeout). This is a strong signal of a shared upstream constraint that
     // ordinary candidate fallback cannot recover from.
@@ -840,7 +845,6 @@ export function createLlmRouterService(options: CreateLlmRouterServiceOptions) {
         }
       }
 
-      options.gatewayMetrics?.keyExhaustedCount.add(1, { provider: providerTag })
       return {
         kind: 'exhausted',
         sawTooManyRequests: result.failures.some(f => f.status === 429),
@@ -935,6 +939,12 @@ export function createLlmRouterService(options: CreateLlmRouterServiceOptions) {
       throw new Error(`Router exhausted with no recorded failures for tts model ${req.modelName}`)
     }
 
+    options.gatewayMetrics?.keyExhaustedCount.add(1, {
+      provider: lastFailure.provider,
+      status_code: typeof lastFailure.status === 'number' ? lastFailure.status : 'timeout',
+      surface: 'tts',
+    })
+
     const distinctStatuses = new Set(allFailures.map(f => f.status))
     if (distinctStatuses.size === 1) {
       const status = allFailures[0].status

server/apps/api/src/services/domain/llm-router/tests/router.test.ts

@@ -310,7 +310,7 @@ describe('createLlmRouterService', () => {
     expect((metrics.keyExhaustedCount.add as ReturnType<typeof vi.fn>).mock.calls.length).toBe(0)
   })
 
-  it('cross-upstream fallback: upstream A keys all 401, upstream B[0] = 200 → returns 200, A exhaustion counted', async () => {
+  it('cross-upstream fallback: upstream A keys all 401, upstream B[0] = 200 → returns 200 without terminal exhaustion', async () => {
     const { config, crypto } = makeConfig({
       upstreams: [
         { baseURL: 'https://up-a.example/v1', keyIds: ['kA1', 'kA2'] },
@@ -336,7 +336,7 @@ describe('createLlmRouterService', () => {
     expect(res.status).toBe(200)
     expect(fetchImpl.mock.calls.length).toBe(3)
 
-    expect((metrics.keyExhaustedCount.add as ReturnType<typeof vi.fn>).mock.calls.length).toBe(1)
+    expect((metrics.keyExhaustedCount.add as ReturnType<typeof vi.fn>).mock.calls.length).toBe(0)
     expect((metrics.fallbackCount.add as ReturnType<typeof vi.fn>).mock.calls.length).toBe(2)
   })
 
@@ -370,7 +370,14 @@ describe('createLlmRouterService', () => {
       expect((err as ApiError).details).toMatchObject({ triedKeys: 2, triedUpstreams: 2, lastStatusCode: 401 })
     }
 
-    expect((metrics.keyExhaustedCount.add as ReturnType<typeof vi.fn>).mock.calls.length).toBe(2)
+    const exhaustionCalls = (metrics.keyExhaustedCount.add as ReturnType<typeof vi.fn>).mock.calls
+    expect(exhaustionCalls.length).toBe(1)
+    expect(exhaustionCalls[0][0]).toBe(1)
+    expect(exhaustionCalls[0][1]).toMatchObject({
+      provider: 'up-b.example',
+      status_code: 401,
+      surface: 'chat',
+    })
   })
 
   it('full exhaustion attaches per-attempt cause (bodySnippet for HTTP, errorMessage for network) so operators can debug 502s', async () => {
@@ -968,6 +975,41 @@ describe('createLlmRouterService', () => {
       expect(fallbackCalls[0][1]).toMatchObject({ reason: '401' })
     })
 
+    it('records a terminal TTS exhaustion with its final status', async () => {
+      const { config, crypto } = makeTtsConfig({
+        upstreams: [{
+          baseURL: 'https://az.example',
+          keyIds: ['kA1'],
+          adapterParams: { region: 'eastasia' },
+        }],
+      })
+      const fetchImpl = vi.fn(async () => failResponse(451))
+      const metrics = makeMetrics()
+
+      const router = createLlmRouterService({
+        configKV: makeConfigKV(config),
+        envelopeCrypto: crypto,
+        gatewayMetrics: metrics,
+        fetchImpl,
+        redis: makeRedisStub(),
+        concurrencyLedger: makeLedger(),
+      })
+
+      await expect(router.routeTts({
+        modelName: 'tts-test',
+        input: { text: 'hi', voice: 'en-US-AvaMultilingualNeural' },
+      })).rejects.toMatchObject({ statusCode: 502 })
+
+      const exhaustionCalls = (metrics.keyExhaustedCount.add as ReturnType<typeof vi.fn>).mock.calls
+      expect(exhaustionCalls.length).toBe(1)
+      expect(exhaustionCalls[0][0]).toBe(1)
+      expect(exhaustionCalls[0][1]).toMatchObject({
+        provider: 'az.example',
+        status_code: 451,
+        surface: 'tts',
+      })
+    })
+
     it('listTtsVoices deduplicates concurrent cold-cache upstream fetches per model', async () => {
       // ROOT CAUSE:
       //

server/apps/api/src/utils/observability.ts

@@ -85,6 +85,9 @@ export const METRIC_AIRI_TTS_PREFLIGHT_REJECTIONS = 'airi.billing.tts.preflight_
 // AIRI observability — self-monitoring for the metric pipeline
 export const METRIC_AIRI_OBSERVABILITY_READ_ERRORS = 'airi.observability.read_errors'
 
+// AIRI database — local pg pool capacity and queue state
+export const METRIC_AIRI_DB_POOL_CONNECTIONS = 'airi.db.pool.connections'
+
 // AIRI revenue — actual money in (smallest currency unit, e.g. cents)
 export const METRIC_AIRI_STRIPE_REVENUE = 'airi.stripe.revenue'