From f2897f7663f20bb0e42004285e757868d48383c1 Mon Sep 17 00:00:00 2001 From: RainbowBird Date: Sat, 30 May 2026 21:20:36 +0800 Subject: [PATCH] feat(server): total user metrics --- .../airi-server-overview-cloud.json | 27 ++++- apps/server/otel/grafana/dashboards/build.ts | 9 +- apps/server/src/app.ts | 7 +- .../src/otel/gauges/total-users.test.ts | 106 ++++++++++++++++++ apps/server/src/otel/gauges/total-users.ts | 81 +++++++++++++ apps/server/src/otel/index.ts | 17 +++ apps/server/src/utils/observability.ts | 1 + 7 files changed, 239 insertions(+), 9 deletions(-) create mode 100644 apps/server/src/otel/gauges/total-users.test.ts create mode 100644 apps/server/src/otel/gauges/total-users.ts diff --git a/apps/server/otel/grafana/dashboards/airi-server-overview-cloud.json b/apps/server/otel/grafana/dashboards/airi-server-overview-cloud.json index ed23f2a5a..190246080 100644 --- a/apps/server/otel/grafana/dashboards/airi-server-overview-cloud.json +++ b/apps/server/otel/grafana/dashboards/airi-server-overview-cloud.json @@ -41,23 +41,42 @@ "group": "prometheus", "kind": "DataQuery", "spec": { - "expr": "sum(increase(user_registered_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[24h]))", - "legendFormat": "new users" + "expr": "max(user_total{service_name=~\"$service\", deployment_environment=~\"$env\"})", + "legendFormat": "total users" }, "version": "v0" }, "refId": "A" } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "grafanacloud-projairi-prom" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum(increase(user_registered_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[24h]))", + "legendFormat": "new today" + }, + "version": "v0" + }, + "refId": "B" + } } ], "queryOptions": {}, "transformations": [] } }, - "description": "Rolling 24h `increase(user.registered)` — counts the Better Auth `databaseHooks.user.create.after` fires over the last 24 hours. The signup half of the funnel; the returning-user half is DAU / WAU / MAU in the Users & Engagement row.", + "description": "Current Better Auth user table size from `user.total` (cluster-wide DB gauge, aggregate with `max()`) plus rolling 24h signup delta from `increase(user.registered)`. Use the delta as today/new-user growth, and DAU / WAU / MAU below for returning-user engagement.", "id": 1, "links": [], - "title": "New Users 24h", + "title": "Total Users", "vizConfig": { "group": "stat", "kind": "VizConfig", diff --git a/apps/server/otel/grafana/dashboards/build.ts b/apps/server/otel/grafana/dashboards/build.ts index e4169c13b..a0ca4036d 100644 --- a/apps/server/otel/grafana/dashboards/build.ts +++ b/apps/server/otel/grafana/dashboards/build.ts @@ -447,9 +447,12 @@ const elements: Record = {} // range the viewer picked. Trends live in their own rows below. elements['panel-1'] = statPanel( 1, - 'New Users 24h', - 'Rolling 24h `increase(user.registered)` — counts the Better Auth `databaseHooks.user.create.after` fires over the last 24 hours. The signup half of the funnel; the returning-user half is DAU / WAU / MAU in the Users & Engagement row.', - [query(`sum(increase(user_registered_total{${SERVICE_FILTER}}[24h]))`, 'new users')], + 'Total Users', + 'Current Better Auth user table size from `user.total` (cluster-wide DB gauge, aggregate with `max()`) plus rolling 24h signup delta from `increase(user.registered)`. Use the delta as today/new-user growth, and DAU / WAU / MAU below for returning-user engagement.', + [ + query(`max(user_total{${SERVICE_FILTER}})`, 'total users', 'A'), + query(`sum(increase(user_registered_total{${SERVICE_FILTER}}[24h]))`, 'new today', 'B'), + ], { unit: 'short', variant: 'count' }, ) diff --git a/apps/server/src/app.ts b/apps/server/src/app.ts index 5c69f7b4a..0589b1b8f 100644 --- a/apps/server/src/app.ts +++ b/apps/server/src/app.ts @@ -49,6 +49,7 @@ import { emitOtelLog, initOtel } from './otel' import { registerActiveSessionsGauge } from './otel/gauges/active-sessions' import { registerDistinctActiveUsersGauge } from './otel/gauges/distinct-active-users' import { registerRollingActiveUsersGauge } from './otel/gauges/rolling-active-users' +import { registerTotalUsersGauge } from './otel/gauges/total-users' import { createAdminRouterConfigRoutes } from './routes/admin/config/router' import { createAdminFluxGrantsRoutes } from './routes/admin/flux-grants' import { createAdminUsersRoutes } from './routes/admin/users' @@ -693,8 +694,9 @@ export async function createApp() { posthog, }) // Register the cluster-wide ObservableGauges for sessions / users. Each - // replica polls the same DB (cached 10s, in-flight coalesced); dashboards - // aggregate with avg(), not sum(). See observability-conventions.md. + // replica polls the same DB (cached inside each gauge, in-flight coalesced); + // dashboards aggregate with avg()/max(), not sum(). See + // observability-conventions.md. // // Both gauges share the same `session` table: `user.active_sessions` is // `COUNT(*)` (row inflation prone), `user.distinct_active` is @@ -702,6 +704,7 @@ export async function createApp() { // surfaces session-row leakage from missing GC + per-OIDC-token row // creation. if (resolved.otel) { + registerTotalUsersGauge(resolved.otel.auth.totalUsers, resolved.db, resolved.otel.observability.metricReadErrors) registerActiveSessionsGauge(resolved.otel.auth.activeSessions, resolved.db, resolved.otel.observability.metricReadErrors) registerDistinctActiveUsersGauge(resolved.otel.auth.distinctActiveUsers, resolved.db, resolved.otel.observability.metricReadErrors) registerRollingActiveUsersGauge(resolved.otel.auth.rollingActiveUsers, resolved.db, resolved.otel.observability.metricReadErrors) diff --git a/apps/server/src/otel/gauges/total-users.test.ts b/apps/server/src/otel/gauges/total-users.test.ts new file mode 100644 index 000000000..4f71677ed --- /dev/null +++ b/apps/server/src/otel/gauges/total-users.test.ts @@ -0,0 +1,106 @@ +import type { AuthMetrics, ObservabilityMetrics } from '..' +import type { Database } from '../../libs/db' + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +import { registerTotalUsersGauge } from './total-users' + +/** + * Capture the callback registered via `gauge.addCallback` and a spyable + * `observe` so tests can drive collection cycles by hand. + * + * @example + * const { gauge, observe, run } = makeGauge() + * registerTotalUsersGauge(gauge, db, errs) + * await run() + * expect(observe).toHaveBeenCalledWith(42) + */ +function makeGauge() { + let cb: ((result: { observe: (v: number) => void }) => void | Promise) | null = null + const observe = vi.fn() + const gauge = { + addCallback: vi.fn((fn: typeof cb) => { cb = fn }), + } as unknown as AuthMetrics['totalUsers'] + return { + gauge, + observe, + run: async () => { + if (!cb) + throw new Error('no callback registered') + await cb({ observe }) + }, + } +} + +/** + * Drizzle's `db.select({...}).from(table)` is awaited directly (thenable + * query builder). Mock it as `select -> { from: () => Promise }`. + * + * @example + * const db = makeDb([{ count: '12' }]) + */ +function makeDb(rows: unknown[], opts: { reject?: boolean } = {}) { + const from = vi.fn(() => (opts.reject ? Promise.reject(new Error('db down')) : Promise.resolve(rows))) + const select = vi.fn(() => ({ from })) + return { db: { select } as unknown as Database, select, from } +} + +function makeReadErrors() { + const add = vi.fn() + return { metricReadErrors: { add } as unknown as ObservabilityMetrics['metricReadErrors'], add } +} + +describe('registerTotalUsersGauge', () => { + beforeEach(() => { + vi.useFakeTimers() + }) + afterEach(() => { + vi.useRealTimers() + vi.restoreAllMocks() + }) + + it('observes the total user count from the user table', async () => { + const { db } = makeDb([{ count: '42' }]) + const { metricReadErrors } = makeReadErrors() + const { gauge, observe, run } = makeGauge() + + registerTotalUsersGauge(gauge, db, metricReadErrors) + await run() + + expect(observe).toHaveBeenCalledWith(42) + }) + + it('serves cached values without re-querying inside the 60s TTL', async () => { + const { db, select } = makeDb([{ count: '3' }]) + const { metricReadErrors } = makeReadErrors() + const { gauge, observe, run } = makeGauge() + + registerTotalUsersGauge(gauge, db, metricReadErrors) + await run() + await vi.advanceTimersByTimeAsync(30_000) + await run() + + expect(select).toHaveBeenCalledTimes(1) + expect(observe).toHaveBeenCalledTimes(2) + }) + + it('on DB error: increments read-errors and does not observe', async () => { + // ROOT CAUSE: + // + // A total-user gauge that observes a stale/zero value on DB failure masks + // the outage forever — an absence-based alert can never fire. + // + // We fixed this by skipping result.observe(...) on error and bumping + // airi.observability.read_errors{metric}, so Prometheus staleness exposes + // the broken DB instead. + const { db } = makeDb([], { reject: true }) + const { metricReadErrors, add } = makeReadErrors() + const { gauge, observe, run } = makeGauge() + + registerTotalUsersGauge(gauge, db, metricReadErrors) + await run() + + expect(observe).not.toHaveBeenCalled() + expect(add).toHaveBeenCalledWith(1, { metric: 'user.total' }) + }) +}) diff --git a/apps/server/src/otel/gauges/total-users.ts b/apps/server/src/otel/gauges/total-users.ts new file mode 100644 index 000000000..df558473d --- /dev/null +++ b/apps/server/src/otel/gauges/total-users.ts @@ -0,0 +1,81 @@ +import type { AuthMetrics, ObservabilityMetrics } from '..' +import type { Database } from '../../libs/db' + +import { useLogger } from '@guiiai/logg' +import { count } from 'drizzle-orm' + +import { user as userTable } from '../../schemas/accounts' + +/** + * Wire the `user.total` ObservableGauge to a Postgres `COUNT(*)` over the + * Better Auth user table. + * + * Use when: + * - Assembling DI in `createApp()`, exactly once per process. + * + * Expects: + * - `gauge` is the ObservableGauge handle created in `initOtel`. + * - `db` is the migrated Drizzle handle. + * - `metricReadErrors` is the shared counter used to track failures inside + * metric callbacks. + * + * Multi-replica note: + * - Cluster-wide gauge — every replica reads the same DB and reports the same + * value. Dashboards MUST aggregate with `max()`/`avg()`, NOT `sum()`. + * + * Concurrency: + * - Same in-flight promise lock as the sibling DB-backed gauges, so concurrent + * OTel collection cycles fold into one DB query. + * + * Failure mode: + * - DB error → increment `airi.observability.read_errors{metric}` and skip + * `result.observe(...)`. Prometheus staleness exposes the outage instead of + * pinning stale values forever. + */ +export function registerTotalUsersGauge( + gauge: AuthMetrics['totalUsers'], + db: Database, + metricReadErrors: ObservabilityMetrics['metricReadErrors'], +) { + const log = useLogger('total-users-gauge').useGlobalConfig() + const CACHE_TTL_MS = 60_000 + + let cachedAt = 0 + let cachedCount = 0 + let refreshInFlight: Promise | null = null + + async function refresh(): Promise { + try { + const rows = await db + .select({ count: count() }) + .from(userTable) + cachedCount = Number(rows[0]?.count ?? 0) + cachedAt = Date.now() + return true + } + catch (err) { + log.withError(err).warn('Failed to read total users for gauge') + metricReadErrors.add(1, { metric: 'user.total' }) + return false + } + } + + gauge.addCallback(async (result) => { + const now = Date.now() + + if (cachedAt !== 0 && now - cachedAt < CACHE_TTL_MS) { + result.observe(cachedCount) + return + } + + if (!refreshInFlight) { + refreshInFlight = refresh().finally(() => { + refreshInFlight = null + }) + } + const ok = await refreshInFlight + + if (ok) + result.observe(cachedCount) + }) +} diff --git a/apps/server/src/otel/index.ts b/apps/server/src/otel/index.ts index 0977e1d35..d750a55e7 100644 --- a/apps/server/src/otel/index.ts +++ b/apps/server/src/otel/index.ts @@ -57,6 +57,7 @@ import { METRIC_USER_DISTINCT_ACTIVE, METRIC_USER_LOGIN, METRIC_USER_REGISTERED, + METRIC_USER_TOTAL, METRIC_WS_CONNECTIONS_ACTIVE, METRIC_WS_MESSAGES_RECEIVED, METRIC_WS_MESSAGES_SENT, @@ -69,6 +70,19 @@ export interface AuthMetrics { failures: Counter userRegistered: Counter userLogin: Counter + /** + * Pull-based gauge for total registered users. + * + * Use when: + * - Reporting current account-base size. Pair with + * {@link AuthMetrics.userRegistered} for signup deltas over a time window. + * + * Expects: + * - Backed by `SELECT COUNT(*) FROM "user"`. Same cluster-wide truth as the + * other DB-backed gauges; dashboards MUST aggregate with `max()`/`avg()`, + * not `sum()`. + */ + totalUsers: ObservableGauge /** * Cluster-wide active session count, sourced from Postgres (Better Auth * `session` table where `expires_at > NOW()`). @@ -335,6 +349,9 @@ export function initOtel(env: Env): OtelInstance | null { userLogin: meter.createCounter(METRIC_USER_LOGIN, { description: 'Number of user sign-ins', }), + totalUsers: meter.createObservableGauge(METRIC_USER_TOTAL, { + description: 'Total registered users sourced from Postgres (cluster-wide; dashboard must use max(), not sum())', + }), activeSessions: meter.createObservableGauge(METRIC_USER_ACTIVE_SESSIONS, { description: 'Active user sessions sourced from Postgres (cluster-wide; dashboard must use avg(), not sum())', }), diff --git a/apps/server/src/utils/observability.ts b/apps/server/src/utils/observability.ts index c9e13cc32..1ca596b06 100644 --- a/apps/server/src/utils/observability.ts +++ b/apps/server/src/utils/observability.ts @@ -47,6 +47,7 @@ export const METRIC_AUTH_ATTEMPTS = 'auth.attempts' export const METRIC_AUTH_FAILURES = 'auth.failures' export const METRIC_USER_REGISTERED = 'user.registered' export const METRIC_USER_LOGIN = 'user.login' +export const METRIC_USER_TOTAL = 'user.total' export const METRIC_USER_ACTIVE_SESSIONS = 'user.active_sessions' // Distinct users with at least one non-expired session row. Pair with // USER_ACTIVE_SESSIONS to detect "session row inflation" (Better Auth