fix(server): correct observability dashboard signals (#2142)

2026-07-28 21:48:03 +08:00
parent ae35b580d8
commit acbbab0bd7
13 changed files with 426 additions and 360 deletions
@@ -101,7 +101,7 @@
 | 域 | 代表性指标 | Truth | 备注 |
 |---|---|---|---|
 | HTTP | `http_server_request_duration_seconds_*` | Grafana | OTel 标准 |
-| WS | `ws_connections_active` / `ws_messages_*_total` | Grafana | |
+| WS | `ws_users_online` / `ws_connections_active` / `ws_messages_*_total` | Grafana | `ws_users_online` 是 Redis Pub/Sub channel 去重后的集群在线用户数；连接数仍用于排查多标签页和连接泄漏 |
 | LLM | `gen_ai_client_operation_count_total` / `gen_ai_client_first_token_duration_seconds` | Grafana | |
 | Billing | `airi_billing_flux_unbilled_total` | Grafana | **告警必须**：`increase(airi_billing_flux_unbilled_total[5m]) > 0` |
 | Auth | `user_active_sessions` / `user_distinct_active` | Postgres → Grafana 派生 | 集群级 gauge，用 `avg()` 不要 `sum()`。两个一起看：`user_active_sessions` = `COUNT(*)`（session row 数，会膨胀）, `user_distinct_active` = `COUNT(DISTINCT user_id)`（真实活跃用户数）|
@@ -219,9 +219,8 @@ PostHog UI 配 cohort：
 Dashboard 上 follow 这条 panel 链可以从"出事了"一路 drill 到"哪个 trace 是真凶"：

 1. **panel-4 `5xx Rate %`**（Row 1）— 数字 / gauge 颜色变红，说明出事
-2. **panel-9 `Top Routes by 5xx`**（Row 2 donut）— "现在哪些 route 在失败"
-3. **panel-44 `5xx Rate by Route`**（Row 5.5 timeseries）— "什么时候开始的、是单点还是普遍"
-4. **panel-91 `5xx Error Logs`**（Row 8 上半）— 实际错误消息，里面有 `trace_id` field 可点 → Tempo 看完整 trace 回放
+2. **panel-94 `Errors by Route`**（HTTP row）— "什么时候开始的、哪些 route / status 在失败"
+3. **panel-91 `Warn / Error Logs`**（Logs row）— 实际 warn/error 消息，里面有 `trace_id` field 可点 → Tempo 看完整 trace 回放

 ### Tempo / Loki derived fields 配置（一次性）

@@ -196,7 +196,7 @@ span name 目前允许保留业务可读格式，例如：
 | `Counter` | ✅ | `sum(rate(x[5m]))` | 每副本本地累加，`rate()` 自动处理重启 |
 | `Histogram` | ✅ | `histogram_quantile(0.95, sum by (le, ...) (rate(x_bucket[5m])))` | 每副本本地 bucket，`sum by (le)` 合并 |
 | `ObservableGauge`（**per-replica 状态**，如 `ws.connections.active`） | ✅ | `sum(x)` | callback 读本地 registry，所有副本求和 = 集群总量 |
-| `ObservableGauge`（**cluster-wide 状态**，如 `user.active_sessions`） | ⚠️ | `max(x)` 或 `avg(x)` | 所有副本读同一份外部状态（DB），sum 会乘以副本数 |
+| `ObservableGauge`（**cluster-wide 状态**，如 `user.active_sessions` / `ws.users.online`） | ⚠️ | `max(x)` 或 `avg(x)` | 所有副本读同一份外部状态（DB / Redis），sum 会乘以副本数 |
 | `UpDownCounter` | ⚠️ | 看场景 | 必须保证 `+1` 和 `-1` 在**同一副本**触发；否则单副本永久 +N 另一副本永久 -N |

 ### `UpDownCounter` 红线
@@ -233,10 +233,12 @@ span name 目前允许保留业务可读格式，例如：
 | 集群总量（per-replica gauge） | `sum(x{...})` |
 | 集群唯一值（cluster-wide gauge） | `max(x{...})` 或 `avg(x{...})` |
 | 按副本拆分调试 | `<agg> by (service_instance_id) (x{...})` |
-| 错误率 | `100 * sum(rate(x_total{...,status_code=~"5.."}[5m])) / clamp_min(sum(rate(x_total{...}[5m])), 1)` |
+| 错误率 | `100 * sum(rate(x_total{...,status_code=~"5.."}[5m])) / sum(rate(x_total{...}[5m]))` |

 红线：**任何 cumulative counter 都不能直接 `sum()` 不 wrap rate/increase**。Counter 在副本重启时归零，没有 rate() 包裹 Prometheus 会跳变；用 `increase($__range)` 看「时间窗口内总量」，用 `rate([interval])` 看「当前速率」。

+比例的分母也不能用 `clamp_min(rate(...), 1)` 防零：这会在流量低于 1 req/s 时系统性低估错误率。零流量应由 Grafana 的 no-value 展示策略处理，不应修改真实分母。
+
 ### 「按副本拆分」何时加

 默认 panel 都聚合到集群层面。但以下场景应该加 `by (service_instance_id)` 拆分图：
@@ -57,9 +57,12 @@ OTel SDK 在导出到 Prometheus 时做两件事：
 | `character.deleted` | Counter | 同上 | — |
 | `character.engagement` | Counter | 同上（like/bookmark） | `action`（`like` / `unlike` / `bookmark` / `unbookmark`） |
 | `ws.connections.active` | ObservableGauge | [routes/chat-ws/index.ts](../../src/routes/chat-ws/index.ts) `addCallback` walks `userConnections` Map | — |
+| `ws.users.online` | ObservableGauge | [otel/gauges/ws-online-users.ts](../../src/otel/gauges/ws-online-users.ts) counts unique active `user:*:chat:broadcast` Redis Pub/Sub channels | — |
 | `ws.messages.sent` | Counter | 同上 | — |
 | `ws.messages.received` | Counter | [services/domain/chats.ts](../../src/services/domain/chats.ts) | — |

+> `ws.users.online` 是 cluster-wide gauge：同一用户无论打开多少标签页、连接到多少个 Server 副本，Redis 都只返回一个活跃 channel。每个副本读取并上报同一个全局值，因此 Dashboard 必须使用 `max()` / `avg()`，不能使用 `sum()`。
+
 ## Product Analytics

 | Metric | 类型 | 落点 | Labels |
@@ -142,7 +145,7 @@ OTel SDK 在导出到 Prometheus 时做两件事：

 | Row | viz | 关键 metric |
 |---|---|---|
-| Service Health | stat / gauge / heatmap | `user.total`（`max()`）、`user.active_sessions`（`avg()`）、`ws.connections.active`（`sum()`）、`http.server.request.duration_count`（req/s + 5xx%）、`gen_ai.client.operation.count` |
+| Service Health | stat / gauge / timeseries | `user.total`（`max()`）、`user.active_sessions`（`avg()`）、`ws.users.online`（`max()`）、`ws.connections.active`（`sum()` trend）、`http.server.request.duration_count`（req/s + 5xx%）、`gen_ai.client.operation.count` |
 | User Engagement | stat | `user.active_rolling`（DAU / WAU / MAU，`max()`） |
 | Product Analytics | stat / gauge / bargauge / timeseries | `airi.product.events`（Prom-safe event volume by `feature` / `action` / `status`；distinct users 仍查 Postgres `product_events`） |
 | HTTP | heatmap / bargauge / timeseries | `http.server.request.duration_count`（status mix、top routes、route errors）、`http.server.request.duration_bucket`（P95 by route） |
@@ -319,7 +319,7 @@
                    "kind": "DataQuery",
                    "spec": {
                      "editorMode": "code",
-                      "expr": "100 * sum(rate(http_server_request_duration_seconds_count{service_name=~\"$service\", deployment_environment=~\"$env\", http_request_method!=\"OPTIONS\", http_response_status_code=~\"5..\"}[5m])) / clamp_min(sum(rate(http_server_request_duration_seconds_count{service_name=~\"$service\", deployment_environment=~\"$env\", http_request_method!=\"OPTIONS\"}[5m])), 1)",
+                      "expr": "100 * sum(rate(http_server_request_duration_seconds_count{service_name=~\"$service\", deployment_environment=~\"$env\", http_request_method!=\"OPTIONS\", http_response_status_code=~\"5..\"}[5m])) / sum(rate(http_server_request_duration_seconds_count{service_name=~\"$service\", deployment_environment=~\"$env\", http_request_method!=\"OPTIONS\"}[5m]))",
                      "legendFormat": "fail %",
                      "range": true
                    },
@@ -365,6 +365,7 @@
                },
                "unit": "percent",
                "decimals": 2,
+                "noValue": "0",
                "min": 0,
                "max": 10
              },
@@ -472,90 +473,6 @@
        }
      }
    },
-    "panel-80": {
-      "kind": "Panel",
-      "spec": {
-        "data": {
-          "kind": "QueryGroup",
-          "spec": {
-            "queries": [
-              {
-                "kind": "PanelQuery",
-                "spec": {
-                  "hidden": false,
-                  "query": {
-                    "datasource": {
-                      "name": "grafanacloud-projairi-prom"
-                    },
-                    "group": "prometheus",
-                    "kind": "DataQuery",
-                    "spec": {
-                      "editorMode": "code",
-                      "expr": "max(user_active_rolling{service_name=~\"$service\", deployment_environment=~\"$env\", window=\"24h\"})",
-                      "legendFormat": "DAU",
-                      "range": true
-                    },
-                    "version": "v0"
-                  },
-                  "refId": "A"
-                }
-              }
-            ],
-            "queryOptions": {},
-            "transformations": []
-          }
-        },
-        "description": "Daily active users — distinct users with activity in the last 24h. Sourced from `user.last_seen_at` (touched on sign-in and every OIDC token refresh) via the `user.active_rolling` gauge. Cluster-wide gauge aggregated with `max()`.",
-        "id": 80,
-        "links": [],
-        "title": "DAU",
-        "vizConfig": {
-          "group": "stat",
-          "kind": "VizConfig",
-          "spec": {
-            "fieldConfig": {
-              "defaults": {
-                "color": {
-                  "mode": "fixed",
-                  "fixedColor": "blue"
-                },
-                "fieldMinMax": false,
-                "thresholds": {
-                  "mode": "absolute",
-                  "steps": [
-                    {
-                      "color": "blue",
-                      "value": 0
-                    }
-                  ]
-                },
-                "unit": "short",
-                "noValue": "0"
-              },
-              "overrides": []
-            },
-            "options": {
-              "colorMode": "none",
-              "graphMode": "area",
-              "justifyMode": "auto",
-              "orientation": "auto",
-              "percentChangeColorMode": "standard",
-              "reduceOptions": {
-                "calcs": [
-                  "lastNotNull"
-                ],
-                "fields": "",
-                "values": false
-              },
-              "showPercentChange": true,
-              "textMode": "value_and_name",
-              "wideLayout": true
-            }
-          },
-          "version": "13.2.0-28666480772"
-        }
-      }
-    },
    "panel-81": {
      "kind": "Panel",
      "spec": {
@@ -743,9 +660,10 @@
                    "kind": "DataQuery",
                    "spec": {
                      "editorMode": "code",
-                      "expr": "sum(ws_connections_active{service_name=~\"$service\", deployment_environment=~\"$env\"})",
-                      "legendFormat": "online",
-                      "range": true
+                      "expr": "max(ws_users_online{service_name=~\"$service\", deployment_environment=~\"$env\"})",
+                      "legendFormat": "users",
+                      "instant": true,
+                      "range": false
                    },
                    "version": "v0"
                  },
@@ -757,10 +675,10 @@
            "transformations": []
          }
        },
-        "description": "Current concurrent WebSocket connections across all replicas (`sum` — each replica holds its own connections). The live-presence counterpart to the rolling DAU/WAU windows.",
+        "description": "Cluster-wide distinct authenticated users with at least one active `/ws/chat` connection. Redis returns each per-user broadcast channel once even when that user has multiple tabs or connections across server replicas; every replica reports the same global value, so the query uses `max()`.",
        "id": 93,
        "links": [],
-        "title": "WS Online",
+        "title": "Online Users",
        "vizConfig": {
          "group": "stat",
          "kind": "VizConfig",
@@ -782,7 +700,7 @@
                  ]
                },
                "unit": "short",
-                "noValue": "0"
+                "noValue": "—"
              },
              "overrides": []
            },
@@ -1326,8 +1244,8 @@
                    "kind": "DataQuery",
                    "spec": {
                      "editorMode": "code",
-                      "expr": "max (user_active_rolling{service_name=~\"$service\", deployment_environment=~\"$env\", window=\"24h\"})",
-                      "legendFormat": "__auto",
+                      "expr": "max(user_active_rolling{service_name=~\"$service\", deployment_environment=~\"$env\", window=\"24h\"})",
+                      "legendFormat": "DAU",
                      "range": true
                    },
                    "version": "v0"
@@ -1340,10 +1258,10 @@
            "transformations": []
          }
        },
-        "description": "",
+        "description": "Rolling 24-hour distinct active users over time. This is the trend view of `user.active_rolling`; the duplicate point-in-time DAU stat is intentionally omitted from this dashboard.",
        "id": 99,
        "links": [],
-        "title": "日活",
+        "title": "DAU Trend",
        "vizConfig": {
          "group": "timeseries",
          "kind": "VizConfig",
@@ -1351,8 +1269,7 @@
            "fieldConfig": {
              "defaults": {
                "color": {
-                  "mode": "continuous-GrYlRd",
-                  "seriesBy": "last"
+                  "mode": "palette-classic"
                },
                "custom": {
                  "axisBorderShow": false,
@@ -1364,19 +1281,16 @@
                  "barWidthFactor": 0.6,
                  "drawStyle": "line",
                  "fillOpacity": 17,
-                  "gradientMode": "scheme",
+                  "gradientMode": "none",
                  "hideFrom": {
                    "legend": false,
                    "tooltip": false,
                    "viz": false
                  },
                  "insertNulls": false,
-                  "lineInterpolation": "linear",
-                  "lineStyle": {
-                    "fill": "solid"
-                  },
-                  "lineWidth": 2,
-                  "pointSize": 3,
+                  "lineInterpolation": "smooth",
+                  "lineWidth": 1,
+                  "pointSize": 5,
                  "scaleDistribution": {
                    "type": "linear"
                  },
@@ -1397,13 +1311,10 @@
                    {
                      "color": "green",
                      "value": 0
-                    },
-                    {
-                      "color": "red",
-                      "value": 80
                    }
                  ]
-                }
+                },
+                "unit": "short"
              },
              "overrides": []
            },
@@ -1413,8 +1324,11 @@
                "multiLane": false
              },
              "legend": {
-                "calcs": [],
-                "displayMode": "list",
+                "calcs": [
+                  "lastNotNull",
+                  "max"
+                ],
+                "displayMode": "table",
                "enableFacetedFilter": false,
                "overflow": "ellipsis",
                "placement": "bottom",
@@ -1422,8 +1336,8 @@
              },
              "tooltip": {
                "hideZeros": false,
-                "mode": "single",
-                "sort": "none"
+                "mode": "multi",
+                "sort": "desc"
              }
            }
          },
@@ -1547,27 +1461,63 @@
            "transformations": []
          }
        },
-        "description": "HTTP status-code mix over time, one row per status code, colour = request rate in each time bucket. The 200 row dominates in steady state; a 5xx / 4xx row suddenly lighting up flags an incident at a glance. Non-OPTIONS traffic only.",
+        "description": "Stacked non-OPTIONS request rate by HTTP status code. A new or growing 4xx / 5xx band flags a traffic-quality or service-health change.",
        "id": 40,
        "links": [],
-        "title": "Status Distribution",
+        "title": "HTTP Status Rate",
        "vizConfig": {
-          "group": "heatmap",
+          "group": "timeseries",
          "kind": "VizConfig",
          "spec": {
            "fieldConfig": {
              "defaults": {
+                "color": {
+                  "mode": "palette-classic"
+                },
                "custom": {
+                  "axisBorderShow": false,
+                  "axisCenteredZero": false,
+                  "axisColorMode": "text",
+                  "axisLabel": "",
+                  "axisPlacement": "auto",
+                  "barAlignment": 0,
+                  "barWidthFactor": 0.6,
+                  "drawStyle": "line",
+                  "fillOpacity": 60,
+                  "gradientMode": "none",
                  "hideFrom": {
                    "legend": false,
                    "tooltip": false,
                    "viz": false
                  },
+                  "insertNulls": false,
+                  "lineInterpolation": "smooth",
+                  "lineWidth": 1,
+                  "pointSize": 5,
                  "scaleDistribution": {
                    "type": "linear"
+                  },
+                  "showPoints": "auto",
+                  "showValues": false,
+                  "spanNulls": false,
+                  "stacking": {
+                    "group": "A",
+                    "mode": "normal"
+                  },
+                  "thresholdsStyle": {
+                    "mode": "off"
                  }
                },
-                "unit": "short"
+                "thresholds": {
+                  "mode": "absolute",
+                  "steps": [
+                    {
+                      "color": "green",
+                      "value": 0
+                    }
+                  ]
+                },
+                "unit": "reqps"
              },
              "overrides": []
            },
@@ -1576,37 +1526,21 @@
                "clustering": -1,
                "multiLane": false
              },
-              "calculate": false,
-              "cellGap": 1,
-              "color": {
-                "exponent": 0.5,
-                "fill": "dark-orange",
-                "mode": "scheme",
-                "reverse": false,
-                "scale": "exponential",
-                "scheme": "RdYlBu",
-                "steps": 64
-              },
-              "exemplars": {
-                "color": "rgba(255,0,255,0.7)"
-              },
-              "filterValues": {
-                "le": 1e-9
-              },
              "legend": {
-                "show": false
-              },
-              "rowsFrame": {
-                "layout": "auto"
+                "calcs": [
+                  "lastNotNull",
+                  "max"
+                ],
+                "displayMode": "table",
+                "enableFacetedFilter": false,
+                "overflow": "ellipsis",
+                "placement": "bottom",
+                "showLegend": true
              },
              "tooltip": {
-                "mode": "single",
-                "showColorScale": false,
-                "yHistogram": false
-              },
-              "yAxis": {
-                "axisPlacement": "left",
-                "reverse": false
+                "hideZeros": false,
+                "mode": "multi",
+                "sort": "desc"
              }
            }
          },
@@ -1633,7 +1567,7 @@
                    "kind": "DataQuery",
                    "spec": {
                      "editorMode": "code",
-                      "expr": "sum by (http_route) (\n    rate(http_server_request_duration_seconds_bucket{service_name=~\"$service\", deployment_environment=~\"$env\", http_request_method!=\"OPTIONS\", http_route!~\"/api/v1/openai/.*\", http_response_status_code!=\"404\"}[$__rate_interval])\n)",
+                      "expr": "histogram_quantile(0.95, sum by (le, http_route) (\n  rate(http_server_request_duration_seconds_bucket{service_name=~\"$service\", deployment_environment=~\"$env\", http_request_method!=\"OPTIONS\", http_route!~\"/api/v1/openai/.*\", http_response_status_code!=\"404\"}[$__rate_interval])\n))",
                      "legendFormat": "{{http_route}}",
                      "range": true
                    },
@@ -1647,10 +1581,10 @@
            "transformations": []
          }
        },
-        "description": "",
+        "description": "P95 Hono request duration by matched route. Histogram buckets are merged across replicas while preserving `le`, then interpolated by `histogram_quantile`; values are estimates bounded by the configured bucket widths.",
        "id": 20,
        "links": [],
-        "title": "Request Latency (by Route)",
+        "title": "Request Latency P95 by Route",
        "vizConfig": {
          "group": "timeseries",
          "kind": "VizConfig",
@@ -1873,9 +1807,10 @@
                    "kind": "DataQuery",
                    "spec": {
                      "editorMode": "code",
-                      "expr": "sum by (gen_ai_request_model) (rate(gen_ai_client_operation_count_total{service_name=~\"$service\", deployment_environment=~\"$env\", gen_ai_request_model!=\"\"}[$__rate_interval]))",
+                      "expr": "sum by (gen_ai_request_model) (increase(gen_ai_client_operation_count_total{service_name=~\"$service\", deployment_environment=~\"$env\", gen_ai_request_model!=\"\"}[$__range]))",
                      "legendFormat": "{{gen_ai_request_model}}",
-                      "range": true
+                      "instant": true,
+                      "range": false
                    },
                    "version": "v0"
                  },
@@ -1887,10 +1822,10 @@
            "transformations": []
          }
        },
-        "description": "Per-model request rate (chat + tts). Useful for capacity planning and spotting model-routing regressions.",
+        "description": "Per-model request count over the visible dashboard range (chat + tts). The pie shows each model's share without depending on Grafana sampling resolution.",
        "id": 11,
        "links": [],
-        "title": "LLM Request Rate by Model",
+        "title": "LLM Requests by Model (range)",
        "vizConfig": {
          "group": "piechart",
          "kind": "VizConfig",
@@ -1908,7 +1843,7 @@
                    "viz": false
                  }
                },
-                "unit": "reqps"
+                "unit": "short"
              },
              "overrides": []
            },
@@ -1925,7 +1860,7 @@
              "pieType": "pie",
              "reduceOptions": {
                "calcs": [
-                  "sum"
+                  "lastNotNull"
                ],
                "fields": "",
                "values": false
@@ -1961,7 +1896,7 @@
                    "kind": "DataQuery",
                    "spec": {
                      "editorMode": "code",
-                      "expr": "sum by () (rate(gen_ai_client_first_token_duration_seconds_bucket{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))",
+                      "expr": "histogram_quantile(0.95, sum by (le) (rate(gen_ai_client_first_token_duration_seconds_bucket{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval])))",
                      "legendFormat": "TTFB p95",
                      "range": true
                    },
@@ -2342,7 +2277,7 @@
                    "kind": "DataQuery",
                    "spec": {
                      "editorMode": "code",
-                      "expr": "100 * sum by (provider) (rate(gen_ai_client_operation_count_total{service_name=~\"$service\", deployment_environment=~\"$env\", provider!=\"\", http_response_status_code=~\"4..|5..\"}[$__rate_interval])) / clamp_min(sum by (provider) (rate(gen_ai_client_operation_count_total{service_name=~\"$service\", deployment_environment=~\"$env\", provider!=\"\"}[$__rate_interval])), 1)",
+                      "expr": "100 * sum by (provider) (rate(gen_ai_client_operation_count_total{service_name=~\"$service\", deployment_environment=~\"$env\", provider!=\"\", http_response_status_code=~\"4..|5..\"}[$__rate_interval])) / sum by (provider) (rate(gen_ai_client_operation_count_total{service_name=~\"$service\", deployment_environment=~\"$env\", provider!=\"\"}[$__rate_interval]))",
                      "legendFormat": "{{provider}}",
                      "range": true
                    },
@@ -3176,7 +3111,7 @@
                    "kind": "DataQuery",
                    "spec": {
                      "editorMode": "code",
-                      "expr": "100 * sum(rate(airi_gen_ai_gateway_fallback_count_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[5m])) / clamp_min(sum(rate(gen_ai_client_operation_count_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[5m])), 1)",
+                      "expr": "100 * sum(rate(airi_gen_ai_gateway_fallback_count_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[5m])) / sum(rate(gen_ai_client_operation_count_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[5m]))",
                      "legendFormat": "fallback %",
                      "range": true
                    },
@@ -3647,7 +3582,7 @@
                    "kind": "DataQuery",
                    "spec": {
                      "editorMode": "code",
-                      "expr": "{service_name=~\"$service\", deployment_environment=~\"$env\"} | json | level=~\"warn|error\"",
+                      "expr": "{service_name=~\"$service\", deployment_environment=~\"$env\"} | detected_level=~\"warn|error\"",
                      "legendFormat": "",
                      "range": true
                    },
@@ -3661,10 +3596,10 @@
            "transformations": []
          }
        },
-        "description": "Server-side error logs (level=warn|error) from Loki. Derived fields make `trace_id` and `req` clickable — `trace_id` jumps to Tempo for full request playback.",
+        "description": "Server-side warn and error logs from Loki structured metadata. Derived fields make `trace_id` and `req` clickable — `trace_id` jumps to Tempo for full request playback.",
        "id": 91,
        "links": [],
-        "title": "5xx Error Logs",
+        "title": "Warn / Error Logs",
        "vizConfig": {
          "group": "logs",
          "kind": "VizConfig",
@@ -4015,7 +3950,7 @@
                    "spec": {
                      "element": {
                        "kind": "ElementReference",
-                        "name": "panel-80"
+                        "name": "panel-1"
                      },
                      "height": 5,
                      "width": 3,
@@ -4028,7 +3963,7 @@
                    "spec": {
                      "element": {
                        "kind": "ElementReference",
-                        "name": "panel-1"
+                        "name": "panel-15"
                      },
                      "height": 5,
                      "width": 3,
@@ -4062,32 +3997,6 @@
                      "y": 0
                    }
                  },
-                  {
-                    "kind": "GridLayoutItem",
-                    "spec": {
-                      "element": {
-                        "kind": "ElementReference",
-                        "name": "panel-82"
-                      },
-                      "height": 3,
-                      "width": 3,
-                      "x": 0,
-                      "y": 5
-                    }
-                  },
-                  {
-                    "kind": "GridLayoutItem",
-                    "spec": {
-                      "element": {
-                        "kind": "ElementReference",
-                        "name": "panel-15"
-                      },
-                      "height": 6,
-                      "width": 3,
-                      "x": 3,
-                      "y": 5
-                    }
-                  },
                  {
                    "kind": "GridLayoutItem",
                    "spec": {
@@ -4098,7 +4007,20 @@
                      "height": 3,
                      "width": 3,
                      "x": 0,
-                      "y": 8
+                      "y": 5
+                    }
+                  },
+                  {
+                    "kind": "GridLayoutItem",
+                    "spec": {
+                      "element": {
+                        "kind": "ElementReference",
+                        "name": "panel-82"
+                      },
+                      "height": 3,
+                      "width": 3,
+                      "x": 3,
+                      "y": 5
                    }
                  },
                  {
@@ -4384,7 +4306,7 @@
    "grafana-cloud"
  ],
  "timeSettings": {
-    "autoRefresh": "",
+    "autoRefresh": "30s",
    "autoRefreshIntervals": [
      "5s",
      "10s",
@@ -54,16 +54,12 @@ describe('grafana dashboard builder', () => {
    expect(result.unusedElems).toEqual([])
  })

-  /**
-   * @example
-   * expect(panelTitle('panel-99')).toBe('TTS Success %')
-   */
  it('keeps the product analytics row focused on Prometheus-safe engagement signals', () => {
    expect(panelTitle('panel-95')).toBe('Product Events (range)')
    expect(panelTitle('panel-96')).toBe('Product Failure %')
    expect(panelTitle('panel-97')).toBe('Top Product Actions (range)')
    expect(panelTitle('panel-98')).toBe('Product Event Rate')
-    expect(panelTitle('panel-99')).toBe('日活')
+    expect(panelTitle('panel-99')).toBe('DAU Trend')
  })

  /**
@@ -86,4 +82,75 @@ describe('grafana dashboard builder', () => {
    expect(productPanelExpressions).not.toContain('session_id')
    expect(productPanelExpressions).not.toContain('request_id')
  })
+
+  it('preserves histogram buckets until HTTP and LLM latency quantiles are calculated', () => {
+    // ROOT CAUSE:
+    //
+    // The latency panels previously summed cumulative `_bucket` rates after
+    // dropping `le`. Grafana then labelled that request-rate-derived value as
+    // seconds, making millisecond HTTP routes appear to take 20+ seconds.
+    //
+    // Quantiles must retain `le` while replicas are merged, then call
+    // histogram_quantile over the merged histogram.
+    const httpLatency = collectQueryExpressions(dashboard.elements['panel-20']).join('\n')
+    const llmLatency = collectQueryExpressions(dashboard.elements['panel-21']).join('\n')
+
+    expect(httpLatency).toContain('histogram_quantile(0.95')
+    expect(httpLatency).toContain('sum by (le, http_route)')
+    expect(llmLatency.match(/histogram_quantile\(0\.95/g)).toHaveLength(2)
+    expect(llmLatency).toContain('gen_ai_client_first_token_duration_seconds_bucket')
+    expect(llmLatency).toContain('sum by (le)')
+  })
+
+  it('does not clamp rate denominators to one request per second', () => {
+    // ROOT CAUSE:
+    //
+    // `clamp_min(rate, 1)` changes the denominator whenever traffic is below
+    // 1 req/s, so low-volume provider and fallback failures are underreported.
+    for (const panelName of ['panel-4', 'panel-62', 'panel-68']) {
+      const expressions = collectQueryExpressions(dashboard.elements[panelName]).join('\n')
+
+      expect(expressions).not.toContain('clamp_min')
+    }
+  })
+
+  it('uses visualizations and reductions that match each query shape', () => {
+    const statusPanel = JSON.stringify(dashboard.elements['panel-40'])
+    const modelMixPanel = JSON.stringify(dashboard.elements['panel-11'])
+
+    expect(statusPanel).toContain('"group":"timeseries"')
+    expect(statusPanel).not.toContain('"group":"heatmap"')
+    expect(modelMixPanel).toContain('increase(')
+    expect(modelMixPanel).toContain('"instant":true')
+    expect(modelMixPanel).toContain('"calcs":["lastNotNull"]')
+  })
+
+  it('filters structured Loki severity without parsing plain-text bodies as JSON', () => {
+    const errorLogsPanel = JSON.stringify(dashboard.elements['panel-91'])
+
+    expect(panelTitle('panel-91')).toBe('Warn / Error Logs')
+    expect(errorLogsPanel).toContain('detected_level=~\\"warn|error\\"')
+    expect(errorLogsPanel).not.toContain('| json')
+  })
+
+  it('keeps one DAU visualization and refreshes the operations dashboard', () => {
+    expect(dashboard.elements['panel-80']).toBeUndefined()
+    expect(panelTitle('panel-99')).toBe('DAU Trend')
+    expect(dashboard.timeSettings.autoRefresh).toBe('30s')
+  })
+
+  it('queries cluster-wide distinct online websocket users as an instant value', () => {
+    // ROOT CAUSE:
+    //
+    // Counting WebSocket contexts measures tabs/connections, not people. The
+    // online-user gauge counts unique Redis broadcast channels cluster-wide,
+    // and every replica reports that same shared value.
+    const wsOnlinePanel = JSON.stringify(dashboard.elements['panel-93'])
+
+    expect(panelTitle('panel-93')).toBe('Online Users')
+    expect(wsOnlinePanel).toContain('max(ws_users_online')
+    expect(wsOnlinePanel).not.toContain('ws_connections_active')
+    expect(wsOnlinePanel).toContain('"instant":true')
+    expect(wsOnlinePanel).toContain('"range":false')
+  })
 })
@@ -370,7 +370,7 @@ function pieChartPanel(id: number, title: string, description: string, queries:
            displayLabels: ['percent'],
            legend: { displayMode: 'table', overflow: 'ellipsis', placement: 'bottom', showLegend: true },
            pieType: 'pie',
-            reduceOptions: { calcs: ['sum'], fields: '', values: false },
+            reduceOptions: { calcs: ['lastNotNull'], fields: '', values: false },
            sort: 'desc',
            tooltip: { hideZeros: false, mode: 'single', sort: 'none' },
          },
@@ -381,130 +381,16 @@ function pieChartPanel(id: number, title: string, description: string, queries:
  }
 }

-// Custom Grafana UI panel retained as code so the generated dashboard matches
-// the latest hand-tuned cloud version without checking in cloud metadata.
+// Keep the rolling 24-hour series as a trend panel; the point-in-time DAU stat
+// would duplicate the same metric at the right edge of this chart.
 function dailyActiveUsersTrendPanel() {
-  return {
-    kind: 'Panel',
-    spec: {
-      data: {
-        kind: 'QueryGroup',
-        spec: {
-          queries: [
-            query(
-              `max (user_active_rolling{${SERVICE_FILTER}, window="24h"})`,
-              '__auto',
-            ),
-          ],
-          queryOptions: {},
-          transformations: [],
-        },
-      },
-      description: '',
-      id: 99,
-      links: [],
-      title: '日活',
-      vizConfig: {
-        group: 'timeseries',
-        kind: 'VizConfig',
-        spec: {
-          fieldConfig: {
-            defaults: {
-              color: { mode: 'continuous-GrYlRd', seriesBy: 'last' },
-              custom: {
-                axisBorderShow: false,
-                axisCenteredZero: false,
-                axisColorMode: 'text',
-                axisLabel: '',
-                axisPlacement: 'auto',
-                barAlignment: 0,
-                barWidthFactor: 0.6,
-                drawStyle: 'line',
-                fillOpacity: 17,
-                gradientMode: 'scheme',
-                hideFrom: { legend: false, tooltip: false, viz: false },
-                insertNulls: false,
-                lineInterpolation: 'linear',
-                lineStyle: { fill: 'solid' },
-                lineWidth: 2,
-                pointSize: 3,
-                scaleDistribution: { type: 'linear' },
-                showPoints: 'auto',
-                showValues: false,
-                spanNulls: false,
-                stacking: { group: 'A', mode: 'none' },
-                thresholdsStyle: { mode: 'off' },
-              },
-              thresholds: thresholds([{ color: 'green', value: 0 }, { color: 'red', value: 80 }]),
-            },
-            overrides: [],
-          },
-          options: {
-            annotations: { clustering: -1, multiLane: false },
-            legend: {
-              calcs: [],
-              displayMode: 'list',
-              enableFacetedFilter: false,
-              overflow: 'ellipsis',
-              placement: 'bottom',
-              showLegend: true,
-            },
-            tooltip: { hideZeros: false, mode: 'single', sort: 'none' },
-          },
-        },
-        version: SCHEMA_VERSION,
-      },
-    },
-  }
-}
-
-interface HeatmapPanelOpts {
-  unit?: string
-}
-
-// Status-code-over-time heatmap: each `sum by (label)` series becomes a Y-axis
-// row, colour encodes the rate at each time bucket. `calculate: false` means
-// the series are treated as pre-bucketed rows (one row per status code) rather
-// than re-binned by value. Reads the traffic mix at a glance — a sudden 5xx
-// row lighting up is obvious in a way a stacked line chart hides.
-function heatmapPanel(id: number, title: string, description: string, queries: PanelQuery[], opts: HeatmapPanelOpts = {}) {
-  const { unit = 'short' } = opts
-  return {
-    kind: 'Panel',
-    spec: {
-      data: { kind: 'QueryGroup', spec: { queries, queryOptions: {}, transformations: [] } },
-      description,
-      id,
-      links: [],
-      title,
-      vizConfig: {
-        group: 'heatmap',
-        kind: 'VizConfig',
-        spec: {
-          fieldConfig: {
-            defaults: {
-              custom: { hideFrom: { legend: false, tooltip: false, viz: false }, scaleDistribution: { type: 'linear' } },
-              unit,
-            },
-            overrides: [],
-          },
-          options: {
-            annotations: { clustering: -1, multiLane: false },
-            calculate: false,
-            cellGap: 1,
-            color: { exponent: 0.5, fill: 'dark-orange', mode: 'scheme', reverse: false, scale: 'exponential', scheme: 'RdYlBu', steps: 64 },
-            exemplars: { color: 'rgba(255,0,255,0.7)' },
-            filterValues: { le: 1e-9 },
-            legend: { show: false },
-            rowsFrame: { layout: 'auto' },
-            tooltip: { mode: 'single', showColorScale: false, yHistogram: false },
-            yAxis: { axisPlacement: 'left', reverse: false },
-          },
-        },
-        version: SCHEMA_VERSION,
-      },
-    },
-  }
+  return timeseriesPanel(
+    99,
+    'DAU Trend',
+    'Rolling 24-hour distinct active users over time. This is the trend view of `user.active_rolling`; the duplicate point-in-time DAU stat is intentionally omitted from this dashboard.',
+    [query(`max(user_active_rolling{${SERVICE_FILTER}, window="24h"})`, 'DAU')],
+    { unit: 'short', fillOpacity: 17, legendPlacement: 'bottom', legendCalcs: ['lastNotNull', 'max'] },
+  )
 }

 function logsPanel(id: number, title: string, description: string, expr: string) {
@@ -608,10 +494,10 @@ elements['panel-4'] = gaugePanel(
  '5xx Rate %',
  '5xx responses ÷ all responses over the last 5m. Fixed 5m window for an on-call glance ("is the service failing right now"). >1% warns, >5% pages.',
  [query(
-    `100 * sum(rate(http_server_request_duration_seconds_count{${SERVICE_FILTER}, http_request_method!="OPTIONS", http_response_status_code=~"5.."}[5m])) / clamp_min(sum(rate(http_server_request_duration_seconds_count{${SERVICE_FILTER}, http_request_method!="OPTIONS"}[5m])), 1)`,
+    `100 * sum(rate(http_server_request_duration_seconds_count{${SERVICE_FILTER}, http_request_method!="OPTIONS", http_response_status_code=~"5.."}[5m])) / sum(rate(http_server_request_duration_seconds_count{${SERVICE_FILTER}, http_request_method!="OPTIONS"}[5m]))`,
    'fail %',
  )],
-  { steps: [{ color: 'green', value: 0 }, { color: 'yellow', value: 1 }, { color: 'red', value: 5 }], max: 10, decimals: 2 },
+  { steps: [{ color: 'green', value: 0 }, { color: 'yellow', value: 1 }, { color: 'red', value: 5 }], max: 10, decimals: 2, noValue: '0' },
 )

 elements['panel-5'] = statPanel(
@@ -627,7 +513,6 @@ elements['panel-5'] = statPanel(
 // filtered by last_seen_at; one series per window). Cluster-wide gauge — every
 // replica reports the same value, so aggregate with max(), NOT sum().
 const ROLLING_USERS = [
-  { id: 80, window: '24h', title: 'DAU', label: 'Daily', span: 'last 24h' },
  { id: 81, window: '7d', title: 'WAU', label: 'Weekly', span: 'last 7d' },
  { id: 82, window: '30d', title: 'MAU', label: 'Monthly', span: 'last 30d' },
 ] as const
@@ -643,10 +528,10 @@ for (const { id, window, title, label, span } of ROLLING_USERS) {

 elements['panel-93'] = statPanel(
  93,
-  'WS Online',
-  'Current concurrent WebSocket connections across all replicas (`sum` — each replica holds its own connections). The live-presence counterpart to the rolling DAU/WAU windows.',
-  [query(`sum(ws_connections_active{${SERVICE_FILTER}})`, 'online')],
-  { unit: 'short', variant: 'count', color: 'purple', noValue: '0' },
+  'Online Users',
+  'Cluster-wide distinct authenticated users with at least one active `/ws/chat` connection. Redis returns each per-user broadcast channel once even when that user has multiple tabs or connections across server replicas; every replica reports the same global value, so the query uses `max()`.',
+  [query(`max(ws_users_online{${SERVICE_FILTER}})`, 'users', 'A', PROM, { instant: true })],
+  { unit: 'short', variant: 'count', color: 'purple', noValue: '—' },
 )

 elements['panel-92'] = timeseriesPanel(
@@ -724,25 +609,25 @@ elements['panel-16'] = barGaugePanel(
  { unit: 'short' },
 )

-elements['panel-40'] = heatmapPanel(
+elements['panel-40'] = timeseriesPanel(
  40,
-  'Status Distribution',
-  'HTTP status-code mix over time, one row per status code, colour = request rate in each time bucket. The 200 row dominates in steady state; a 5xx / 4xx row suddenly lighting up flags an incident at a glance. Non-OPTIONS traffic only.',
+  'HTTP Status Rate',
+  'Stacked non-OPTIONS request rate by HTTP status code. A new or growing 4xx / 5xx band flags a traffic-quality or service-health change.',
  [query(
    `sum by (http_response_status_code) (rate(http_server_request_duration_seconds_count{${SERVICE_FILTER}, http_request_method!="OPTIONS"}[$__rate_interval]))`,
    '{{http_response_status_code}}',
  )],
-  { unit: 'short' },
+  { unit: 'reqps', stack: true, fillOpacity: 60, legendPlacement: 'bottom' },
 )

 elements['panel-20'] = timeseriesPanel(
  20,
-  'Request Latency (by Route)',
-  '',
+  'Request Latency P95 by Route',
+  'P95 Hono request duration by matched route. Histogram buckets are merged across replicas while preserving `le`, then interpolated by `histogram_quantile`; values are estimates bounded by the configured bucket widths.',
  [query(
-    `sum by (http_route) (
-    rate(http_server_request_duration_seconds_bucket{${SERVICE_FILTER}, http_request_method!="OPTIONS", http_route!~"/api/v1/openai/.*", http_response_status_code!="404"}[$__rate_interval])
-)`,
+    `histogram_quantile(0.95, sum by (le, http_route) (
+  rate(http_server_request_duration_seconds_bucket{${SERVICE_FILTER}, http_request_method!="OPTIONS", http_route!~"/api/v1/openai/.*", http_response_status_code!="404"}[$__rate_interval])
+))`,
    '{{http_route}}',
  )],
  { unit: 's', legendPlacement: 'bottom' },
@@ -762,13 +647,16 @@ elements['panel-94'] = timeseriesPanel(
 // --- Row 3: LLM Gateway — request mix + latency ----------------------------
 elements['panel-11'] = pieChartPanel(
  11,
-  'LLM Request Rate by Model',
-  'Per-model request rate (chat + tts). Useful for capacity planning and spotting model-routing regressions.',
+  'LLM Requests by Model (range)',
+  'Per-model request count over the visible dashboard range (chat + tts). The pie shows each model\'s share without depending on Grafana sampling resolution.',
  [query(
-    `sum by (gen_ai_request_model) (rate(gen_ai_client_operation_count_total{${SERVICE_FILTER}, gen_ai_request_model!=""}[$__rate_interval]))`,
+    `sum by (gen_ai_request_model) (increase(gen_ai_client_operation_count_total{${SERVICE_FILTER}, gen_ai_request_model!=""}[$__range]))`,
    '{{gen_ai_request_model}}',
+    'A',
+    PROM,
+    { instant: true },
  )],
-  'reqps',
+  'short',
 )

 elements['panel-21'] = timeseriesPanel(
@@ -776,7 +664,7 @@ elements['panel-21'] = timeseriesPanel(
  'LLM Latency P95',
  'Two P95 latency signals for the LLM gateway, aggregated across models. TTFB = time to first streamed token (streaming chat UX). End-to-end = full operation duration — the only latency signal for non-streaming chat and TTS, which have no first-token event.',
  [
-    query(`sum by () (rate(gen_ai_client_first_token_duration_seconds_bucket{${SERVICE_FILTER}}[$__rate_interval]))`, 'TTFB p95', 'A'),
+    query(`histogram_quantile(0.95, sum by (le) (rate(gen_ai_client_first_token_duration_seconds_bucket{${SERVICE_FILTER}}[$__rate_interval])))`, 'TTFB p95', 'A'),
    query(`histogram_quantile(0.95, sum by (le) (rate(gen_ai_client_operation_duration_seconds_bucket{${SERVICE_FILTER}}[$__rate_interval])))`, 'end-to-end p95', 'B'),
  ],
  { unit: 's' },
@@ -816,7 +704,7 @@ elements['panel-68'] = timeseriesPanel(
  'Provider Failure %',
  '4xx + 5xx ÷ all requests per provider, our side of the call. Matches each provider 失败率 panel. Pair with Upstream Errors by Status Code (LLM Router Health) to see which codes drive it.',
  [query(
-    `100 * sum by (provider) (rate(gen_ai_client_operation_count_total{${SERVICE_FILTER}, provider!="", http_response_status_code=~"4..|5.."}[$__rate_interval])) / clamp_min(sum by (provider) (rate(gen_ai_client_operation_count_total{${SERVICE_FILTER}, provider!=""}[$__rate_interval])), 1)`,
+    `100 * sum by (provider) (rate(gen_ai_client_operation_count_total{${SERVICE_FILTER}, provider!="", http_response_status_code=~"4..|5.."}[$__rate_interval])) / sum by (provider) (rate(gen_ai_client_operation_count_total{${SERVICE_FILTER}, provider!=""}[$__rate_interval]))`,
    '{{provider}}',
  )],
  { unit: 'percent' },
@@ -904,7 +792,7 @@ elements['panel-62'] = gaugePanel(
  'Fallback Ratio % (5m)',
  'Fallback attempts ÷ total LLM operations over the last 5m. Sustained > 30% means one provider is degraded and the router is silently masking it for users while burning quota on the failing upstream.',
  [query(
-    `100 * sum(rate(airi_gen_ai_gateway_fallback_count_total{${SERVICE_FILTER}}[5m])) / clamp_min(sum(rate(gen_ai_client_operation_count_total{${SERVICE_FILTER}}[5m])), 1)`,
+    `100 * sum(rate(airi_gen_ai_gateway_fallback_count_total{${SERVICE_FILTER}}[5m])) / sum(rate(gen_ai_client_operation_count_total{${SERVICE_FILTER}}[5m]))`,
    'fallback %',
  )],
  { steps: [{ color: 'green', value: 0 }, { color: 'yellow', value: 10 }, { color: 'red', value: 30 }], max: 100, decimals: 1, noValue: '0' },
@@ -958,9 +846,9 @@ elements['panel-32'] = statPanel(
 // --- Row 8: Logs ------------------------------------------------------------
 elements['panel-91'] = logsPanel(
  91,
-  '5xx Error Logs',
-  'Server-side error logs (level=warn|error) from Loki. Derived fields make `trace_id` and `req` clickable — `trace_id` jumps to Tempo for full request playback.',
-  `{${SERVICE_FILTER}} | json | level=~"warn|error"`,
+  'Warn / Error Logs',
+  'Server-side warn and error logs from Loki structured metadata. Derived fields make `trace_id` and `req` clickable — `trace_id` jumps to Tempo for full request playback.',
+  `{${SERVICE_FILTER}} | detected_level=~"warn|error"`,
 )

 elements['panel-90'] = logsPanel(
@@ -998,16 +886,15 @@ const rows = [
    item('panel-68', 13, 32, 5, 6),
  ]),
  // Row 2: User Engagement — rolling-window active users and Prom-safe product
-  // analytics. The hand-tuned "日活" trend gives the row a visual engagement
-  // anchor while compact stats keep user/session totals nearby.
+  // analytics. DAU uses one trend panel; compact stats cover the longer
+  // WAU/MAU windows without duplicating the same 24-hour gauge.
  row('User Engagement', [
-    item('panel-80', 0, 0, 3, 5),
-    item('panel-1', 3, 0, 3, 5),
+    item('panel-1', 0, 0, 3, 5),
+    item('panel-15', 3, 0, 3, 5),
    item('panel-99', 6, 0, 12, 11),
    item('panel-98', 18, 0, 6, 11),
-    item('panel-82', 0, 5, 3, 3),
-    item('panel-15', 3, 5, 3, 6),
-    item('panel-81', 0, 8, 3, 3),
+    item('panel-81', 0, 5, 3, 3),
+    item('panel-82', 3, 5, 3, 3),
    item('panel-95', 0, 11, 6, 9),
    item('panel-96', 6, 11, 6, 9),
    item('panel-97', 12, 11, 12, 9),
@@ -1154,7 +1041,7 @@ export const dashboard = {
  preload: false,
  tags: ['airi', 'observability', 'grafana-cloud'],
  timeSettings: {
-    autoRefresh: '',
+    autoRefresh: '30s',
    autoRefreshIntervals: ['5s', '10s', '30s', '1m', '5m', '15m', '30m', '1h', '2h', '1d'],
    fiscalYearStartMonth: 0,
    from: 'now-6h',
@@ -54,6 +54,7 @@ import { registerDistinctActiveUsersGauge } from './otel/gauges/distinct-active-
 import { registerRollingActiveUsersGauge } from './otel/gauges/rolling-active-users'
 import { registerTotalUsersGauge } from './otel/gauges/total-users'
 import { registerTtsPoolGauge } from './otel/gauges/tts-pool'
+import { registerWsOnlineUsersGauge } from './otel/gauges/ws-online-users'
 import { createAdminRoutes } from './routes/admin'
 import { createAdminUiRoutes } from './routes/admin-ui'
 import { createAdminCapabilityAliasRoutes } from './routes/admin/capability-aliases'
@@ -838,6 +839,7 @@ export async function createApp() {
    registerDistinctActiveUsersGauge(resolved.otel.auth.distinctActiveUsers, resolved.db, resolved.otel.observability.metricReadErrors)
    registerRollingActiveUsersGauge(resolved.otel.auth.rollingActiveUsers, resolved.db, resolved.otel.observability.metricReadErrors)
    registerTtsPoolGauge(resolved.otel.gateway.poolInflight, resolved.ttsConcurrencyLedger, resolved.otel.observability.metricReadErrors)
+    registerWsOnlineUsersGauge(resolved.otel.engagement.wsUsersOnline, resolved.redis, resolved.otel.observability.metricReadErrors)
  }

  const { app, injectWebSocket } = await buildApp({
@@ -0,0 +1,87 @@
+import type { EngagementMetrics, ObservabilityMetrics } from '..'
+
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+
+import { registerWsOnlineUsersGauge } from './ws-online-users'
+
+function makeGauge() {
+  let callback: ((result: { observe: (value: number) => void }) => void | Promise<void>) | null = null
+  const observe = vi.fn()
+  const gauge = {
+    addCallback: vi.fn((registeredCallback: typeof callback) => {
+      callback = registeredCallback
+    }),
+  } as unknown as EngagementMetrics['wsUsersOnline']
+
+  return {
+    gauge,
+    observe,
+    run: async () => {
+      if (!callback)
+        throw new Error('No callback registered')
+      await callback({ observe })
+    },
+  }
+}
+
+function makeReadErrors() {
+  const add = vi.fn()
+  return {
+    metricReadErrors: { add } as unknown as ObservabilityMetrics['metricReadErrors'],
+    add,
+  }
+}
+
+describe('registerWsOnlineUsersGauge', () => {
+  beforeEach(() => {
+    vi.useFakeTimers()
+  })
+
+  afterEach(() => {
+    vi.useRealTimers()
+    vi.restoreAllMocks()
+  })
+
+  it('counts each active per-user broadcast channel once', async () => {
+    const pubsub = vi.fn(async () => [
+      'user:user-1:chat:broadcast',
+      'user:user-2:chat:broadcast',
+    ])
+    const { metricReadErrors } = makeReadErrors()
+    const { gauge, observe, run } = makeGauge()
+
+    registerWsOnlineUsersGauge(gauge, { pubsub }, metricReadErrors)
+    await run()
+
+    expect(pubsub).toHaveBeenCalledWith('CHANNELS', 'user:*:chat:broadcast')
+    expect(observe).toHaveBeenCalledWith(2)
+  })
+
+  it('serves the cached count for repeated collections within ten seconds', async () => {
+    const pubsub = vi.fn(async () => ['user:user-1:chat:broadcast'])
+    const { metricReadErrors } = makeReadErrors()
+    const { gauge, observe, run } = makeGauge()
+
+    registerWsOnlineUsersGauge(gauge, { pubsub }, metricReadErrors)
+    await run()
+    vi.advanceTimersByTime(5_000)
+    await run()
+
+    expect(pubsub).toHaveBeenCalledTimes(1)
+    expect(observe).toHaveBeenCalledTimes(2)
+  })
+
+  it('skips observation and records a read error when Redis is unavailable', async () => {
+    const pubsub = vi.fn(async () => {
+      throw new Error('Redis unavailable')
+    })
+    const { metricReadErrors, add } = makeReadErrors()
+    const { gauge, observe, run } = makeGauge()
+
+    registerWsOnlineUsersGauge(gauge, { pubsub }, metricReadErrors)
+    await run()
+
+    expect(observe).not.toHaveBeenCalled()
+    expect(add).toHaveBeenCalledWith(1, { metric: 'ws.users.online' })
+  })
+})
@@ -0,0 +1,69 @@
+import type { EngagementMetrics, ObservabilityMetrics } from '..'
+
+import { useLogger } from '@guiiai/logg'
+
+import { userChatBroadcastRedisPattern } from '../../utils/redis-keys'
+
+interface PubSubChannelReader {
+  pubsub: (subcommand: 'CHANNELS', pattern: string) => Promise<unknown[]>
+}
+
+/**
+ * Wires `ws.users.online` to the cluster-wide set of active chat broadcast
+ * channels in Redis.
+ *
+ * Each process subscribes once to `user:<id>:chat:broadcast` while it has at
+ * least one local WebSocket for that user. `PUBSUB CHANNELS` returns channel
+ * names uniquely, so multiple tabs and multiple server replicas still count as
+ * one online user. Redis removes subscriptions automatically when a process
+ * disconnects.
+ *
+ * Every replica reads the same cluster-wide value. Dashboards must aggregate
+ * this gauge with `max()` or `avg()`, never `sum()`.
+ */
+export function registerWsOnlineUsersGauge(
+  gauge: EngagementMetrics['wsUsersOnline'],
+  redis: PubSubChannelReader,
+  metricReadErrors: ObservabilityMetrics['metricReadErrors'],
+) {
+  const log = useLogger('ws-online-users-gauge').useGlobalConfig()
+  const CACHE_TTL_MS = 10_000
+
+  let cachedAt = 0
+  let cachedCount = 0
+  let refreshInFlight: Promise<boolean> | null = null
+
+  async function refresh(): Promise<boolean> {
+    try {
+      const channels = await redis.pubsub('CHANNELS', userChatBroadcastRedisPattern())
+      cachedCount = channels.length
+      cachedAt = Date.now()
+      return true
+    }
+    catch (error) {
+      log.withError(error).warn('Failed to read online websocket users for gauge')
+      metricReadErrors.add(1, { metric: 'ws.users.online' })
+      return false
+    }
+  }
+
+  gauge.addCallback(async (result) => {
+    const now = Date.now()
+
+    if (cachedAt !== 0 && now - cachedAt < CACHE_TTL_MS) {
+      result.observe(cachedCount)
+      return
+    }
+
+    if (!refreshInFlight) {
+      refreshInFlight = refresh().finally(() => {
+        refreshInFlight = null
+      })
+    }
+
+    if (await refreshInFlight)
+      result.observe(cachedCount)
+    // Redis failures intentionally skip this export cycle. Reporting zero
+    // would turn an observability outage into a false "nobody is online".
+  })
+}
@@ -65,6 +65,7 @@ import {
  METRIC_WS_CONNECTIONS_ACTIVE,
  METRIC_WS_MESSAGES_RECEIVED,
  METRIC_WS_MESSAGES_SENT,
+  METRIC_WS_USERS_ONLINE,
 } from '../utils/observability'

 const logger = useLogger('otel')
@@ -166,6 +167,16 @@ export interface EngagementMetrics {
   *   `addCallback`. Multiple callbacks would double-count.
   */
  wsConnectionsActive: ObservableGauge
+  /**
+   * Cluster-wide distinct users with at least one active chat WebSocket.
+   *
+   * The callback reads Redis Pub/Sub channels, where each authenticated user
+   * owns one `user:<id>:chat:broadcast` channel regardless of how many tabs or
+   * server replicas are connected. Every replica therefore reports the same
+   * global value; dashboards MUST aggregate it with `max()`/`avg()`, not
+   * `sum()`.
+   */
+  wsUsersOnline: ObservableGauge
  wsMessagesSent: Counter
  wsMessagesReceived: Counter
 }
@@ -421,6 +432,9 @@ export function initOtel(env: Env): OtelInstance | null {
    wsConnectionsActive: meter.createObservableGauge(METRIC_WS_CONNECTIONS_ACTIVE, {
      description: 'Active WebSocket connections (live registry size, scraped per export interval)',
    }),
+    wsUsersOnline: meter.createObservableGauge(METRIC_WS_USERS_ONLINE, {
+      description: 'Cluster-wide distinct users with an active chat WebSocket, sourced from unique Redis Pub/Sub channels',
+    }),
    wsMessagesSent: meter.createCounter(METRIC_WS_MESSAGES_SENT, {
      description: 'Messages sent via WebSocket',
    }),
@@ -67,6 +67,7 @@ export const METRIC_CHARACTER_CREATED = 'character.created'
 export const METRIC_CHARACTER_DELETED = 'character.deleted'
 export const METRIC_CHARACTER_ENGAGEMENT = 'character.engagement'
 export const METRIC_WS_CONNECTIONS_ACTIVE = 'ws.connections.active'
+export const METRIC_WS_USERS_ONLINE = 'ws.users.online'
 export const METRIC_WS_MESSAGES_SENT = 'ws.messages.sent'
 export const METRIC_WS_MESSAGES_RECEIVED = 'ws.messages.received'

@@ -29,6 +29,17 @@ export function userChatBroadcastRedisKey(userId: string): string {
  return redisKeyFrom('user', userId, 'chat', 'broadcast')
 }

+/**
+ * Matches every per-user chat broadcast channel currently known to Redis.
+ *
+ * `PUBSUB CHANNELS` returns each active channel once even when several server
+ * replicas subscribe for the same user, so the result count is the
+ * cluster-wide distinct online-user count.
+ */
+export function userChatBroadcastRedisPattern(): string {
+  return redisKeyFrom('user', '*', 'chat', 'broadcast')
+}
+
 export function lockRedisKey(domain: string, ...identifiers: RedisKeyPart[]): string {
  return redisKeyFrom('lock', domain, ...identifiers)
 }
@@ -5,6 +5,7 @@ import {
  lockRedisKey,
  redisKeyFrom,
  userChatBroadcastRedisKey,
+  userChatBroadcastRedisPattern,
  userFluxRedisKey,
 } from '../redis-keys'

@@ -23,6 +24,7 @@ describe('redis key utils', () => {
    expect(configRedisKey('FLUX_PER_REQUEST')).toBe('config:FLUX_PER_REQUEST')
    expect(userFluxRedisKey('user-1')).toBe('user:user-1:flux')
    expect(userChatBroadcastRedisKey('user-1')).toBe('user:user-1:chat:broadcast')
+    expect(userChatBroadcastRedisPattern()).toBe('user:*:chat:broadcast')
    expect(lockRedisKey('user', 'user-1', 'flux')).toBe('lock:user:user-1:flux')
  })
 })