From acbbab0bd7d2b16112b342fcffb7390c9425fcc2 Mon Sep 17 00:00:00 2001
From: RainbowBird <git@luoling.moe>
Date: Tue, 28 Jul 2026 21:48:03 +0800
Subject: [PATCH] fix(server): correct observability dashboard signals (#2142)

---
 .../docs/ai-context/metrics-ownership.md      |   7 +-
 .../ai-context/observability-conventions.md   |   6 +-
 .../docs/ai-context/observability-metrics.md  |   5 +-
 .../airi-server-overview-cloud.json           | 302 +++++++-----------
 .../otel/grafana/dashboards/build.test.ts     |  77 ++++-
 apps/server/otel/grafana/dashboards/build.ts  | 203 +++---------
 apps/server/src/app.ts                        |   2 +
 .../src/otel/gauges/ws-online-users.test.ts   |  87 +++++
 .../server/src/otel/gauges/ws-online-users.ts |  69 ++++
 apps/server/src/otel/index.ts                 |  14 +
 apps/server/src/utils/observability.ts        |   1 +
 apps/server/src/utils/redis-keys.ts           |  11 +
 .../server/src/utils/tests/redis-keys.test.ts |   2 +
 13 files changed, 426 insertions(+), 360 deletions(-)
 create mode 100644 apps/server/src/otel/gauges/ws-online-users.test.ts
 create mode 100644 apps/server/src/otel/gauges/ws-online-users.ts

diff --git a/apps/server/docs/ai-context/metrics-ownership.md b/apps/server/docs/ai-context/metrics-ownership.md
index 5f455b653..bd96903bf 100644
--- a/apps/server/docs/ai-context/metrics-ownership.md
+++ b/apps/server/docs/ai-context/metrics-ownership.md
@@ -101,7 +101,7 @@
 | 域 | 代表性指标 | Truth | 备注 |
 |---|---|---|---|
 | HTTP | `http_server_request_duration_seconds_*` | Grafana | OTel 标准 |
-| WS | `ws_connections_active` / `ws_messages_*_total` | Grafana | |
+| WS | `ws_users_online` / `ws_connections_active` / `ws_messages_*_total` | Grafana | `ws_users_online` 是 Redis Pub/Sub channel 去重后的集群在线用户数；连接数仍用于排查多标签页和连接泄漏 |
 | LLM | `gen_ai_client_operation_count_total` / `gen_ai_client_first_token_duration_seconds` | Grafana | |
 | Billing | `airi_billing_flux_unbilled_total` | Grafana | **告警必须**：`increase(airi_billing_flux_unbilled_total[5m]) > 0` |
 | Auth | `user_active_sessions` / `user_distinct_active` | Postgres → Grafana 派生 | 集群级 gauge，用 `avg()` 不要 `sum()`。两个一起看：`user_active_sessions` = `COUNT(*)`（session row 数，会膨胀）, `user_distinct_active` = `COUNT(DISTINCT user_id)`（真实活跃用户数）|
@@ -219,9 +219,8 @@ PostHog UI 配 cohort：
 Dashboard 上 follow 这条 panel 链可以从"出事了"一路 drill 到"哪个 trace 是真凶"：
 
 1. **panel-4 `5xx Rate %`**（Row 1）— 数字 / gauge 颜色变红，说明出事
-2. **panel-9 `Top Routes by 5xx`**（Row 2 donut）— "现在哪些 route 在失败"
-3. **panel-44 `5xx Rate by Route`**（Row 5.5 timeseries）— "什么时候开始的、是单点还是普遍"
-4. **panel-91 `5xx Error Logs`**（Row 8 上半）— 实际错误消息，里面有 `trace_id` field 可点 → Tempo 看完整 trace 回放
+2. **panel-94 `Errors by Route`**（HTTP row）— "什么时候开始的、哪些 route / status 在失败"
+3. **panel-91 `Warn / Error Logs`**（Logs row）— 实际 warn/error 消息，里面有 `trace_id` field 可点 → Tempo 看完整 trace 回放
 
 ### Tempo / Loki derived fields 配置（一次性）
 
diff --git a/apps/server/docs/ai-context/observability-conventions.md b/apps/server/docs/ai-context/observability-conventions.md
index 95bc945cb..86e28c882 100644
--- a/apps/server/docs/ai-context/observability-conventions.md
+++ b/apps/server/docs/ai-context/observability-conventions.md
@@ -196,7 +196,7 @@ span name 目前允许保留业务可读格式，例如：
 | `Counter` | ✅ | `sum(rate(x[5m]))` | 每副本本地累加，`rate()` 自动处理重启 |
 | `Histogram` | ✅ | `histogram_quantile(0.95, sum by (le, ...) (rate(x_bucket[5m])))` | 每副本本地 bucket，`sum by (le)` 合并 |
 | `ObservableGauge`（**per-replica 状态**，如 `ws.connections.active`） | ✅ | `sum(x)` | callback 读本地 registry，所有副本求和 = 集群总量 |
-| `ObservableGauge`（**cluster-wide 状态**，如 `user.active_sessions`） | ⚠️ | `max(x)` 或 `avg(x)` | 所有副本读同一份外部状态（DB），sum 会乘以副本数 |
+| `ObservableGauge`（**cluster-wide 状态**，如 `user.active_sessions` / `ws.users.online`） | ⚠️ | `max(x)` 或 `avg(x)` | 所有副本读同一份外部状态（DB / Redis），sum 会乘以副本数 |
 | `UpDownCounter` | ⚠️ | 看场景 | 必须保证 `+1` 和 `-1` 在**同一副本**触发；否则单副本永久 +N 另一副本永久 -N |
 
 ### `UpDownCounter` 红线
@@ -233,10 +233,12 @@ span name 目前允许保留业务可读格式，例如：
 | 集群总量（per-replica gauge） | `sum(x{...})` |
 | 集群唯一值（cluster-wide gauge） | `max(x{...})` 或 `avg(x{...})` |
 | 按副本拆分调试 | `<agg> by (service_instance_id) (x{...})` |
-| 错误率 | `100 * sum(rate(x_total{...,status_code=~"5.."}[5m])) / clamp_min(sum(rate(x_total{...}[5m])), 1)` |
+| 错误率 | `100 * sum(rate(x_total{...,status_code=~"5.."}[5m])) / sum(rate(x_total{...}[5m]))` |
 
 红线：**任何 cumulative counter 都不能直接 `sum()` 不 wrap rate/increase**。Counter 在副本重启时归零，没有 rate() 包裹 Prometheus 会跳变；用 `increase($__range)` 看「时间窗口内总量」，用 `rate([interval])` 看「当前速率」。
 
+比例的分母也不能用 `clamp_min(rate(...), 1)` 防零：这会在流量低于 1 req/s 时系统性低估错误率。零流量应由 Grafana 的 no-value 展示策略处理，不应修改真实分母。
+
 ### 「按副本拆分」何时加
 
 默认 panel 都聚合到集群层面。但以下场景应该加 `by (service_instance_id)` 拆分图：
diff --git a/apps/server/docs/ai-context/observability-metrics.md b/apps/server/docs/ai-context/observability-metrics.md
index 07035b3d9..036eed1a4 100644
--- a/apps/server/docs/ai-context/observability-metrics.md
+++ b/apps/server/docs/ai-context/observability-metrics.md
@@ -57,9 +57,12 @@ OTel SDK 在导出到 Prometheus 时做两件事：
 | `character.deleted` | Counter | 同上 | — |
 | `character.engagement` | Counter | 同上（like/bookmark） | `action`（`like` / `unlike` / `bookmark` / `unbookmark`） |
 | `ws.connections.active` | ObservableGauge | [routes/chat-ws/index.ts](../../src/routes/chat-ws/index.ts) `addCallback` walks `userConnections` Map | — |
+| `ws.users.online` | ObservableGauge | [otel/gauges/ws-online-users.ts](../../src/otel/gauges/ws-online-users.ts) counts unique active `user:*:chat:broadcast` Redis Pub/Sub channels | — |
 | `ws.messages.sent` | Counter | 同上 | — |
 | `ws.messages.received` | Counter | [services/domain/chats.ts](../../src/services/domain/chats.ts) | — |
 
+> `ws.users.online` 是 cluster-wide gauge：同一用户无论打开多少标签页、连接到多少个 Server 副本，Redis 都只返回一个活跃 channel。每个副本读取并上报同一个全局值，因此 Dashboard 必须使用 `max()` / `avg()`，不能使用 `sum()`。
+
 ## Product Analytics
 
 | Metric | 类型 | 落点 | Labels |
@@ -142,7 +145,7 @@ OTel SDK 在导出到 Prometheus 时做两件事：
 
 | Row | viz | 关键 metric |
 |---|---|---|
-| Service Health | stat / gauge / heatmap | `user.total`（`max()`）、`user.active_sessions`（`avg()`）、`ws.connections.active`（`sum()`）、`http.server.request.duration_count`（req/s + 5xx%）、`gen_ai.client.operation.count` |
+| Service Health | stat / gauge / timeseries | `user.total`（`max()`）、`user.active_sessions`（`avg()`）、`ws.users.online`（`max()`）、`ws.connections.active`（`sum()` trend）、`http.server.request.duration_count`（req/s + 5xx%）、`gen_ai.client.operation.count` |
 | User Engagement | stat | `user.active_rolling`（DAU / WAU / MAU，`max()`） |
 | Product Analytics | stat / gauge / bargauge / timeseries | `airi.product.events`（Prom-safe event volume by `feature` / `action` / `status`；distinct users 仍查 Postgres `product_events`） |
 | HTTP | heatmap / bargauge / timeseries | `http.server.request.duration_count`（status mix、top routes、route errors）、`http.server.request.duration_bucket`（P95 by route） |
diff --git a/apps/server/otel/grafana/dashboards/airi-server-overview-cloud.json b/apps/server/otel/grafana/dashboards/airi-server-overview-cloud.json
index 2ba18149b..3f2ac1861 100644
--- a/apps/server/otel/grafana/dashboards/airi-server-overview-cloud.json
+++ b/apps/server/otel/grafana/dashboards/airi-server-overview-cloud.json
@@ -319,7 +319,7 @@
                     "kind": "DataQuery",
                     "spec": {
                       "editorMode": "code",
-                      "expr": "100 * sum(rate(http_server_request_duration_seconds_count{service_name=~\"$service\", deployment_environment=~\"$env\", http_request_method!=\"OPTIONS\", http_response_status_code=~\"5..\"}[5m])) / clamp_min(sum(rate(http_server_request_duration_seconds_count{service_name=~\"$service\", deployment_environment=~\"$env\", http_request_method!=\"OPTIONS\"}[5m])), 1)",
+                      "expr": "100 * sum(rate(http_server_request_duration_seconds_count{service_name=~\"$service\", deployment_environment=~\"$env\", http_request_method!=\"OPTIONS\", http_response_status_code=~\"5..\"}[5m])) / sum(rate(http_server_request_duration_seconds_count{service_name=~\"$service\", deployment_environment=~\"$env\", http_request_method!=\"OPTIONS\"}[5m]))",
                       "legendFormat": "fail %",
                       "range": true
                     },
@@ -365,6 +365,7 @@
                 },
                 "unit": "percent",
                 "decimals": 2,
+                "noValue": "0",
                 "min": 0,
                 "max": 10
               },
@@ -472,90 +473,6 @@
         }
       }
     },
-    "panel-80": {
-      "kind": "Panel",
-      "spec": {
-        "data": {
-          "kind": "QueryGroup",
-          "spec": {
-            "queries": [
-              {
-                "kind": "PanelQuery",
-                "spec": {
-                  "hidden": false,
-                  "query": {
-                    "datasource": {
-                      "name": "grafanacloud-projairi-prom"
-                    },
-                    "group": "prometheus",
-                    "kind": "DataQuery",
-                    "spec": {
-                      "editorMode": "code",
-                      "expr": "max(user_active_rolling{service_name=~\"$service\", deployment_environment=~\"$env\", window=\"24h\"})",
-                      "legendFormat": "DAU",
-                      "range": true
-                    },
-                    "version": "v0"
-                  },
-                  "refId": "A"
-                }
-              }
-            ],
-            "queryOptions": {},
-            "transformations": []
-          }
-        },
-        "description": "Daily active users — distinct users with activity in the last 24h. Sourced from `user.last_seen_at` (touched on sign-in and every OIDC token refresh) via the `user.active_rolling` gauge. Cluster-wide gauge aggregated with `max()`.",
-        "id": 80,
-        "links": [],
-        "title": "DAU",
-        "vizConfig": {
-          "group": "stat",
-          "kind": "VizConfig",
-          "spec": {
-            "fieldConfig": {
-              "defaults": {
-                "color": {
-                  "mode": "fixed",
-                  "fixedColor": "blue"
-                },
-                "fieldMinMax": false,
-                "thresholds": {
-                  "mode": "absolute",
-                  "steps": [
-                    {
-                      "color": "blue",
-                      "value": 0
-                    }
-                  ]
-                },
-                "unit": "short",
-                "noValue": "0"
-              },
-              "overrides": []
-            },
-            "options": {
-              "colorMode": "none",
-              "graphMode": "area",
-              "justifyMode": "auto",
-              "orientation": "auto",
-              "percentChangeColorMode": "standard",
-              "reduceOptions": {
-                "calcs": [
-                  "lastNotNull"
-                ],
-                "fields": "",
-                "values": false
-              },
-              "showPercentChange": true,
-              "textMode": "value_and_name",
-              "wideLayout": true
-            }
-          },
-          "version": "13.2.0-28666480772"
-        }
-      }
-    },
     "panel-81": {
       "kind": "Panel",
       "spec": {
@@ -743,9 +660,10 @@
                     "kind": "DataQuery",
                     "spec": {
                       "editorMode": "code",
-                      "expr": "sum(ws_connections_active{service_name=~\"$service\", deployment_environment=~\"$env\"})",
-                      "legendFormat": "online",
-                      "range": true
+                      "expr": "max(ws_users_online{service_name=~\"$service\", deployment_environment=~\"$env\"})",
+                      "legendFormat": "users",
+                      "instant": true,
+                      "range": false
                     },
                     "version": "v0"
                   },
@@ -757,10 +675,10 @@
             "transformations": []
           }
         },
-        "description": "Current concurrent WebSocket connections across all replicas (`sum` — each replica holds its own connections). The live-presence counterpart to the rolling DAU/WAU windows.",
+        "description": "Cluster-wide distinct authenticated users with at least one active `/ws/chat` connection. Redis returns each per-user broadcast channel once even when that user has multiple tabs or connections across server replicas; every replica reports the same global value, so the query uses `max()`.",
         "id": 93,
         "links": [],
-        "title": "WS Online",
+        "title": "Online Users",
         "vizConfig": {
           "group": "stat",
           "kind": "VizConfig",
@@ -782,7 +700,7 @@
                   ]
                 },
                 "unit": "short",
-                "noValue": "0"
+                "noValue": "—"
               },
               "overrides": []
             },
@@ -1326,8 +1244,8 @@
                     "kind": "DataQuery",
                     "spec": {
                       "editorMode": "code",
-                      "expr": "max (user_active_rolling{service_name=~\"$service\", deployment_environment=~\"$env\", window=\"24h\"})",
-                      "legendFormat": "__auto",
+                      "expr": "max(user_active_rolling{service_name=~\"$service\", deployment_environment=~\"$env\", window=\"24h\"})",
+                      "legendFormat": "DAU",
                       "range": true
                     },
                     "version": "v0"
@@ -1340,10 +1258,10 @@
             "transformations": []
           }
         },
-        "description": "",
+        "description": "Rolling 24-hour distinct active users over time. This is the trend view of `user.active_rolling`; the duplicate point-in-time DAU stat is intentionally omitted from this dashboard.",
         "id": 99,
         "links": [],
-        "title": "日活",
+        "title": "DAU Trend",
         "vizConfig": {
           "group": "timeseries",
           "kind": "VizConfig",
@@ -1351,8 +1269,7 @@
             "fieldConfig": {
               "defaults": {
                 "color": {
-                  "mode": "continuous-GrYlRd",
-                  "seriesBy": "last"
+                  "mode": "palette-classic"
                 },
                 "custom": {
                   "axisBorderShow": false,
@@ -1364,19 +1281,16 @@
                   "barWidthFactor": 0.6,
                   "drawStyle": "line",
                   "fillOpacity": 17,
-                  "gradientMode": "scheme",
+                  "gradientMode": "none",
                   "hideFrom": {
                     "legend": false,
                     "tooltip": false,
                     "viz": false
                   },
                   "insertNulls": false,
-                  "lineInterpolation": "linear",
-                  "lineStyle": {
-                    "fill": "solid"
-                  },
-                  "lineWidth": 2,
-                  "pointSize": 3,
+                  "lineInterpolation": "smooth",
+                  "lineWidth": 1,
+                  "pointSize": 5,
                   "scaleDistribution": {
                     "type": "linear"
                   },
@@ -1397,13 +1311,10 @@
                     {
                       "color": "green",
                       "value": 0
-                    },
-                    {
-                      "color": "red",
-                      "value": 80
                     }
                   ]
-                }
+                },
+                "unit": "short"
               },
               "overrides": []
             },
@@ -1413,8 +1324,11 @@
                 "multiLane": false
               },
               "legend": {
-                "calcs": [],
-                "displayMode": "list",
+                "calcs": [
+                  "lastNotNull",
+                  "max"
+                ],
+                "displayMode": "table",
                 "enableFacetedFilter": false,
                 "overflow": "ellipsis",
                 "placement": "bottom",
@@ -1422,8 +1336,8 @@
               },
               "tooltip": {
                 "hideZeros": false,
-                "mode": "single",
-                "sort": "none"
+                "mode": "multi",
+                "sort": "desc"
               }
             }
           },
@@ -1547,27 +1461,63 @@
             "transformations": []
           }
         },
-        "description": "HTTP status-code mix over time, one row per status code, colour = request rate in each time bucket. The 200 row dominates in steady state; a 5xx / 4xx row suddenly lighting up flags an incident at a glance. Non-OPTIONS traffic only.",
+        "description": "Stacked non-OPTIONS request rate by HTTP status code. A new or growing 4xx / 5xx band flags a traffic-quality or service-health change.",
         "id": 40,
         "links": [],
-        "title": "Status Distribution",
+        "title": "HTTP Status Rate",
         "vizConfig": {
-          "group": "heatmap",
+          "group": "timeseries",
           "kind": "VizConfig",
           "spec": {
             "fieldConfig": {
               "defaults": {
+                "color": {
+                  "mode": "palette-classic"
+                },
                 "custom": {
+                  "axisBorderShow": false,
+                  "axisCenteredZero": false,
+                  "axisColorMode": "text",
+                  "axisLabel": "",
+                  "axisPlacement": "auto",
+                  "barAlignment": 0,
+                  "barWidthFactor": 0.6,
+                  "drawStyle": "line",
+                  "fillOpacity": 60,
+                  "gradientMode": "none",
                   "hideFrom": {
                     "legend": false,
                     "tooltip": false,
                     "viz": false
                   },
+                  "insertNulls": false,
+                  "lineInterpolation": "smooth",
+                  "lineWidth": 1,
+                  "pointSize": 5,
                   "scaleDistribution": {
                     "type": "linear"
+                  },
+                  "showPoints": "auto",
+                  "showValues": false,
+                  "spanNulls": false,
+                  "stacking": {
+                    "group": "A",
+                    "mode": "normal"
+                  },
+                  "thresholdsStyle": {
+                    "mode": "off"
                   }
                 },
-                "unit": "short"
+                "thresholds": {
+                  "mode": "absolute",
+                  "steps": [
+                    {
+                      "color": "green",
+                      "value": 0
+                    }
+                  ]
+                },
+                "unit": "reqps"
               },
               "overrides": []
             },
@@ -1576,37 +1526,21 @@
                 "clustering": -1,
                 "multiLane": false
               },
-              "calculate": false,
-              "cellGap": 1,
-              "color": {
-                "exponent": 0.5,
-                "fill": "dark-orange",
-                "mode": "scheme",
-                "reverse": false,
-                "scale": "exponential",
-                "scheme": "RdYlBu",
-                "steps": 64
-              },
-              "exemplars": {
-                "color": "rgba(255,0,255,0.7)"
-              },
-              "filterValues": {
-                "le": 1e-9
-              },
               "legend": {
-                "show": false
-              },
-              "rowsFrame": {
-                "layout": "auto"
+                "calcs": [
+                  "lastNotNull",
+                  "max"
+                ],
+                "displayMode": "table",
+                "enableFacetedFilter": false,
+                "overflow": "ellipsis",
+                "placement": "bottom",
+                "showLegend": true
               },
               "tooltip": {
-                "mode": "single",
-                "showColorScale": false,
-                "yHistogram": false
-              },
-              "yAxis": {
-                "axisPlacement": "left",
-                "reverse": false
+                "hideZeros": false,
+                "mode": "multi",
+                "sort": "desc"
               }
             }
           },
@@ -1633,7 +1567,7 @@
                     "kind": "DataQuery",
                     "spec": {
                       "editorMode": "code",
-                      "expr": "sum by (http_route) (\n    rate(http_server_request_duration_seconds_bucket{service_name=~\"$service\", deployment_environment=~\"$env\", http_request_method!=\"OPTIONS\", http_route!~\"/api/v1/openai/.*\", http_response_status_code!=\"404\"}[$__rate_interval])\n)",
+                      "expr": "histogram_quantile(0.95, sum by (le, http_route) (\n  rate(http_server_request_duration_seconds_bucket{service_name=~\"$service\", deployment_environment=~\"$env\", http_request_method!=\"OPTIONS\", http_route!~\"/api/v1/openai/.*\", http_response_status_code!=\"404\"}[$__rate_interval])\n))",
                       "legendFormat": "{{http_route}}",
                       "range": true
                     },
@@ -1647,10 +1581,10 @@
             "transformations": []
           }
         },
-        "description": "",
+        "description": "P95 Hono request duration by matched route. Histogram buckets are merged across replicas while preserving `le`, then interpolated by `histogram_quantile`; values are estimates bounded by the configured bucket widths.",
         "id": 20,
         "links": [],
-        "title": "Request Latency (by Route)",
+        "title": "Request Latency P95 by Route",
         "vizConfig": {
           "group": "timeseries",
           "kind": "VizConfig",
@@ -1873,9 +1807,10 @@
                     "kind": "DataQuery",
                     "spec": {
                       "editorMode": "code",
-                      "expr": "sum by (gen_ai_request_model) (rate(gen_ai_client_operation_count_total{service_name=~\"$service\", deployment_environment=~\"$env\", gen_ai_request_model!=\"\"}[$__rate_interval]))",
+                      "expr": "sum by (gen_ai_request_model) (increase(gen_ai_client_operation_count_total{service_name=~\"$service\", deployment_environment=~\"$env\", gen_ai_request_model!=\"\"}[$__range]))",
                       "legendFormat": "{{gen_ai_request_model}}",
-                      "range": true
+                      "instant": true,
+                      "range": false
                     },
                     "version": "v0"
                   },
@@ -1887,10 +1822,10 @@
             "transformations": []
           }
         },
-        "description": "Per-model request rate (chat + tts). Useful for capacity planning and spotting model-routing regressions.",
+        "description": "Per-model request count over the visible dashboard range (chat + tts). The pie shows each model's share without depending on Grafana sampling resolution.",
         "id": 11,
         "links": [],
-        "title": "LLM Request Rate by Model",
+        "title": "LLM Requests by Model (range)",
         "vizConfig": {
           "group": "piechart",
           "kind": "VizConfig",
@@ -1908,7 +1843,7 @@
                     "viz": false
                   }
                 },
-                "unit": "reqps"
+                "unit": "short"
               },
               "overrides": []
             },
@@ -1925,7 +1860,7 @@
               "pieType": "pie",
               "reduceOptions": {
                 "calcs": [
-                  "sum"
+                  "lastNotNull"
                 ],
                 "fields": "",
                 "values": false
@@ -1961,7 +1896,7 @@
                     "kind": "DataQuery",
                     "spec": {
                       "editorMode": "code",
-                      "expr": "sum by () (rate(gen_ai_client_first_token_duration_seconds_bucket{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))",
+                      "expr": "histogram_quantile(0.95, sum by (le) (rate(gen_ai_client_first_token_duration_seconds_bucket{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval])))",
                       "legendFormat": "TTFB p95",
                       "range": true
                     },
@@ -2342,7 +2277,7 @@
                     "kind": "DataQuery",
                     "spec": {
                       "editorMode": "code",
-                      "expr": "100 * sum by (provider) (rate(gen_ai_client_operation_count_total{service_name=~\"$service\", deployment_environment=~\"$env\", provider!=\"\", http_response_status_code=~\"4..|5..\"}[$__rate_interval])) / clamp_min(sum by (provider) (rate(gen_ai_client_operation_count_total{service_name=~\"$service\", deployment_environment=~\"$env\", provider!=\"\"}[$__rate_interval])), 1)",
+                      "expr": "100 * sum by (provider) (rate(gen_ai_client_operation_count_total{service_name=~\"$service\", deployment_environment=~\"$env\", provider!=\"\", http_response_status_code=~\"4..|5..\"}[$__rate_interval])) / sum by (provider) (rate(gen_ai_client_operation_count_total{service_name=~\"$service\", deployment_environment=~\"$env\", provider!=\"\"}[$__rate_interval]))",
                       "legendFormat": "{{provider}}",
                       "range": true
                     },
@@ -3176,7 +3111,7 @@
                     "kind": "DataQuery",
                     "spec": {
                       "editorMode": "code",
-                      "expr": "100 * sum(rate(airi_gen_ai_gateway_fallback_count_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[5m])) / clamp_min(sum(rate(gen_ai_client_operation_count_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[5m])), 1)",
+                      "expr": "100 * sum(rate(airi_gen_ai_gateway_fallback_count_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[5m])) / sum(rate(gen_ai_client_operation_count_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[5m]))",
                       "legendFormat": "fallback %",
                       "range": true
                     },
@@ -3647,7 +3582,7 @@
                     "kind": "DataQuery",
                     "spec": {
                       "editorMode": "code",
-                      "expr": "{service_name=~\"$service\", deployment_environment=~\"$env\"} | json | level=~\"warn|error\"",
+                      "expr": "{service_name=~\"$service\", deployment_environment=~\"$env\"} | detected_level=~\"warn|error\"",
                       "legendFormat": "",
                       "range": true
                     },
@@ -3661,10 +3596,10 @@
             "transformations": []
           }
         },
-        "description": "Server-side error logs (level=warn|error) from Loki. Derived fields make `trace_id` and `req` clickable — `trace_id` jumps to Tempo for full request playback.",
+        "description": "Server-side warn and error logs from Loki structured metadata. Derived fields make `trace_id` and `req` clickable — `trace_id` jumps to Tempo for full request playback.",
         "id": 91,
         "links": [],
-        "title": "5xx Error Logs",
+        "title": "Warn / Error Logs",
         "vizConfig": {
           "group": "logs",
           "kind": "VizConfig",
@@ -4015,7 +3950,7 @@
                     "spec": {
                       "element": {
                         "kind": "ElementReference",
-                        "name": "panel-80"
+                        "name": "panel-1"
                       },
                       "height": 5,
                       "width": 3,
@@ -4028,7 +3963,7 @@
                     "spec": {
                       "element": {
                         "kind": "ElementReference",
-                        "name": "panel-1"
+                        "name": "panel-15"
                       },
                       "height": 5,
                       "width": 3,
@@ -4062,32 +3997,6 @@
                       "y": 0
                     }
                   },
-                  {
-                    "kind": "GridLayoutItem",
-                    "spec": {
-                      "element": {
-                        "kind": "ElementReference",
-                        "name": "panel-82"
-                      },
-                      "height": 3,
-                      "width": 3,
-                      "x": 0,
-                      "y": 5
-                    }
-                  },
-                  {
-                    "kind": "GridLayoutItem",
-                    "spec": {
-                      "element": {
-                        "kind": "ElementReference",
-                        "name": "panel-15"
-                      },
-                      "height": 6,
-                      "width": 3,
-                      "x": 3,
-                      "y": 5
-                    }
-                  },
                   {
                     "kind": "GridLayoutItem",
                     "spec": {
@@ -4098,7 +4007,20 @@
                       "height": 3,
                       "width": 3,
                       "x": 0,
-                      "y": 8
+                      "y": 5
+                    }
+                  },
+                  {
+                    "kind": "GridLayoutItem",
+                    "spec": {
+                      "element": {
+                        "kind": "ElementReference",
+                        "name": "panel-82"
+                      },
+                      "height": 3,
+                      "width": 3,
+                      "x": 3,
+                      "y": 5
                     }
                   },
                   {
@@ -4384,7 +4306,7 @@
     "grafana-cloud"
   ],
   "timeSettings": {
-    "autoRefresh": "",
+    "autoRefresh": "30s",
     "autoRefreshIntervals": [
       "5s",
       "10s",
diff --git a/apps/server/otel/grafana/dashboards/build.test.ts b/apps/server/otel/grafana/dashboards/build.test.ts
index f9dcd328e..a20e798ad 100644
--- a/apps/server/otel/grafana/dashboards/build.test.ts
+++ b/apps/server/otel/grafana/dashboards/build.test.ts
@@ -54,16 +54,12 @@ describe('grafana dashboard builder', () => {
     expect(result.unusedElems).toEqual([])
   })
 
-  /**
-   * @example
-   * expect(panelTitle('panel-99')).toBe('TTS Success %')
-   */
   it('keeps the product analytics row focused on Prometheus-safe engagement signals', () => {
     expect(panelTitle('panel-95')).toBe('Product Events (range)')
     expect(panelTitle('panel-96')).toBe('Product Failure %')
     expect(panelTitle('panel-97')).toBe('Top Product Actions (range)')
     expect(panelTitle('panel-98')).toBe('Product Event Rate')
-    expect(panelTitle('panel-99')).toBe('日活')
+    expect(panelTitle('panel-99')).toBe('DAU Trend')
   })
 
   /**
@@ -86,4 +82,75 @@ describe('grafana dashboard builder', () => {
     expect(productPanelExpressions).not.toContain('session_id')
     expect(productPanelExpressions).not.toContain('request_id')
   })
+
+  it('preserves histogram buckets until HTTP and LLM latency quantiles are calculated', () => {
+    // ROOT CAUSE:
+    //
+    // The latency panels previously summed cumulative `_bucket` rates after
+    // dropping `le`. Grafana then labelled that request-rate-derived value as
+    // seconds, making millisecond HTTP routes appear to take 20+ seconds.
+    //
+    // Quantiles must retain `le` while replicas are merged, then call
+    // histogram_quantile over the merged histogram.
+    const httpLatency = collectQueryExpressions(dashboard.elements['panel-20']).join('\n')
+    const llmLatency = collectQueryExpressions(dashboard.elements['panel-21']).join('\n')
+
+    expect(httpLatency).toContain('histogram_quantile(0.95')
+    expect(httpLatency).toContain('sum by (le, http_route)')
+    expect(llmLatency.match(/histogram_quantile\(0\.95/g)).toHaveLength(2)
+    expect(llmLatency).toContain('gen_ai_client_first_token_duration_seconds_bucket')
+    expect(llmLatency).toContain('sum by (le)')
+  })
+
+  it('does not clamp rate denominators to one request per second', () => {
+    // ROOT CAUSE:
+    //
+    // `clamp_min(rate, 1)` changes the denominator whenever traffic is below
+    // 1 req/s, so low-volume provider and fallback failures are underreported.
+    for (const panelName of ['panel-4', 'panel-62', 'panel-68']) {
+      const expressions = collectQueryExpressions(dashboard.elements[panelName]).join('\n')
+
+      expect(expressions).not.toContain('clamp_min')
+    }
+  })
+
+  it('uses visualizations and reductions that match each query shape', () => {
+    const statusPanel = JSON.stringify(dashboard.elements['panel-40'])
+    const modelMixPanel = JSON.stringify(dashboard.elements['panel-11'])
+
+    expect(statusPanel).toContain('"group":"timeseries"')
+    expect(statusPanel).not.toContain('"group":"heatmap"')
+    expect(modelMixPanel).toContain('increase(')
+    expect(modelMixPanel).toContain('"instant":true')
+    expect(modelMixPanel).toContain('"calcs":["lastNotNull"]')
+  })
+
+  it('filters structured Loki severity without parsing plain-text bodies as JSON', () => {
+    const errorLogsPanel = JSON.stringify(dashboard.elements['panel-91'])
+
+    expect(panelTitle('panel-91')).toBe('Warn / Error Logs')
+    expect(errorLogsPanel).toContain('detected_level=~\\"warn|error\\"')
+    expect(errorLogsPanel).not.toContain('| json')
+  })
+
+  it('keeps one DAU visualization and refreshes the operations dashboard', () => {
+    expect(dashboard.elements['panel-80']).toBeUndefined()
+    expect(panelTitle('panel-99')).toBe('DAU Trend')
+    expect(dashboard.timeSettings.autoRefresh).toBe('30s')
+  })
+
+  it('queries cluster-wide distinct online websocket users as an instant value', () => {
+    // ROOT CAUSE:
+    //
+    // Counting WebSocket contexts measures tabs/connections, not people. The
+    // online-user gauge counts unique Redis broadcast channels cluster-wide,
+    // and every replica reports that same shared value.
+    const wsOnlinePanel = JSON.stringify(dashboard.elements['panel-93'])
+
+    expect(panelTitle('panel-93')).toBe('Online Users')
+    expect(wsOnlinePanel).toContain('max(ws_users_online')
+    expect(wsOnlinePanel).not.toContain('ws_connections_active')
+    expect(wsOnlinePanel).toContain('"instant":true')
+    expect(wsOnlinePanel).toContain('"range":false')
+  })
 })
diff --git a/apps/server/otel/grafana/dashboards/build.ts b/apps/server/otel/grafana/dashboards/build.ts
index bb438ce12..ec89a68f6 100644
--- a/apps/server/otel/grafana/dashboards/build.ts
+++ b/apps/server/otel/grafana/dashboards/build.ts
@@ -370,7 +370,7 @@ function pieChartPanel(id: number, title: string, description: string, queries:
             displayLabels: ['percent'],
             legend: { displayMode: 'table', overflow: 'ellipsis', placement: 'bottom', showLegend: true },
             pieType: 'pie',
-            reduceOptions: { calcs: ['sum'], fields: '', values: false },
+            reduceOptions: { calcs: ['lastNotNull'], fields: '', values: false },
             sort: 'desc',
             tooltip: { hideZeros: false, mode: 'single', sort: 'none' },
           },
@@ -381,130 +381,16 @@ function pieChartPanel(id: number, title: string, description: string, queries:
   }
 }
 
-// Custom Grafana UI panel retained as code so the generated dashboard matches
-// the latest hand-tuned cloud version without checking in cloud metadata.
+// Keep the rolling 24-hour series as a trend panel; the point-in-time DAU stat
+// would duplicate the same metric at the right edge of this chart.
 function dailyActiveUsersTrendPanel() {
-  return {
-    kind: 'Panel',
-    spec: {
-      data: {
-        kind: 'QueryGroup',
-        spec: {
-          queries: [
-            query(
-              `max (user_active_rolling{${SERVICE_FILTER}, window="24h"})`,
-              '__auto',
-            ),
-          ],
-          queryOptions: {},
-          transformations: [],
-        },
-      },
-      description: '',
-      id: 99,
-      links: [],
-      title: '日活',
-      vizConfig: {
-        group: 'timeseries',
-        kind: 'VizConfig',
-        spec: {
-          fieldConfig: {
-            defaults: {
-              color: { mode: 'continuous-GrYlRd', seriesBy: 'last' },
-              custom: {
-                axisBorderShow: false,
-                axisCenteredZero: false,
-                axisColorMode: 'text',
-                axisLabel: '',
-                axisPlacement: 'auto',
-                barAlignment: 0,
-                barWidthFactor: 0.6,
-                drawStyle: 'line',
-                fillOpacity: 17,
-                gradientMode: 'scheme',
-                hideFrom: { legend: false, tooltip: false, viz: false },
-                insertNulls: false,
-                lineInterpolation: 'linear',
-                lineStyle: { fill: 'solid' },
-                lineWidth: 2,
-                pointSize: 3,
-                scaleDistribution: { type: 'linear' },
-                showPoints: 'auto',
-                showValues: false,
-                spanNulls: false,
-                stacking: { group: 'A', mode: 'none' },
-                thresholdsStyle: { mode: 'off' },
-              },
-              thresholds: thresholds([{ color: 'green', value: 0 }, { color: 'red', value: 80 }]),
-            },
-            overrides: [],
-          },
-          options: {
-            annotations: { clustering: -1, multiLane: false },
-            legend: {
-              calcs: [],
-              displayMode: 'list',
-              enableFacetedFilter: false,
-              overflow: 'ellipsis',
-              placement: 'bottom',
-              showLegend: true,
-            },
-            tooltip: { hideZeros: false, mode: 'single', sort: 'none' },
-          },
-        },
-        version: SCHEMA_VERSION,
-      },
-    },
-  }
-}
-
-interface HeatmapPanelOpts {
-  unit?: string
-}
-
-// Status-code-over-time heatmap: each `sum by (label)` series becomes a Y-axis
-// row, colour encodes the rate at each time bucket. `calculate: false` means
-// the series are treated as pre-bucketed rows (one row per status code) rather
-// than re-binned by value. Reads the traffic mix at a glance — a sudden 5xx
-// row lighting up is obvious in a way a stacked line chart hides.
-function heatmapPanel(id: number, title: string, description: string, queries: PanelQuery[], opts: HeatmapPanelOpts = {}) {
-  const { unit = 'short' } = opts
-  return {
-    kind: 'Panel',
-    spec: {
-      data: { kind: 'QueryGroup', spec: { queries, queryOptions: {}, transformations: [] } },
-      description,
-      id,
-      links: [],
-      title,
-      vizConfig: {
-        group: 'heatmap',
-        kind: 'VizConfig',
-        spec: {
-          fieldConfig: {
-            defaults: {
-              custom: { hideFrom: { legend: false, tooltip: false, viz: false }, scaleDistribution: { type: 'linear' } },
-              unit,
-            },
-            overrides: [],
-          },
-          options: {
-            annotations: { clustering: -1, multiLane: false },
-            calculate: false,
-            cellGap: 1,
-            color: { exponent: 0.5, fill: 'dark-orange', mode: 'scheme', reverse: false, scale: 'exponential', scheme: 'RdYlBu', steps: 64 },
-            exemplars: { color: 'rgba(255,0,255,0.7)' },
-            filterValues: { le: 1e-9 },
-            legend: { show: false },
-            rowsFrame: { layout: 'auto' },
-            tooltip: { mode: 'single', showColorScale: false, yHistogram: false },
-            yAxis: { axisPlacement: 'left', reverse: false },
-          },
-        },
-        version: SCHEMA_VERSION,
-      },
-    },
-  }
+  return timeseriesPanel(
+    99,
+    'DAU Trend',
+    'Rolling 24-hour distinct active users over time. This is the trend view of `user.active_rolling`; the duplicate point-in-time DAU stat is intentionally omitted from this dashboard.',
+    [query(`max(user_active_rolling{${SERVICE_FILTER}, window="24h"})`, 'DAU')],
+    { unit: 'short', fillOpacity: 17, legendPlacement: 'bottom', legendCalcs: ['lastNotNull', 'max'] },
+  )
 }
 
 function logsPanel(id: number, title: string, description: string, expr: string) {
@@ -608,10 +494,10 @@ elements['panel-4'] = gaugePanel(
   '5xx Rate %',
   '5xx responses ÷ all responses over the last 5m. Fixed 5m window for an on-call glance ("is the service failing right now"). >1% warns, >5% pages.',
   [query(
-    `100 * sum(rate(http_server_request_duration_seconds_count{${SERVICE_FILTER}, http_request_method!="OPTIONS", http_response_status_code=~"5.."}[5m])) / clamp_min(sum(rate(http_server_request_duration_seconds_count{${SERVICE_FILTER}, http_request_method!="OPTIONS"}[5m])), 1)`,
+    `100 * sum(rate(http_server_request_duration_seconds_count{${SERVICE_FILTER}, http_request_method!="OPTIONS", http_response_status_code=~"5.."}[5m])) / sum(rate(http_server_request_duration_seconds_count{${SERVICE_FILTER}, http_request_method!="OPTIONS"}[5m]))`,
     'fail %',
   )],
-  { steps: [{ color: 'green', value: 0 }, { color: 'yellow', value: 1 }, { color: 'red', value: 5 }], max: 10, decimals: 2 },
+  { steps: [{ color: 'green', value: 0 }, { color: 'yellow', value: 1 }, { color: 'red', value: 5 }], max: 10, decimals: 2, noValue: '0' },
 )
 
 elements['panel-5'] = statPanel(
@@ -627,7 +513,6 @@ elements['panel-5'] = statPanel(
 // filtered by last_seen_at; one series per window). Cluster-wide gauge — every
 // replica reports the same value, so aggregate with max(), NOT sum().
 const ROLLING_USERS = [
-  { id: 80, window: '24h', title: 'DAU', label: 'Daily', span: 'last 24h' },
   { id: 81, window: '7d', title: 'WAU', label: 'Weekly', span: 'last 7d' },
   { id: 82, window: '30d', title: 'MAU', label: 'Monthly', span: 'last 30d' },
 ] as const
@@ -643,10 +528,10 @@ for (const { id, window, title, label, span } of ROLLING_USERS) {
 
 elements['panel-93'] = statPanel(
   93,
-  'WS Online',
-  'Current concurrent WebSocket connections across all replicas (`sum` — each replica holds its own connections). The live-presence counterpart to the rolling DAU/WAU windows.',
-  [query(`sum(ws_connections_active{${SERVICE_FILTER}})`, 'online')],
-  { unit: 'short', variant: 'count', color: 'purple', noValue: '0' },
+  'Online Users',
+  'Cluster-wide distinct authenticated users with at least one active `/ws/chat` connection. Redis returns each per-user broadcast channel once even when that user has multiple tabs or connections across server replicas; every replica reports the same global value, so the query uses `max()`.',
+  [query(`max(ws_users_online{${SERVICE_FILTER}})`, 'users', 'A', PROM, { instant: true })],
+  { unit: 'short', variant: 'count', color: 'purple', noValue: '—' },
 )
 
 elements['panel-92'] = timeseriesPanel(
@@ -724,25 +609,25 @@ elements['panel-16'] = barGaugePanel(
   { unit: 'short' },
 )
 
-elements['panel-40'] = heatmapPanel(
+elements['panel-40'] = timeseriesPanel(
   40,
-  'Status Distribution',
-  'HTTP status-code mix over time, one row per status code, colour = request rate in each time bucket. The 200 row dominates in steady state; a 5xx / 4xx row suddenly lighting up flags an incident at a glance. Non-OPTIONS traffic only.',
+  'HTTP Status Rate',
+  'Stacked non-OPTIONS request rate by HTTP status code. A new or growing 4xx / 5xx band flags a traffic-quality or service-health change.',
   [query(
     `sum by (http_response_status_code) (rate(http_server_request_duration_seconds_count{${SERVICE_FILTER}, http_request_method!="OPTIONS"}[$__rate_interval]))`,
     '{{http_response_status_code}}',
   )],
-  { unit: 'short' },
+  { unit: 'reqps', stack: true, fillOpacity: 60, legendPlacement: 'bottom' },
 )
 
 elements['panel-20'] = timeseriesPanel(
   20,
-  'Request Latency (by Route)',
-  '',
+  'Request Latency P95 by Route',
+  'P95 Hono request duration by matched route. Histogram buckets are merged across replicas while preserving `le`, then interpolated by `histogram_quantile`; values are estimates bounded by the configured bucket widths.',
   [query(
-    `sum by (http_route) (
-    rate(http_server_request_duration_seconds_bucket{${SERVICE_FILTER}, http_request_method!="OPTIONS", http_route!~"/api/v1/openai/.*", http_response_status_code!="404"}[$__rate_interval])
-)`,
+    `histogram_quantile(0.95, sum by (le, http_route) (
+  rate(http_server_request_duration_seconds_bucket{${SERVICE_FILTER}, http_request_method!="OPTIONS", http_route!~"/api/v1/openai/.*", http_response_status_code!="404"}[$__rate_interval])
+))`,
     '{{http_route}}',
   )],
   { unit: 's', legendPlacement: 'bottom' },
@@ -762,13 +647,16 @@ elements['panel-94'] = timeseriesPanel(
 // --- Row 3: LLM Gateway — request mix + latency ----------------------------
 elements['panel-11'] = pieChartPanel(
   11,
-  'LLM Request Rate by Model',
-  'Per-model request rate (chat + tts). Useful for capacity planning and spotting model-routing regressions.',
+  'LLM Requests by Model (range)',
+  'Per-model request count over the visible dashboard range (chat + tts). The pie shows each model\'s share without depending on Grafana sampling resolution.',
   [query(
-    `sum by (gen_ai_request_model) (rate(gen_ai_client_operation_count_total{${SERVICE_FILTER}, gen_ai_request_model!=""}[$__rate_interval]))`,
+    `sum by (gen_ai_request_model) (increase(gen_ai_client_operation_count_total{${SERVICE_FILTER}, gen_ai_request_model!=""}[$__range]))`,
     '{{gen_ai_request_model}}',
+    'A',
+    PROM,
+    { instant: true },
   )],
-  'reqps',
+  'short',
 )
 
 elements['panel-21'] = timeseriesPanel(
@@ -776,7 +664,7 @@ elements['panel-21'] = timeseriesPanel(
   'LLM Latency P95',
   'Two P95 latency signals for the LLM gateway, aggregated across models. TTFB = time to first streamed token (streaming chat UX). End-to-end = full operation duration — the only latency signal for non-streaming chat and TTS, which have no first-token event.',
   [
-    query(`sum by () (rate(gen_ai_client_first_token_duration_seconds_bucket{${SERVICE_FILTER}}[$__rate_interval]))`, 'TTFB p95', 'A'),
+    query(`histogram_quantile(0.95, sum by (le) (rate(gen_ai_client_first_token_duration_seconds_bucket{${SERVICE_FILTER}}[$__rate_interval])))`, 'TTFB p95', 'A'),
     query(`histogram_quantile(0.95, sum by (le) (rate(gen_ai_client_operation_duration_seconds_bucket{${SERVICE_FILTER}}[$__rate_interval])))`, 'end-to-end p95', 'B'),
   ],
   { unit: 's' },
@@ -816,7 +704,7 @@ elements['panel-68'] = timeseriesPanel(
   'Provider Failure %',
   '4xx + 5xx ÷ all requests per provider, our side of the call. Matches each provider 失败率 panel. Pair with Upstream Errors by Status Code (LLM Router Health) to see which codes drive it.',
   [query(
-    `100 * sum by (provider) (rate(gen_ai_client_operation_count_total{${SERVICE_FILTER}, provider!="", http_response_status_code=~"4..|5.."}[$__rate_interval])) / clamp_min(sum by (provider) (rate(gen_ai_client_operation_count_total{${SERVICE_FILTER}, provider!=""}[$__rate_interval])), 1)`,
+    `100 * sum by (provider) (rate(gen_ai_client_operation_count_total{${SERVICE_FILTER}, provider!="", http_response_status_code=~"4..|5.."}[$__rate_interval])) / sum by (provider) (rate(gen_ai_client_operation_count_total{${SERVICE_FILTER}, provider!=""}[$__rate_interval]))`,
     '{{provider}}',
   )],
   { unit: 'percent' },
@@ -904,7 +792,7 @@ elements['panel-62'] = gaugePanel(
   'Fallback Ratio % (5m)',
   'Fallback attempts ÷ total LLM operations over the last 5m. Sustained > 30% means one provider is degraded and the router is silently masking it for users while burning quota on the failing upstream.',
   [query(
-    `100 * sum(rate(airi_gen_ai_gateway_fallback_count_total{${SERVICE_FILTER}}[5m])) / clamp_min(sum(rate(gen_ai_client_operation_count_total{${SERVICE_FILTER}}[5m])), 1)`,
+    `100 * sum(rate(airi_gen_ai_gateway_fallback_count_total{${SERVICE_FILTER}}[5m])) / sum(rate(gen_ai_client_operation_count_total{${SERVICE_FILTER}}[5m]))`,
     'fallback %',
   )],
   { steps: [{ color: 'green', value: 0 }, { color: 'yellow', value: 10 }, { color: 'red', value: 30 }], max: 100, decimals: 1, noValue: '0' },
@@ -958,9 +846,9 @@ elements['panel-32'] = statPanel(
 // --- Row 8: Logs ------------------------------------------------------------
 elements['panel-91'] = logsPanel(
   91,
-  '5xx Error Logs',
-  'Server-side error logs (level=warn|error) from Loki. Derived fields make `trace_id` and `req` clickable — `trace_id` jumps to Tempo for full request playback.',
-  `{${SERVICE_FILTER}} | json | level=~"warn|error"`,
+  'Warn / Error Logs',
+  'Server-side warn and error logs from Loki structured metadata. Derived fields make `trace_id` and `req` clickable — `trace_id` jumps to Tempo for full request playback.',
+  `{${SERVICE_FILTER}} | detected_level=~"warn|error"`,
 )
 
 elements['panel-90'] = logsPanel(
@@ -998,16 +886,15 @@ const rows = [
     item('panel-68', 13, 32, 5, 6),
   ]),
   // Row 2: User Engagement — rolling-window active users and Prom-safe product
-  // analytics. The hand-tuned "日活" trend gives the row a visual engagement
-  // anchor while compact stats keep user/session totals nearby.
+  // analytics. DAU uses one trend panel; compact stats cover the longer
+  // WAU/MAU windows without duplicating the same 24-hour gauge.
   row('User Engagement', [
-    item('panel-80', 0, 0, 3, 5),
-    item('panel-1', 3, 0, 3, 5),
+    item('panel-1', 0, 0, 3, 5),
+    item('panel-15', 3, 0, 3, 5),
     item('panel-99', 6, 0, 12, 11),
     item('panel-98', 18, 0, 6, 11),
-    item('panel-82', 0, 5, 3, 3),
-    item('panel-15', 3, 5, 3, 6),
-    item('panel-81', 0, 8, 3, 3),
+    item('panel-81', 0, 5, 3, 3),
+    item('panel-82', 3, 5, 3, 3),
     item('panel-95', 0, 11, 6, 9),
     item('panel-96', 6, 11, 6, 9),
     item('panel-97', 12, 11, 12, 9),
@@ -1154,7 +1041,7 @@ export const dashboard = {
   preload: false,
   tags: ['airi', 'observability', 'grafana-cloud'],
   timeSettings: {
-    autoRefresh: '',
+    autoRefresh: '30s',
     autoRefreshIntervals: ['5s', '10s', '30s', '1m', '5m', '15m', '30m', '1h', '2h', '1d'],
     fiscalYearStartMonth: 0,
     from: 'now-6h',
diff --git a/apps/server/src/app.ts b/apps/server/src/app.ts
index c47f6819d..fbdb946e4 100644
--- a/apps/server/src/app.ts
+++ b/apps/server/src/app.ts
@@ -54,6 +54,7 @@ import { registerDistinctActiveUsersGauge } from './otel/gauges/distinct-active-
 import { registerRollingActiveUsersGauge } from './otel/gauges/rolling-active-users'
 import { registerTotalUsersGauge } from './otel/gauges/total-users'
 import { registerTtsPoolGauge } from './otel/gauges/tts-pool'
+import { registerWsOnlineUsersGauge } from './otel/gauges/ws-online-users'
 import { createAdminRoutes } from './routes/admin'
 import { createAdminUiRoutes } from './routes/admin-ui'
 import { createAdminCapabilityAliasRoutes } from './routes/admin/capability-aliases'
@@ -838,6 +839,7 @@ export async function createApp() {
     registerDistinctActiveUsersGauge(resolved.otel.auth.distinctActiveUsers, resolved.db, resolved.otel.observability.metricReadErrors)
     registerRollingActiveUsersGauge(resolved.otel.auth.rollingActiveUsers, resolved.db, resolved.otel.observability.metricReadErrors)
     registerTtsPoolGauge(resolved.otel.gateway.poolInflight, resolved.ttsConcurrencyLedger, resolved.otel.observability.metricReadErrors)
+    registerWsOnlineUsersGauge(resolved.otel.engagement.wsUsersOnline, resolved.redis, resolved.otel.observability.metricReadErrors)
   }
 
   const { app, injectWebSocket } = await buildApp({
diff --git a/apps/server/src/otel/gauges/ws-online-users.test.ts b/apps/server/src/otel/gauges/ws-online-users.test.ts
new file mode 100644
index 000000000..8122fc227
--- /dev/null
+++ b/apps/server/src/otel/gauges/ws-online-users.test.ts
@@ -0,0 +1,87 @@
+import type { EngagementMetrics, ObservabilityMetrics } from '..'
+
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+
+import { registerWsOnlineUsersGauge } from './ws-online-users'
+
+function makeGauge() {
+  let callback: ((result: { observe: (value: number) => void }) => void | Promise<void>) | null = null
+  const observe = vi.fn()
+  const gauge = {
+    addCallback: vi.fn((registeredCallback: typeof callback) => {
+      callback = registeredCallback
+    }),
+  } as unknown as EngagementMetrics['wsUsersOnline']
+
+  return {
+    gauge,
+    observe,
+    run: async () => {
+      if (!callback)
+        throw new Error('No callback registered')
+      await callback({ observe })
+    },
+  }
+}
+
+function makeReadErrors() {
+  const add = vi.fn()
+  return {
+    metricReadErrors: { add } as unknown as ObservabilityMetrics['metricReadErrors'],
+    add,
+  }
+}
+
+describe('registerWsOnlineUsersGauge', () => {
+  beforeEach(() => {
+    vi.useFakeTimers()
+  })
+
+  afterEach(() => {
+    vi.useRealTimers()
+    vi.restoreAllMocks()
+  })
+
+  it('counts each active per-user broadcast channel once', async () => {
+    const pubsub = vi.fn(async () => [
+      'user:user-1:chat:broadcast',
+      'user:user-2:chat:broadcast',
+    ])
+    const { metricReadErrors } = makeReadErrors()
+    const { gauge, observe, run } = makeGauge()
+
+    registerWsOnlineUsersGauge(gauge, { pubsub }, metricReadErrors)
+    await run()
+
+    expect(pubsub).toHaveBeenCalledWith('CHANNELS', 'user:*:chat:broadcast')
+    expect(observe).toHaveBeenCalledWith(2)
+  })
+
+  it('serves the cached count for repeated collections within ten seconds', async () => {
+    const pubsub = vi.fn(async () => ['user:user-1:chat:broadcast'])
+    const { metricReadErrors } = makeReadErrors()
+    const { gauge, observe, run } = makeGauge()
+
+    registerWsOnlineUsersGauge(gauge, { pubsub }, metricReadErrors)
+    await run()
+    vi.advanceTimersByTime(5_000)
+    await run()
+
+    expect(pubsub).toHaveBeenCalledTimes(1)
+    expect(observe).toHaveBeenCalledTimes(2)
+  })
+
+  it('skips observation and records a read error when Redis is unavailable', async () => {
+    const pubsub = vi.fn(async () => {
+      throw new Error('Redis unavailable')
+    })
+    const { metricReadErrors, add } = makeReadErrors()
+    const { gauge, observe, run } = makeGauge()
+
+    registerWsOnlineUsersGauge(gauge, { pubsub }, metricReadErrors)
+    await run()
+
+    expect(observe).not.toHaveBeenCalled()
+    expect(add).toHaveBeenCalledWith(1, { metric: 'ws.users.online' })
+  })
+})
diff --git a/apps/server/src/otel/gauges/ws-online-users.ts b/apps/server/src/otel/gauges/ws-online-users.ts
new file mode 100644
index 000000000..0df7c14be
--- /dev/null
+++ b/apps/server/src/otel/gauges/ws-online-users.ts
@@ -0,0 +1,69 @@
+import type { EngagementMetrics, ObservabilityMetrics } from '..'
+
+import { useLogger } from '@guiiai/logg'
+
+import { userChatBroadcastRedisPattern } from '../../utils/redis-keys'
+
+interface PubSubChannelReader {
+  pubsub: (subcommand: 'CHANNELS', pattern: string) => Promise<unknown[]>
+}
+
+/**
+ * Wires `ws.users.online` to the cluster-wide set of active chat broadcast
+ * channels in Redis.
+ *
+ * Each process subscribes once to `user:<id>:chat:broadcast` while it has at
+ * least one local WebSocket for that user. `PUBSUB CHANNELS` returns channel
+ * names uniquely, so multiple tabs and multiple server replicas still count as
+ * one online user. Redis removes subscriptions automatically when a process
+ * disconnects.
+ *
+ * Every replica reads the same cluster-wide value. Dashboards must aggregate
+ * this gauge with `max()` or `avg()`, never `sum()`.
+ */
+export function registerWsOnlineUsersGauge(
+  gauge: EngagementMetrics['wsUsersOnline'],
+  redis: PubSubChannelReader,
+  metricReadErrors: ObservabilityMetrics['metricReadErrors'],
+) {
+  const log = useLogger('ws-online-users-gauge').useGlobalConfig()
+  const CACHE_TTL_MS = 10_000
+
+  let cachedAt = 0
+  let cachedCount = 0
+  let refreshInFlight: Promise<boolean> | null = null
+
+  async function refresh(): Promise<boolean> {
+    try {
+      const channels = await redis.pubsub('CHANNELS', userChatBroadcastRedisPattern())
+      cachedCount = channels.length
+      cachedAt = Date.now()
+      return true
+    }
+    catch (error) {
+      log.withError(error).warn('Failed to read online websocket users for gauge')
+      metricReadErrors.add(1, { metric: 'ws.users.online' })
+      return false
+    }
+  }
+
+  gauge.addCallback(async (result) => {
+    const now = Date.now()
+
+    if (cachedAt !== 0 && now - cachedAt < CACHE_TTL_MS) {
+      result.observe(cachedCount)
+      return
+    }
+
+    if (!refreshInFlight) {
+      refreshInFlight = refresh().finally(() => {
+        refreshInFlight = null
+      })
+    }
+
+    if (await refreshInFlight)
+      result.observe(cachedCount)
+    // Redis failures intentionally skip this export cycle. Reporting zero
+    // would turn an observability outage into a false "nobody is online".
+  })
+}
diff --git a/apps/server/src/otel/index.ts b/apps/server/src/otel/index.ts
index b5d1d60d2..67aca602a 100644
--- a/apps/server/src/otel/index.ts
+++ b/apps/server/src/otel/index.ts
@@ -65,6 +65,7 @@ import {
   METRIC_WS_CONNECTIONS_ACTIVE,
   METRIC_WS_MESSAGES_RECEIVED,
   METRIC_WS_MESSAGES_SENT,
+  METRIC_WS_USERS_ONLINE,
 } from '../utils/observability'
 
 const logger = useLogger('otel')
@@ -166,6 +167,16 @@ export interface EngagementMetrics {
    *   `addCallback`. Multiple callbacks would double-count.
    */
   wsConnectionsActive: ObservableGauge
+  /**
+   * Cluster-wide distinct users with at least one active chat WebSocket.
+   *
+   * The callback reads Redis Pub/Sub channels, where each authenticated user
+   * owns one `user:<id>:chat:broadcast` channel regardless of how many tabs or
+   * server replicas are connected. Every replica therefore reports the same
+   * global value; dashboards MUST aggregate it with `max()`/`avg()`, not
+   * `sum()`.
+   */
+  wsUsersOnline: ObservableGauge
   wsMessagesSent: Counter
   wsMessagesReceived: Counter
 }
@@ -421,6 +432,9 @@ export function initOtel(env: Env): OtelInstance | null {
     wsConnectionsActive: meter.createObservableGauge(METRIC_WS_CONNECTIONS_ACTIVE, {
       description: 'Active WebSocket connections (live registry size, scraped per export interval)',
     }),
+    wsUsersOnline: meter.createObservableGauge(METRIC_WS_USERS_ONLINE, {
+      description: 'Cluster-wide distinct users with an active chat WebSocket, sourced from unique Redis Pub/Sub channels',
+    }),
     wsMessagesSent: meter.createCounter(METRIC_WS_MESSAGES_SENT, {
       description: 'Messages sent via WebSocket',
     }),
diff --git a/apps/server/src/utils/observability.ts b/apps/server/src/utils/observability.ts
index a2ce26ba6..ce9df6cc1 100644
--- a/apps/server/src/utils/observability.ts
+++ b/apps/server/src/utils/observability.ts
@@ -67,6 +67,7 @@ export const METRIC_CHARACTER_CREATED = 'character.created'
 export const METRIC_CHARACTER_DELETED = 'character.deleted'
 export const METRIC_CHARACTER_ENGAGEMENT = 'character.engagement'
 export const METRIC_WS_CONNECTIONS_ACTIVE = 'ws.connections.active'
+export const METRIC_WS_USERS_ONLINE = 'ws.users.online'
 export const METRIC_WS_MESSAGES_SENT = 'ws.messages.sent'
 export const METRIC_WS_MESSAGES_RECEIVED = 'ws.messages.received'
 
diff --git a/apps/server/src/utils/redis-keys.ts b/apps/server/src/utils/redis-keys.ts
index 2a6db2679..435db6d86 100644
--- a/apps/server/src/utils/redis-keys.ts
+++ b/apps/server/src/utils/redis-keys.ts
@@ -29,6 +29,17 @@ export function userChatBroadcastRedisKey(userId: string): string {
   return redisKeyFrom('user', userId, 'chat', 'broadcast')
 }
 
+/**
+ * Matches every per-user chat broadcast channel currently known to Redis.
+ *
+ * `PUBSUB CHANNELS` returns each active channel once even when several server
+ * replicas subscribe for the same user, so the result count is the
+ * cluster-wide distinct online-user count.
+ */
+export function userChatBroadcastRedisPattern(): string {
+  return redisKeyFrom('user', '*', 'chat', 'broadcast')
+}
+
 export function lockRedisKey(domain: string, ...identifiers: RedisKeyPart[]): string {
   return redisKeyFrom('lock', domain, ...identifiers)
 }
diff --git a/apps/server/src/utils/tests/redis-keys.test.ts b/apps/server/src/utils/tests/redis-keys.test.ts
index ae82225db..e27147eff 100644
--- a/apps/server/src/utils/tests/redis-keys.test.ts
+++ b/apps/server/src/utils/tests/redis-keys.test.ts
@@ -5,6 +5,7 @@ import {
   lockRedisKey,
   redisKeyFrom,
   userChatBroadcastRedisKey,
+  userChatBroadcastRedisPattern,
   userFluxRedisKey,
 } from '../redis-keys'
 
@@ -23,6 +24,7 @@ describe('redis key utils', () => {
     expect(configRedisKey('FLUX_PER_REQUEST')).toBe('config:FLUX_PER_REQUEST')
     expect(userFluxRedisKey('user-1')).toBe('user:user-1:flux')
     expect(userChatBroadcastRedisKey('user-1')).toBe('user:user-1:chat:broadcast')
+    expect(userChatBroadcastRedisPattern()).toBe('user:*:chat:broadcast')
     expect(lockRedisKey('user', 'user-1', 'flux')).toBe('lock:user:user-1:flux')
   })
 })