diff --git a/apps/server/docs/ai-context/observability-metrics.md b/apps/server/docs/ai-context/observability-metrics.md index db34679fa..07035b3d9 100644 --- a/apps/server/docs/ai-context/observability-metrics.md +++ b/apps/server/docs/ai-context/observability-metrics.md @@ -60,6 +60,14 @@ OTel SDK 在导出到 Prometheus 时做两件事： | `ws.messages.sent` | Counter | 同上 | — | | `ws.messages.received` | Counter | [services/domain/chats.ts](../../src/services/domain/chats.ts) | — | +## Product Analytics + +| Metric | 类型 | 落点 | Labels | +|---|---|---|---| +| `airi.product.events` | Counter | [services/domain/product-events.ts](../../src/services/domain/product-events.ts) `track` | `feature`、`action`、`status`、`source`（可选） | + +> **Prometheus 只看事件量，不看独立用户**：`airi.product.events` 的 labels 必须保持低基数，不能加 `user_id`、`session_id`、request id、raw error message 或 prompt。要回答"每个功能有多少独立用户"，查 Postgres `product_events`：`count(distinct user_id)`。 + ## Revenue & Billing ### Stripe lifecycle @@ -134,12 +142,15 @@ OTel SDK 在导出到 Prometheus 时做两件事： | Row | viz | 关键 metric | |---|---|---| -| Service Health | stat / gauge | `user.active_sessions`（`max()`）、`ws.connections.active`（`sum()`）、`http.server.request.duration_count`（req/s + 5xx%）、`gen_ai.client.operation.count`、`airi.email.{send,failures}` 失败率 | -| Distribution (now) | donut | HTTP methods / LLM models / HTTP status codes — `increase([5m])` | -| Traffic Trends | timeseries | 同 distribution 的数据 over time | -| Latency | timeseries | `http.server.request.duration_bucket`（P95 by route）、`gen_ai.client.first_token.duration_bucket`（P95 by model） | -| Errors / Quality | mix | 4xx/5xx stacked area、`airi.gen_ai.stream.interrupted`、`airi.rate_limit.blocked` | -| Business | stat / gauge / donut | `airi.stripe.revenue`（by currency）、checkout conversion %、`stripe.events` 分布 | +| Service Health | stat / gauge / heatmap | `user.total`（`max()`）、`user.active_sessions`（`avg()`）、`ws.connections.active`（`sum()`）、`http.server.request.duration_count`（req/s + 5xx%）、`gen_ai.client.operation.count` | +| User Engagement | stat | `user.active_rolling`（DAU / WAU / MAU，`max()`） | +| Product Analytics | stat / gauge / bargauge / timeseries | `airi.product.events`（Prom-safe event volume by `feature` / `action` / `status`；distinct users 仍查 Postgres `product_events`） | +| HTTP | heatmap / bargauge / timeseries | `http.server.request.duration_count`（status mix、top routes、route errors）、`http.server.request.duration_bucket`（P95 by route） | +| LLM Gateway | timeseries | `gen_ai.client.operation.count`（by model）、`gen_ai.client.first_token.duration_bucket` / `gen_ai.client.operation.duration_bucket`（TTFB + end-to-end P95） | +| Provider Upstreams | timeseries | `gen_ai.client.operation.count` / `gen_ai.client.operation.duration_bucket` by `provider`、`airi.billing.tts.chars` | +| LLM Tokens & Quality | stat / timeseries | `gen_ai.client.token.usage.{input,output}`、`airi.billing.flux.unbilled`、`airi.gen_ai.stream.interrupted` | +| LLM Router Health | stat / gauge / timeseries | `airi.gen_ai.gateway.{key.exhausted,decrypt.failures,fallback.count,upstream.errors}` | +| Business | stat / gauge | `airi.stripe.revenue`（by currency）、checkout conversion %、`stripe.events` 分布 | | Infrastructure (collapsed, **by `service_instance_id`**) | timeseries | `db_client_operation_duration` P95（cluster）、`db_client_connection_count`、`v8js_memory_heap_used_bytes` %、`nodejs_eventloop_delay_p99_seconds` | | Logs | logs | Loki，不是 Prometheus | diff --git a/apps/server/otel/grafana/dashboards/airi-server-overview-cloud.json b/apps/server/otel/grafana/dashboards/airi-server-overview-cloud.json index 190246080..723ce42ed 100644 --- a/apps/server/otel/grafana/dashboards/airi-server-overview-cloud.json +++ b/apps/server/otel/grafana/dashboards/airi-server-overview-cloud.json @@ -904,6 +904,376 @@ } } }, + "panel-95": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "grafanacloud-projairi-prom" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum(increase(airi_product_events_total{service_name=~\"$service\", deployment_environment=~\"$env\", feature!=\"\", action!=\"\"}[$__range]))", + "legendFormat": "events" + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Total first-party product analytics events over the dashboard range. This is event volume, not distinct users — distinct-user counts come from the Postgres `product_events` table.", + "id": 95, + "links": [], + "title": "Product Events (range)", + "vizConfig": { + "group": "stat", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "fixed", + "fixedColor": "blue" + }, + "fieldMinMax": false, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": 0 + } + ] + }, + "unit": "short", + "noValue": "0" + }, + "overrides": [] + }, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": true, + "textMode": "value_and_name", + "wideLayout": true + } + }, + "version": "13.0.0-23630096546" + } + } + }, + "panel-96": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "grafanacloud-projairi-prom" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "100 * sum(increase(airi_product_events_total{service_name=~\"$service\", deployment_environment=~\"$env\", feature!=\"\", action!=\"\", status=\"failed\"}[$__range])) / clamp_min(sum(increase(airi_product_events_total{service_name=~\"$service\", deployment_environment=~\"$env\", feature!=\"\", action!=\"\"}[$__range])), 1)", + "legendFormat": "failed %" + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Failed product events ÷ all product events over the dashboard range. Uses only bounded labels (`feature`, `action`, `status`, `source`); no user/session/request identifiers are present in Prometheus.", + "id": 96, + "links": [], + "title": "Product Failure %", + "vizConfig": { + "group": "gauge", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "yellow", + "value": 2 + }, + { + "color": "red", + "value": 10 + } + ] + }, + "unit": "percent", + "decimals": 2, + "noValue": "0", + "min": 0, + "max": 20 + }, + "overrides": [] + }, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + } + }, + "version": "13.0.0-23630096546" + } + } + }, + "panel-97": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "grafanacloud-projairi-prom" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "topk(12, sum by (feature, action, status) (increase(airi_product_events_total{service_name=~\"$service\", deployment_environment=~\"$env\", feature!=\"\", action!=\"\"}[$__range])))", + "legendFormat": "{{feature}} · {{action}} · {{status}}", + "instant": true, + "range": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Top product actions by event count over the dashboard range. Use this to see which features are actually being exercised after deployment; pair with DB `count(distinct user_id)` for user counts.", + "id": 97, + "links": [], + "title": "Top Product Actions (range)", + "vizConfig": { + "group": "bargauge", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "short", + "noValue": "0" + }, + "overrides": [] + }, + "options": { + "displayMode": "gradient", + "maxVizHeight": 300, + "minVizHeight": 12, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + } + }, + "version": "13.0.0-23630096546" + } + } + }, + "panel-98": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "grafanacloud-projairi-prom" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum by (feature, action, status) (rate(airi_product_events_total{service_name=~\"$service\", deployment_environment=~\"$env\", feature!=\"\", action!=\"\"}[$__rate_interval]))", + "legendFormat": "{{feature}} · {{action}} · {{status}}" + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Product event rate by feature/action/status. This is the Prometheus-safe trend view; user-level analysis remains in Postgres `product_events`.", + "id": 98, + "links": [], + "title": "Product Event Rate", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 15, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "eps" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + } + }, + "version": "13.0.0-23630096546" + } + } + }, "panel-16": { "kind": "Panel", "spec": { @@ -3808,6 +4178,72 @@ "title": "User Engagement" } }, + { + "kind": "RowsLayoutRow", + "spec": { + "collapse": false, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-95" + }, + "height": 5, + "width": 6, + "x": 0, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-96" + }, + "height": 5, + "width": 6, + "x": 6, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-97" + }, + "height": 9, + "width": 12, + "x": 12, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-98" + }, + "height": 4, + "width": 12, + "x": 0, + "y": 5 + } + } + ] + } + }, + "title": "Product Analytics" + } + }, { "kind": "RowsLayoutRow", "spec": { diff --git a/apps/server/otel/grafana/dashboards/build.ts b/apps/server/otel/grafana/dashboards/build.ts index a0ca4036d..1de48db6f 100644 --- a/apps/server/otel/grafana/dashboards/build.ts +++ b/apps/server/otel/grafana/dashboards/build.ts @@ -39,6 +39,7 @@ const SCHEMA_VERSION = '13.0.0-23630096546' // Service / env filter applied to every Prom query. Pulled into a helper so // the variable name only appears once. const SERVICE_FILTER = 'service_name=~"$service", deployment_environment=~"$env"' +const PRODUCT_EVENT_FILTER = `${SERVICE_FILTER}, feature!="", action!=""` // Build-script local types. Kept loose — Grafana owns the schema, and we // validate the rendered JSON by re-importing it into Grafana, not by typing. @@ -526,6 +527,56 @@ elements['panel-92'] = timeseriesPanel( { unit: 'short', fillOpacity: 30 }, ) +// --- Product Analytics — event-volume view only --------------------------- +// Prometheus deliberately does not carry user_id. These panels answer +// "which product actions are happening and failing"; DB-side product_events +// queries answer "how many distinct users used each feature". +elements['panel-95'] = statPanel( + 95, + 'Product Events (range)', + 'Total first-party product analytics events over the dashboard range. This is event volume, not distinct users — distinct-user counts come from the Postgres `product_events` table.', + [ + query(`sum(increase(airi_product_events_total{${PRODUCT_EVENT_FILTER}}[$__range]))`, 'events'), + ], + { unit: 'short', variant: 'count', noValue: '0', graphMode: 'none' }, +) + +elements['panel-96'] = gaugePanel( + 96, + 'Product Failure %', + 'Failed product events ÷ all product events over the dashboard range. Uses only bounded labels (`feature`, `action`, `status`, `source`); no user/session/request identifiers are present in Prometheus.', + [query( + `100 * sum(increase(airi_product_events_total{${PRODUCT_EVENT_FILTER}, status="failed"}[$__range])) / clamp_min(sum(increase(airi_product_events_total{${PRODUCT_EVENT_FILTER}}[$__range])), 1)`, + 'failed %', + )], + { steps: [{ color: 'green', value: 0 }, { color: 'yellow', value: 2 }, { color: 'red', value: 10 }], max: 20, decimals: 2, noValue: '0' }, +) + +elements['panel-97'] = barGaugePanel( + 97, + 'Top Product Actions (range)', + 'Top product actions by event count over the dashboard range. Use this to see which features are actually being exercised after deployment; pair with DB `count(distinct user_id)` for user counts.', + [query( + `topk(12, sum by (feature, action, status) (increase(airi_product_events_total{${PRODUCT_EVENT_FILTER}}[$__range])))`, + '{{feature}} · {{action}} · {{status}}', + 'A', + PROM, + { instant: true }, + )], + { unit: 'short', noValue: '0' }, +) + +elements['panel-98'] = timeseriesPanel( + 98, + 'Product Event Rate', + 'Product event rate by feature/action/status. This is the Prometheus-safe trend view; user-level analysis remains in Postgres `product_events`.', + [query( + `sum by (feature, action, status) (rate(airi_product_events_total{${PRODUCT_EVENT_FILTER}}[$__rate_interval]))`, + '{{feature}} · {{action}} · {{status}}', + )], + { unit: 'eps', fillOpacity: 15 }, +) + // --- Row 2: HTTP — traffic ranking, error trend, latency trend ------------- elements['panel-16'] = barGaugePanel( 16, @@ -857,6 +908,15 @@ const rows = [ item('panel-81', 8, 0, 8, 4), item('panel-82', 16, 0, 8, 4), ]), + // Row 3: Product Analytics — Prom-safe event volume and failure trend. + // Distinct-user analytics stay in Postgres `product_events`; this row + // intentionally never uses user_id/session/request labels. + row('Product Analytics', [ + item('panel-95', 0, 0, 6, 5), + item('panel-96', 6, 0, 6, 5), + item('panel-97', 12, 0, 12, 9), + item('panel-98', 0, 5, 12, 4), + ]), // Row 3: HTTP — full-width error breakdown on top, then traffic ranking + // latency trend side by side. row('HTTP', [ @@ -979,14 +1039,15 @@ const variables = [ * 1. Service Health — signup/sessions/WS counts, req-rate, 5xx, status-code * heatmap, live WS trend: "is anything broken right now?" * 2. User Engagement — rolling DAU/WAU/MAU from user.last_seen_at - * 3. HTTP — error breakdown by route, request ranking, latency by route - * 4. LLM Gateway — per-model request rate + latency (TTFB + end-to-end) - * 5. Provider Upstreams — per-provider rate/latency/failure + TTS chars - * 6. LLM Tokens & Quality — token totals/throughput, revenue-leak alerts - * 7. LLM Router Health — key/decrypt/fallback "wake someone up" signals - * 8. Business — Stripe / Flux money flow - * 9. Infrastructure (collapsed) — DB / runtime health for triage - * 10. Logs — Loki for live debugging + * 3. Product Analytics — Prom-safe product event volume + failure trend + * 4. HTTP — error breakdown by route, request ranking, latency by route + * 5. LLM Gateway — per-model request rate + latency (TTFB + end-to-end) + * 6. Provider Upstreams — per-provider rate/latency/failure + TTS chars + * 7. LLM Tokens & Quality — token totals/throughput, revenue-leak alerts + * 8. LLM Router Health — key/decrypt/fallback "wake someone up" signals + * 9. Business — Stripe / Flux money flow + * 10. Infrastructure (collapsed) — DB / runtime health for triage + * 11. Logs — Loki for live debugging * * One metric, one panel: we deliberately do not duplicate a metric across * stat/trend/bar/pie forms. Counter conventions: rate() for "now" trends,