diff --git a/apps/server/docker-compose.otel.yml b/apps/server/docker-compose.otel.yml deleted file mode 100644 index c8e539084..000000000 --- a/apps/server/docker-compose.otel.yml +++ /dev/null @@ -1,122 +0,0 @@ -name: proj-airi-otel - -services: - # ============================================================ - # OpenTelemetry Collector - # Receives traces, metrics, and logs from the application - # and exports them to the appropriate backends. - # ============================================================ - otel-collector: - image: otel/opentelemetry-collector-contrib:0.120.0 - command: ['--config=/etc/otelcol/otel-collector.yaml'] - volumes: - - ./otel/collector/otel-collector.yaml:/etc/otelcol/otel-collector.yaml:ro - ports: - - '4317:4317' # OTLP gRPC - - '4318:4318' # OTLP HTTP - depends_on: - loki: - condition: service_started - tempo: - condition: service_started - healthcheck: - test: ['CMD', 'wget', '--spider', '-q', 'http://localhost:13133/'] - interval: 10s - timeout: 5s - retries: 5 - restart: unless-stopped - - # ============================================================ - # Prometheus - Metrics storage and querying - # ============================================================ - prometheus: - image: prom/prometheus:v3.2.1 - command: - - --config.file=/etc/prometheus/prometheus.yaml - - --storage.tsdb.path=/prometheus - - --storage.tsdb.retention.time=7d - - --web.enable-remote-write-receiver - - --enable-feature=exemplar-storage - - --enable-feature=native-histograms - volumes: - - ./otel/prometheus/prometheus.yaml:/etc/prometheus/prometheus.yaml:ro - - prometheus_data:/prometheus - healthcheck: - test: ['CMD', 'wget', '--spider', '-q', 'http://localhost:9090/-/healthy'] - interval: 10s - timeout: 5s - retries: 5 - restart: unless-stopped - - # ============================================================ - # Loki - Log aggregation - # ============================================================ - loki: - image: grafana/loki:3.4.3 - command: -config.file=/etc/loki/loki.yaml - volumes: - - ./otel/loki/loki.yaml:/etc/loki/loki.yaml:ro - - loki_data:/loki - healthcheck: - test: ['CMD', 'wget', '--spider', '-q', 'http://localhost:3100/ready'] - interval: 10s - timeout: 5s - retries: 5 - restart: unless-stopped - - # ============================================================ - # Tempo - Distributed tracing backend - # ============================================================ - tempo: - image: grafana/tempo:2.7.2 - command: ['-config.file=/etc/tempo/tempo.yaml'] - volumes: - - ./otel/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro - - tempo_data:/var/tempo - healthcheck: - test: ['CMD', 'wget', '--spider', '-q', 'http://localhost:3200/ready'] - interval: 10s - timeout: 5s - retries: 5 - restart: unless-stopped - - # ============================================================ - # Grafana - Visualization and dashboards - # ============================================================ - grafana: - image: grafana/grafana:11.5.2 - environment: - - GF_SECURITY_ADMIN_USER=admin - - GF_SECURITY_ADMIN_PASSWORD=admin - - GF_AUTH_ANONYMOUS_ENABLED=true - - GF_AUTH_ANONYMOUS_ORG_ROLE=Viewer - - GF_FEATURE_TOGGLES_ENABLE=traceqlEditor tempoSearch tempoServiceGraph - volumes: - - ./otel/grafana/provisioning:/etc/grafana/provisioning:ro - - ./otel/grafana/dashboards:/var/lib/grafana/dashboards:ro - - grafana_data:/var/lib/grafana - ports: - - '3001:3000' - depends_on: - prometheus: - condition: service_healthy - loki: - condition: service_healthy - tempo: - condition: service_healthy - healthcheck: - test: ['CMD', 'wget', '--spider', '-q', 'http://localhost:3000/livez'] - interval: 10s - timeout: 5s - retries: 5 - restart: unless-stopped - -volumes: - prometheus_data: - driver: local - loki_data: - driver: local - tempo_data: - driver: local - grafana_data: - driver: local diff --git a/apps/server/otel/collector/otel-collector.yaml b/apps/server/otel/collector/otel-collector.yaml deleted file mode 100644 index 4889147a1..000000000 --- a/apps/server/otel/collector/otel-collector.yaml +++ /dev/null @@ -1,88 +0,0 @@ -receivers: - otlp: - protocols: - grpc: - endpoint: 0.0.0.0:4317 - http: - endpoint: 0.0.0.0:4318 - -processors: - batch: - timeout: 5s - send_batch_size: 1024 - memory_limiter: - check_interval: 1s - limit_mib: 512 - spike_limit_mib: 128 - resource: - attributes: - - key: service.namespace - value: proj-airi - action: upsert - # Tail-based sampling: keeps all errors and slow requests, - # samples a percentage of normal traffic to reduce storage costs. - tail_sampling: - decision_wait: 10s - num_traces: 100000 - policies: - # Always keep traces that contain errors - - name: errors-policy - type: status_code - status_code: - status_codes: [ERROR] - # Always keep slow requests (> 500ms) - - name: slow-requests-policy - type: latency - latency: - threshold_ms: 500 - # Sample 10% of remaining normal traffic - - name: probabilistic-policy - type: probabilistic - probabilistic: - sampling_percentage: 10 - -exporters: - # Prometheus exporter for metrics - prometheus: - endpoint: 0.0.0.0:8889 - namespace: airi - resource_to_telemetry_conversion: - enabled: true - - # Loki exporter for logs - loki: - endpoint: http://loki:3100/loki/api/v1/push - - # Tempo exporter for traces - otlp/tempo: - endpoint: tempo:4317 - tls: - insecure: true - - debug: - verbosity: basic - -extensions: - health_check: - endpoint: 0.0.0.0:13133 - zpages: - endpoint: 0.0.0.0:55679 - -service: - extensions: [health_check, zpages] - pipelines: - traces: - receivers: [otlp] - processors: [memory_limiter, resource, tail_sampling, batch] - exporters: [otlp/tempo, debug] - metrics: - receivers: [otlp] - processors: [memory_limiter, resource, batch] - exporters: [prometheus, debug] - logs: - receivers: [otlp] - processors: [memory_limiter, resource, batch] - exporters: [loki, debug] - telemetry: - logs: - level: info diff --git a/apps/server/otel/grafana/dashboards/airi-server-overview-cloud.json b/apps/server/otel/grafana/dashboards/airi-server-overview-cloud.json deleted file mode 100644 index 3f2ac1861..000000000 --- a/apps/server/otel/grafana/dashboards/airi-server-overview-cloud.json +++ /dev/null @@ -1,4399 +0,0 @@ -{ - "annotations": [ - { - "kind": "AnnotationQuery", - "spec": { - "builtIn": true, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "query": { - "datasource": { - "name": "-- Grafana --" - }, - "group": "grafana", - "kind": "DataQuery", - "spec": {}, - "version": "v0" - } - } - } - ], - "cursorSync": "Crosshair", - "editable": true, - "elements": { - "panel-1": { - "kind": "Panel", - "spec": { - "data": { - "kind": "QueryGroup", - "spec": { - "queries": [ - { - "kind": "PanelQuery", - "spec": { - "hidden": false, - "query": { - "datasource": { - "name": "grafanacloud-projairi-prom" - }, - "group": "prometheus", - "kind": "DataQuery", - "spec": { - "editorMode": "code", - "expr": "max(user_total{service_name=~\"$service\", deployment_environment=~\"$env\"})", - "legendFormat": "total users", - "range": true - }, - "version": "v0" - }, - "refId": "A" - } - }, - { - "kind": "PanelQuery", - "spec": { - "hidden": false, - "query": { - "datasource": { - "name": "grafanacloud-projairi-prom" - }, - "group": "prometheus", - "kind": "DataQuery", - "spec": { - "editorMode": "code", - "expr": "sum(increase(user_registered_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[24h]))", - "legendFormat": "new today", - "range": true - }, - "version": "v0" - }, - "refId": "B" - } - } - ], - "queryOptions": {}, - "transformations": [] - } - }, - "description": "Current Better Auth user table size from `user.total` (cluster-wide DB gauge, aggregate with `max()`) plus rolling 24h signup delta from `increase(user.registered)`. Use the delta as today/new-user growth, and DAU / WAU / MAU below for returning-user engagement.", - "id": 1, - "links": [], - "title": "Total Users", - "vizConfig": { - "group": "stat", - "kind": "VizConfig", - "spec": { - "fieldConfig": { - "defaults": { - "color": { - "mode": "fixed", - "fixedColor": "blue" - }, - "fieldMinMax": false, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": 0 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "options": { - "colorMode": "none", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": true, - "textMode": "value_and_name", - "wideLayout": true - } - }, - "version": "13.2.0-28666480772" - } - } - }, - "panel-15": { - "kind": "Panel", - "spec": { - "data": { - "kind": "QueryGroup", - "spec": { - "queries": [ - { - "kind": "PanelQuery", - "spec": { - "hidden": false, - "query": { - "datasource": { - "name": "grafanacloud-projairi-prom" - }, - "group": "prometheus", - "kind": "DataQuery", - "spec": { - "editorMode": "code", - "expr": "avg(user_active_sessions{service_name=~\"$service\", deployment_environment=~\"$env\"})", - "legendFormat": "sessions", - "range": true - }, - "version": "v0" - }, - "refId": "A" - } - } - ], - "queryOptions": {}, - "transformations": [] - } - }, - "description": "COUNT(*) over the Better Auth `session` table where `expires_at > now()`, aggregated with `avg()` (cluster-wide gauge). Counts session **rows**, not users — compare against DAU to spot session-row inflation.", - "id": 15, - "links": [], - "title": "Active Sessions", - "vizConfig": { - "group": "stat", - "kind": "VizConfig", - "spec": { - "fieldConfig": { - "defaults": { - "color": { - "mode": "fixed", - "fixedColor": "blue" - }, - "fieldMinMax": false, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": 0 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "options": { - "colorMode": "none", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": true, - "textMode": "value_and_name", - "wideLayout": true - } - }, - "version": "13.2.0-28666480772" - } - } - }, - "panel-3": { - "kind": "Panel", - "spec": { - "data": { - "kind": "QueryGroup", - "spec": { - "queries": [ - { - "kind": "PanelQuery", - "spec": { - "hidden": false, - "query": { - "datasource": { - "name": "grafanacloud-projairi-prom" - }, - "group": "prometheus", - "kind": "DataQuery", - "spec": { - "editorMode": "code", - "expr": "sum(rate(http_server_request_duration_seconds_count{service_name=~\"$service\", deployment_environment=~\"$env\", http_request_method!=\"OPTIONS\"}[5m]))", - "legendFormat": "req/s", - "range": true - }, - "version": "v0" - }, - "refId": "A" - } - } - ], - "queryOptions": {}, - "transformations": [] - } - }, - "description": "5-minute average inbound HTTP request rate. /livez and /readyz (K8s probes) are excluded at the @hono/otel middleware level so this reflects real user traffic.", - "id": 3, - "links": [], - "title": "Req/s (5m)", - "vizConfig": { - "group": "stat", - "kind": "VizConfig", - "spec": { - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "yellow", - "value": 100 - }, - { - "color": "red", - "value": 500 - } - ] - }, - "unit": "reqps", - "decimals": 2 - }, - "overrides": [] - }, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - } - }, - "version": "13.2.0-28666480772" - } - } - }, - "panel-4": { - "kind": "Panel", - "spec": { - "data": { - "kind": "QueryGroup", - "spec": { - "queries": [ - { - "kind": "PanelQuery", - "spec": { - "hidden": false, - "query": { - "datasource": { - "name": "grafanacloud-projairi-prom" - }, - "group": "prometheus", - "kind": "DataQuery", - "spec": { - "editorMode": "code", - "expr": "100 * sum(rate(http_server_request_duration_seconds_count{service_name=~\"$service\", deployment_environment=~\"$env\", http_request_method!=\"OPTIONS\", http_response_status_code=~\"5..\"}[5m])) / sum(rate(http_server_request_duration_seconds_count{service_name=~\"$service\", deployment_environment=~\"$env\", http_request_method!=\"OPTIONS\"}[5m]))", - "legendFormat": "fail %", - "range": true - }, - "version": "v0" - }, - "refId": "A" - } - } - ], - "queryOptions": {}, - "transformations": [] - } - }, - "description": "5xx responses ÷ all responses over the last 5m. Fixed 5m window for an on-call glance (\"is the service failing right now\"). >1% warns, >5% pages.", - "id": 4, - "links": [], - "title": "5xx Rate %", - "vizConfig": { - "group": "gauge", - "kind": "VizConfig", - "spec": { - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "yellow", - "value": 1 - }, - { - "color": "red", - "value": 5 - } - ] - }, - "unit": "percent", - "decimals": 2, - "noValue": "0", - "min": 0, - "max": 10 - }, - "overrides": [] - }, - "options": { - "minVizHeight": 75, - "minVizWidth": 75, - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "sizing": "auto" - } - }, - "version": "13.2.0-28666480772" - } - } - }, - "panel-5": { - "kind": "Panel", - "spec": { - "data": { - "kind": "QueryGroup", - "spec": { - "queries": [ - { - "kind": "PanelQuery", - "spec": { - "hidden": false, - "query": { - "datasource": { - "name": "grafanacloud-projairi-prom" - }, - "group": "prometheus", - "kind": "DataQuery", - "spec": { - "editorMode": "code", - "expr": "sum(rate(gen_ai_client_operation_count_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[5m]))", - "legendFormat": "req/s", - "range": true - }, - "version": "v0" - }, - "refId": "A" - } - } - ], - "queryOptions": {}, - "transformations": [] - } - }, - "description": "5-minute average LLM gateway request rate (chat + tts). For per-model trends see the LLM Gateway row.", - "id": 5, - "links": [], - "title": "LLM Req/s (5m)", - "vizConfig": { - "group": "stat", - "kind": "VizConfig", - "spec": { - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - } - ] - }, - "unit": "reqps", - "decimals": 2 - }, - "overrides": [] - }, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - } - }, - "version": "13.2.0-28666480772" - } - } - }, - "panel-81": { - "kind": "Panel", - "spec": { - "data": { - "kind": "QueryGroup", - "spec": { - "queries": [ - { - "kind": "PanelQuery", - "spec": { - "hidden": false, - "query": { - "datasource": { - "name": "grafanacloud-projairi-prom" - }, - "group": "prometheus", - "kind": "DataQuery", - "spec": { - "editorMode": "code", - "expr": "max(user_active_rolling{service_name=~\"$service\", deployment_environment=~\"$env\", window=\"7d\"})", - "legendFormat": "WAU", - "range": true - }, - "version": "v0" - }, - "refId": "A" - } - } - ], - "queryOptions": {}, - "transformations": [] - } - }, - "description": "Weekly active users — distinct users with activity in the last 7d. Sourced from `user.last_seen_at` (touched on sign-in and every OIDC token refresh) via the `user.active_rolling` gauge. Cluster-wide gauge aggregated with `max()`.", - "id": 81, - "links": [], - "title": "WAU", - "vizConfig": { - "group": "stat", - "kind": "VizConfig", - "spec": { - "fieldConfig": { - "defaults": { - "color": { - "mode": "fixed", - "fixedColor": "blue" - }, - "fieldMinMax": false, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": 0 - } - ] - }, - "unit": "short", - "noValue": "0" - }, - "overrides": [] - }, - "options": { - "colorMode": "none", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": true, - "textMode": "value_and_name", - "wideLayout": true - } - }, - "version": "13.2.0-28666480772" - } - } - }, - "panel-82": { - "kind": "Panel", - "spec": { - "data": { - "kind": "QueryGroup", - "spec": { - "queries": [ - { - "kind": "PanelQuery", - "spec": { - "hidden": false, - "query": { - "datasource": { - "name": "grafanacloud-projairi-prom" - }, - "group": "prometheus", - "kind": "DataQuery", - "spec": { - "editorMode": "code", - "expr": "max(user_active_rolling{service_name=~\"$service\", deployment_environment=~\"$env\", window=\"30d\"})", - "legendFormat": "MAU", - "range": true - }, - "version": "v0" - }, - "refId": "A" - } - } - ], - "queryOptions": {}, - "transformations": [] - } - }, - "description": "Monthly active users — distinct users with activity in the last 30d. Sourced from `user.last_seen_at` (touched on sign-in and every OIDC token refresh) via the `user.active_rolling` gauge. Cluster-wide gauge aggregated with `max()`.", - "id": 82, - "links": [], - "title": "MAU", - "vizConfig": { - "group": "stat", - "kind": "VizConfig", - "spec": { - "fieldConfig": { - "defaults": { - "color": { - "mode": "fixed", - "fixedColor": "blue" - }, - "fieldMinMax": false, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": 0 - } - ] - }, - "unit": "short", - "noValue": "0" - }, - "overrides": [] - }, - "options": { - "colorMode": "none", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": true, - "textMode": "value_and_name", - "wideLayout": true - } - }, - "version": "13.2.0-28666480772" - } - } - }, - "panel-93": { - "kind": "Panel", - "spec": { - "data": { - "kind": "QueryGroup", - "spec": { - "queries": [ - { - "kind": "PanelQuery", - "spec": { - "hidden": false, - "query": { - "datasource": { - "name": "grafanacloud-projairi-prom" - }, - "group": "prometheus", - "kind": "DataQuery", - "spec": { - "editorMode": "code", - "expr": "max(ws_users_online{service_name=~\"$service\", deployment_environment=~\"$env\"})", - "legendFormat": "users", - "instant": true, - "range": false - }, - "version": "v0" - }, - "refId": "A" - } - } - ], - "queryOptions": {}, - "transformations": [] - } - }, - "description": "Cluster-wide distinct authenticated users with at least one active `/ws/chat` connection. Redis returns each per-user broadcast channel once even when that user has multiple tabs or connections across server replicas; every replica reports the same global value, so the query uses `max()`.", - "id": 93, - "links": [], - "title": "Online Users", - "vizConfig": { - "group": "stat", - "kind": "VizConfig", - "spec": { - "fieldConfig": { - "defaults": { - "color": { - "mode": "fixed", - "fixedColor": "purple" - }, - "fieldMinMax": false, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "purple", - "value": 0 - } - ] - }, - "unit": "short", - "noValue": "—" - }, - "overrides": [] - }, - "options": { - "colorMode": "none", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": true, - "textMode": "value_and_name", - "wideLayout": true - } - }, - "version": "13.2.0-28666480772" - } - } - }, - "panel-92": { - "kind": "Panel", - "spec": { - "data": { - "kind": "QueryGroup", - "spec": { - "queries": [ - { - "kind": "PanelQuery", - "spec": { - "hidden": false, - "query": { - "datasource": { - "name": "grafanacloud-projairi-prom" - }, - "group": "prometheus", - "kind": "DataQuery", - "spec": { - "editorMode": "code", - "expr": "sum(ws_connections_active{service_name=~\"$service\", deployment_environment=~\"$env\"})", - "legendFormat": "connections", - "range": true - }, - "version": "v0" - }, - "refId": "A" - } - } - ], - "queryOptions": {}, - "transformations": [] - } - }, - "description": "Concurrent WebSocket connections over time (`sum` across replicas). A cliff to zero with no matching deploy = mass disconnect (LB drop, network blackhole); a slow ramp without disconnects = connection leak.", - "id": 92, - "links": [], - "title": "WS Connections", - "vizConfig": { - "group": "timeseries", - "kind": "VizConfig", - "spec": { - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 30, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "options": { - "annotations": { - "clustering": -1, - "multiLane": false - }, - "legend": { - "calcs": [ - "lastNotNull", - "max" - ], - "displayMode": "table", - "enableFacetedFilter": false, - "overflow": "ellipsis", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - } - }, - "version": "13.2.0-28666480772" - } - } - }, - "panel-95": { - "kind": "Panel", - "spec": { - "data": { - "kind": "QueryGroup", - "spec": { - "queries": [ - { - "kind": "PanelQuery", - "spec": { - "hidden": false, - "query": { - "datasource": { - "name": "grafanacloud-projairi-prom" - }, - "group": "prometheus", - "kind": "DataQuery", - "spec": { - "editorMode": "code", - "expr": "sum(increase(airi_product_events_total{service_name=~\"$service\", deployment_environment=~\"$env\", feature!=\"\", action!=\"\"}[$__range]))", - "legendFormat": "events", - "range": true - }, - "version": "v0" - }, - "refId": "A" - } - } - ], - "queryOptions": {}, - "transformations": [] - } - }, - "description": "Total first-party product analytics events over the dashboard range. This is event volume, not distinct users — distinct-user counts come from the Postgres `product_events` table.", - "id": 95, - "links": [], - "title": "Product Events (range)", - "vizConfig": { - "group": "stat", - "kind": "VizConfig", - "spec": { - "fieldConfig": { - "defaults": { - "color": { - "mode": "fixed", - "fixedColor": "blue" - }, - "fieldMinMax": false, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": 0 - } - ] - }, - "unit": "short", - "noValue": "0" - }, - "overrides": [] - }, - "options": { - "colorMode": "none", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": true, - "textMode": "value_and_name", - "wideLayout": true - } - }, - "version": "13.2.0-28666480772" - } - } - }, - "panel-96": { - "kind": "Panel", - "spec": { - "data": { - "kind": "QueryGroup", - "spec": { - "queries": [ - { - "kind": "PanelQuery", - "spec": { - "hidden": false, - "query": { - "datasource": { - "name": "grafanacloud-projairi-prom" - }, - "group": "prometheus", - "kind": "DataQuery", - "spec": { - "editorMode": "code", - "expr": "100 * sum(increase(airi_product_events_total{service_name=~\"$service\", deployment_environment=~\"$env\", feature!=\"\", action!=\"\", status=\"failed\"}[$__range])) / clamp_min(sum(increase(airi_product_events_total{service_name=~\"$service\", deployment_environment=~\"$env\", feature!=\"\", action!=\"\"}[$__range])), 1)", - "legendFormat": "failed %", - "range": true - }, - "version": "v0" - }, - "refId": "A" - } - } - ], - "queryOptions": {}, - "transformations": [] - } - }, - "description": "Failed product events ÷ all product events over the dashboard range. Uses only bounded labels (`feature`, `action`, `status`, `source`); no user/session/request identifiers are present in Prometheus.", - "id": 96, - "links": [], - "title": "Product Failure %", - "vizConfig": { - "group": "gauge", - "kind": "VizConfig", - "spec": { - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "yellow", - "value": 2 - }, - { - "color": "red", - "value": 10 - } - ] - }, - "unit": "percent", - "decimals": 2, - "noValue": "0", - "min": 0, - "max": 20 - }, - "overrides": [] - }, - "options": { - "minVizHeight": 75, - "minVizWidth": 75, - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "sizing": "auto" - } - }, - "version": "13.2.0-28666480772" - } - } - }, - "panel-97": { - "kind": "Panel", - "spec": { - "data": { - "kind": "QueryGroup", - "spec": { - "queries": [ - { - "kind": "PanelQuery", - "spec": { - "hidden": false, - "query": { - "datasource": { - "name": "grafanacloud-projairi-prom" - }, - "group": "prometheus", - "kind": "DataQuery", - "spec": { - "editorMode": "code", - "expr": "topk(12, sum by (feature, action, status) (increase(airi_product_events_total{service_name=~\"$service\", deployment_environment=~\"$env\", feature!=\"\", action!=\"\"}[$__range])))", - "legendFormat": "{{feature}} · {{action}} · {{status}}", - "instant": true, - "range": false - }, - "version": "v0" - }, - "refId": "A" - } - } - ], - "queryOptions": {}, - "transformations": [] - } - }, - "description": "Top product actions by event count over the dashboard range. Use this to see which features are actually being exercised after deployment; pair with DB `count(distinct user_id)` for user counts.", - "id": 97, - "links": [], - "title": "Top Product Actions (range)", - "vizConfig": { - "group": "bargauge", - "kind": "VizConfig", - "spec": { - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - } - ] - }, - "unit": "short", - "noValue": "0" - }, - "overrides": [] - }, - "options": { - "displayMode": "gradient", - "maxVizHeight": 300, - "minVizHeight": 12, - "minVizWidth": 8, - "namePlacement": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showUnfilled": true, - "sizing": "auto", - "valueMode": "color" - } - }, - "version": "13.2.0-28666480772" - } - } - }, - "panel-98": { - "kind": "Panel", - "spec": { - "data": { - "kind": "QueryGroup", - "spec": { - "queries": [ - { - "kind": "PanelQuery", - "spec": { - "hidden": false, - "query": { - "datasource": { - "name": "grafanacloud-projairi-prom" - }, - "group": "prometheus", - "kind": "DataQuery", - "spec": { - "editorMode": "code", - "expr": "sum by (feature, action, status) (rate(airi_product_events_total{service_name=~\"$service\", deployment_environment=~\"$env\", feature!=\"\", action!=\"\"}[$__rate_interval]))", - "legendFormat": "{{feature}} · {{action}} · {{status}}", - "range": true - }, - "version": "v0" - }, - "refId": "A" - } - } - ], - "queryOptions": {}, - "transformations": [] - } - }, - "description": "Product event rate by feature/action/status. This is the Prometheus-safe trend view; user-level analysis remains in Postgres `product_events`.", - "id": 98, - "links": [], - "title": "Product Event Rate", - "vizConfig": { - "group": "timeseries", - "kind": "VizConfig", - "spec": { - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 15, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - } - ] - }, - "unit": "eps" - }, - "overrides": [] - }, - "options": { - "annotations": { - "clustering": -1, - "multiLane": false - }, - "legend": { - "calcs": [ - "lastNotNull", - "max" - ], - "displayMode": "table", - "enableFacetedFilter": false, - "overflow": "ellipsis", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - } - }, - "version": "13.2.0-28666480772" - } - } - }, - "panel-99": { - "kind": "Panel", - "spec": { - "data": { - "kind": "QueryGroup", - "spec": { - "queries": [ - { - "kind": "PanelQuery", - "spec": { - "hidden": false, - "query": { - "datasource": { - "name": "grafanacloud-projairi-prom" - }, - "group": "prometheus", - "kind": "DataQuery", - "spec": { - "editorMode": "code", - "expr": "max(user_active_rolling{service_name=~\"$service\", deployment_environment=~\"$env\", window=\"24h\"})", - "legendFormat": "DAU", - "range": true - }, - "version": "v0" - }, - "refId": "A" - } - } - ], - "queryOptions": {}, - "transformations": [] - } - }, - "description": "Rolling 24-hour distinct active users over time. This is the trend view of `user.active_rolling`; the duplicate point-in-time DAU stat is intentionally omitted from this dashboard.", - "id": 99, - "links": [], - "title": "DAU Trend", - "vizConfig": { - "group": "timeseries", - "kind": "VizConfig", - "spec": { - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 17, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "options": { - "annotations": { - "clustering": -1, - "multiLane": false - }, - "legend": { - "calcs": [ - "lastNotNull", - "max" - ], - "displayMode": "table", - "enableFacetedFilter": false, - "overflow": "ellipsis", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - } - }, - "version": "13.2.0-28666480772" - } - } - }, - "panel-16": { - "kind": "Panel", - "spec": { - "data": { - "kind": "QueryGroup", - "spec": { - "queries": [ - { - "kind": "PanelQuery", - "spec": { - "hidden": false, - "query": { - "datasource": { - "name": "grafanacloud-projairi-prom" - }, - "group": "prometheus", - "kind": "DataQuery", - "spec": { - "editorMode": "code", - "expr": "topk(50, sum by (http_route) (increase(http_server_request_duration_seconds_count{service_name=~\"$service\", deployment_environment=~\"$env\", http_request_method!=\"OPTIONS\", http_route!=\"\"}[$__range])))", - "legendFormat": "{{http_route}}", - "instant": true, - "range": false - }, - "version": "v0" - }, - "refId": "A" - } - } - ], - "queryOptions": {}, - "transformations": [] - } - }, - "description": "Top Hono-matched routes by request count over the dashboard range. The main traffic list: which API surfaces are hottest. Wildcard patterns like `/api/v1/openai/*` are requests that did not reach a concrete handler (404 / auth-rejected); concrete paths are successful routes.", - "id": 16, - "links": [], - "title": "Top Routes by Requests (range)", - "vizConfig": { - "group": "bargauge", - "kind": "VizConfig", - "spec": { - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "options": { - "displayMode": "gradient", - "maxVizHeight": 300, - "minVizHeight": 12, - "minVizWidth": 8, - "namePlacement": "auto", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showUnfilled": true, - "sizing": "auto", - "valueMode": "color" - } - }, - "version": "13.2.0-28666480772" - } - } - }, - "panel-40": { - "kind": "Panel", - "spec": { - "data": { - "kind": "QueryGroup", - "spec": { - "queries": [ - { - "kind": "PanelQuery", - "spec": { - "hidden": false, - "query": { - "datasource": { - "name": "grafanacloud-projairi-prom" - }, - "group": "prometheus", - "kind": "DataQuery", - "spec": { - "editorMode": "code", - "expr": "sum by (http_response_status_code) (rate(http_server_request_duration_seconds_count{service_name=~\"$service\", deployment_environment=~\"$env\", http_request_method!=\"OPTIONS\"}[$__rate_interval]))", - "legendFormat": "{{http_response_status_code}}", - "range": true - }, - "version": "v0" - }, - "refId": "A" - } - } - ], - "queryOptions": {}, - "transformations": [] - } - }, - "description": "Stacked non-OPTIONS request rate by HTTP status code. A new or growing 4xx / 5xx band flags a traffic-quality or service-health change.", - "id": 40, - "links": [], - "title": "HTTP Status Rate", - "vizConfig": { - "group": "timeseries", - "kind": "VizConfig", - "spec": { - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 60, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "options": { - "annotations": { - "clustering": -1, - "multiLane": false - }, - "legend": { - "calcs": [ - "lastNotNull", - "max" - ], - "displayMode": "table", - "enableFacetedFilter": false, - "overflow": "ellipsis", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - } - }, - "version": "13.2.0-28666480772" - } - } - }, - "panel-20": { - "kind": "Panel", - "spec": { - "data": { - "kind": "QueryGroup", - "spec": { - "queries": [ - { - "kind": "PanelQuery", - "spec": { - "hidden": false, - "query": { - "datasource": { - "name": "grafanacloud-projairi-prom" - }, - "group": "prometheus", - "kind": "DataQuery", - "spec": { - "editorMode": "code", - "expr": "histogram_quantile(0.95, sum by (le, http_route) (\n rate(http_server_request_duration_seconds_bucket{service_name=~\"$service\", deployment_environment=~\"$env\", http_request_method!=\"OPTIONS\", http_route!~\"/api/v1/openai/.*\", http_response_status_code!=\"404\"}[$__rate_interval])\n))", - "legendFormat": "{{http_route}}", - "range": true - }, - "version": "v0" - }, - "refId": "A" - } - } - ], - "queryOptions": {}, - "transformations": [] - } - }, - "description": "P95 Hono request duration by matched route. Histogram buckets are merged across replicas while preserving `le`, then interpolated by `histogram_quantile`; values are estimates bounded by the configured bucket widths.", - "id": 20, - "links": [], - "title": "Request Latency P95 by Route", - "vizConfig": { - "group": "timeseries", - "kind": "VizConfig", - "spec": { - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "options": { - "annotations": { - "clustering": -1, - "multiLane": false - }, - "legend": { - "calcs": [ - "lastNotNull", - "max" - ], - "displayMode": "table", - "enableFacetedFilter": false, - "overflow": "ellipsis", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - } - }, - "version": "13.2.0-28666480772" - } - } - }, - "panel-94": { - "kind": "Panel", - "spec": { - "data": { - "kind": "QueryGroup", - "spec": { - "queries": [ - { - "kind": "PanelQuery", - "spec": { - "hidden": false, - "query": { - "datasource": { - "name": "grafanacloud-projairi-prom" - }, - "group": "prometheus", - "kind": "DataQuery", - "spec": { - "editorMode": "code", - "expr": "sum by (http_route, http_response_status_code) (increase(http_server_request_duration_seconds_count{service_name=~\"$service\", deployment_environment=~\"$env\", http_request_method!=\"OPTIONS\", http_response_status_code!~\"2..|3..|401|402|404\"}[$__rate_interval]))", - "legendFormat": "{{http_response_status_code}} {{http_route}}", - "range": true - }, - "version": "v0" - }, - "refId": "A" - } - } - ], - "queryOptions": {}, - "transformations": [] - } - }, - "description": "Error responses per route, broken out by status code. Excludes success (2xx/3xx) and the expected-client-error codes 401/402/404 (auth-required / payment-required / not-found noise) so the curve isolates real failures: 4xx like 400/403/422/429 and all 5xx. The per-route companion to the aggregate Error Rate % stat.", - "id": 94, - "links": [], - "title": "Errors by Route", - "vizConfig": { - "group": "timeseries", - "kind": "VizConfig", - "spec": { - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "options": { - "annotations": { - "clustering": -1, - "multiLane": false - }, - "legend": { - "calcs": [ - "lastNotNull", - "max" - ], - "displayMode": "table", - "enableFacetedFilter": false, - "overflow": "ellipsis", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - } - }, - "version": "13.2.0-28666480772" - } - } - }, - "panel-11": { - "kind": "Panel", - "spec": { - "data": { - "kind": "QueryGroup", - "spec": { - "queries": [ - { - "kind": "PanelQuery", - "spec": { - "hidden": false, - "query": { - "datasource": { - "name": "grafanacloud-projairi-prom" - }, - "group": "prometheus", - "kind": "DataQuery", - "spec": { - "editorMode": "code", - "expr": "sum by (gen_ai_request_model) (increase(gen_ai_client_operation_count_total{service_name=~\"$service\", deployment_environment=~\"$env\", gen_ai_request_model!=\"\"}[$__range]))", - "legendFormat": "{{gen_ai_request_model}}", - "instant": true, - "range": false - }, - "version": "v0" - }, - "refId": "A" - } - } - ], - "queryOptions": {}, - "transformations": [] - } - }, - "description": "Per-model request count over the visible dashboard range (chat + tts). The pie shows each model's share without depending on Grafana sampling resolution.", - "id": 11, - "links": [], - "title": "LLM Requests by Model (range)", - "vizConfig": { - "group": "piechart", - "kind": "VizConfig", - "spec": { - "fieldConfig": { - "defaults": { - "color": { - "fixedColor": "#73BF69", - "mode": "palette-classic" - }, - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - } - }, - "unit": "short" - }, - "overrides": [] - }, - "options": { - "displayLabels": [ - "percent" - ], - "legend": { - "displayMode": "table", - "overflow": "ellipsis", - "placement": "bottom", - "showLegend": true - }, - "pieType": "pie", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "sort": "desc", - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } - } - }, - "version": "13.2.0-28666480772" - } - } - }, - "panel-21": { - "kind": "Panel", - "spec": { - "data": { - "kind": "QueryGroup", - "spec": { - "queries": [ - { - "kind": "PanelQuery", - "spec": { - "hidden": false, - "query": { - "datasource": { - "name": "grafanacloud-projairi-prom" - }, - "group": "prometheus", - "kind": "DataQuery", - "spec": { - "editorMode": "code", - "expr": "histogram_quantile(0.95, sum by (le) (rate(gen_ai_client_first_token_duration_seconds_bucket{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval])))", - "legendFormat": "TTFB p95", - "range": true - }, - "version": "v0" - }, - "refId": "A" - } - }, - { - "kind": "PanelQuery", - "spec": { - "hidden": false, - "query": { - "datasource": { - "name": "grafanacloud-projairi-prom" - }, - "group": "prometheus", - "kind": "DataQuery", - "spec": { - "editorMode": "code", - "expr": "histogram_quantile(0.95, sum by (le) (rate(gen_ai_client_operation_duration_seconds_bucket{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval])))", - "legendFormat": "end-to-end p95", - "range": true - }, - "version": "v0" - }, - "refId": "B" - } - } - ], - "queryOptions": {}, - "transformations": [] - } - }, - "description": "Two P95 latency signals for the LLM gateway, aggregated across models. TTFB = time to first streamed token (streaming chat UX). End-to-end = full operation duration — the only latency signal for non-streaming chat and TTS, which have no first-token event.", - "id": 21, - "links": [], - "title": "LLM Latency P95", - "vizConfig": { - "group": "timeseries", - "kind": "VizConfig", - "spec": { - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "options": { - "annotations": { - "clustering": -1, - "multiLane": false - }, - "legend": { - "calcs": [ - "lastNotNull", - "max" - ], - "displayMode": "table", - "enableFacetedFilter": false, - "overflow": "ellipsis", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - } - }, - "version": "13.2.0-28666480772" - } - } - }, - "panel-66": { - "kind": "Panel", - "spec": { - "data": { - "kind": "QueryGroup", - "spec": { - "queries": [ - { - "kind": "PanelQuery", - "spec": { - "hidden": false, - "query": { - "datasource": { - "name": "grafanacloud-projairi-prom" - }, - "group": "prometheus", - "kind": "DataQuery", - "spec": { - "editorMode": "code", - "expr": "sum by (provider) (rate(gen_ai_client_operation_count_total{service_name=~\"$service\", deployment_environment=~\"$env\", provider!=\"\"}[$__rate_interval]))", - "legendFormat": "{{provider}}", - "range": true - }, - "version": "v0" - }, - "refId": "A" - } - } - ], - "queryOptions": {}, - "transformations": [] - } - }, - "description": "Outbound request rate to each upstream provider (chat + tts), as our gateway sees it. The RPM / 调用次数 screens on the provider consoles, unified. provider = upstream hostname the router used.", - "id": 66, - "links": [], - "title": "Requests/s by Provider", - "vizConfig": { - "group": "timeseries", - "kind": "VizConfig", - "spec": { - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "options": { - "annotations": { - "clustering": -1, - "multiLane": false - }, - "legend": { - "calcs": [ - "lastNotNull", - "max" - ], - "displayMode": "table", - "enableFacetedFilter": false, - "overflow": "ellipsis", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - } - }, - "version": "13.2.0-28666480772" - } - } - }, - "panel-67": { - "kind": "Panel", - "spec": { - "data": { - "kind": "QueryGroup", - "spec": { - "queries": [ - { - "kind": "PanelQuery", - "spec": { - "hidden": false, - "query": { - "datasource": { - "name": "grafanacloud-projairi-prom" - }, - "group": "prometheus", - "kind": "DataQuery", - "spec": { - "editorMode": "code", - "expr": "histogram_quantile(0.95, sum by (le, provider) (rate(gen_ai_client_operation_duration_seconds_bucket{service_name=~\"$service\", deployment_environment=~\"$env\", provider!=\"\"}[$__rate_interval])))", - "legendFormat": "{{provider}}", - "range": true - }, - "version": "v0" - }, - "refId": "A" - } - } - ], - "queryOptions": {}, - "transformations": [] - } - }, - "description": "P95 upstream call duration per provider (chat + tts), across models. Mirrors each provider console's 调用时长 p95/p99 panel — but here every provider is on one axis.", - "id": 67, - "links": [], - "title": "Provider Latency P95", - "vizConfig": { - "group": "timeseries", - "kind": "VizConfig", - "spec": { - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "options": { - "annotations": { - "clustering": -1, - "multiLane": false - }, - "legend": { - "calcs": [ - "lastNotNull", - "max" - ], - "displayMode": "table", - "enableFacetedFilter": false, - "overflow": "ellipsis", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - } - }, - "version": "13.2.0-28666480772" - } - } - }, - "panel-68": { - "kind": "Panel", - "spec": { - "data": { - "kind": "QueryGroup", - "spec": { - "queries": [ - { - "kind": "PanelQuery", - "spec": { - "hidden": false, - "query": { - "datasource": { - "name": "grafanacloud-projairi-prom" - }, - "group": "prometheus", - "kind": "DataQuery", - "spec": { - "editorMode": "code", - "expr": "100 * sum by (provider) (rate(gen_ai_client_operation_count_total{service_name=~\"$service\", deployment_environment=~\"$env\", provider!=\"\", http_response_status_code=~\"4..|5..\"}[$__rate_interval])) / sum by (provider) (rate(gen_ai_client_operation_count_total{service_name=~\"$service\", deployment_environment=~\"$env\", provider!=\"\"}[$__rate_interval]))", - "legendFormat": "{{provider}}", - "range": true - }, - "version": "v0" - }, - "refId": "A" - } - } - ], - "queryOptions": {}, - "transformations": [] - } - }, - "description": "4xx + 5xx ÷ all requests per provider, our side of the call. Matches each provider 失败率 panel. Pair with Upstream Errors by Status Code (LLM Router Health) to see which codes drive it.", - "id": 68, - "links": [], - "title": "Provider Failure %", - "vizConfig": { - "group": "timeseries", - "kind": "VizConfig", - "spec": { - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "options": { - "annotations": { - "clustering": -1, - "multiLane": false - }, - "legend": { - "calcs": [ - "lastNotNull", - "max" - ], - "displayMode": "table", - "enableFacetedFilter": false, - "overflow": "ellipsis", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - } - }, - "version": "13.2.0-28666480772" - } - } - }, - "panel-69": { - "kind": "Panel", - "spec": { - "data": { - "kind": "QueryGroup", - "spec": { - "queries": [ - { - "kind": "PanelQuery", - "spec": { - "hidden": false, - "query": { - "datasource": { - "name": "grafanacloud-projairi-prom" - }, - "group": "prometheus", - "kind": "DataQuery", - "spec": { - "editorMode": "code", - "expr": "sum by (model) (rate(airi_billing_tts_chars_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))", - "legendFormat": "{{model}}", - "range": true - }, - "version": "v0" - }, - "refId": "A" - } - } - ], - "queryOptions": {}, - "transformations": [] - } - }, - "description": "Billed TTS characters per second by model (from `airi.billing.tts.chars`). The 用量统计「字数」screen on the TTS consoles (豆包 / 阿里), unified. Integrate over the range for a window total.", - "id": 69, - "links": [], - "title": "TTS Characters/s by Model", - "vizConfig": { - "group": "timeseries", - "kind": "VizConfig", - "spec": { - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "options": { - "annotations": { - "clustering": -1, - "multiLane": false - }, - "legend": { - "calcs": [ - "lastNotNull", - "max" - ], - "displayMode": "table", - "enableFacetedFilter": false, - "overflow": "ellipsis", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - } - }, - "version": "13.2.0-28666480772" - } - } - }, - "panel-73": { - "kind": "Panel", - "spec": { - "data": { - "kind": "QueryGroup", - "spec": { - "queries": [ - { - "kind": "PanelQuery", - "spec": { - "hidden": false, - "query": { - "datasource": { - "name": "grafanacloud-projairi-prom" - }, - "group": "prometheus", - "kind": "DataQuery", - "spec": { - "editorMode": "code", - "expr": "sum(increase(gen_ai_client_token_usage_input_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__range]))", - "legendFormat": "input", - "range": true - }, - "version": "v0" - }, - "refId": "A" - } - }, - { - "kind": "PanelQuery", - "spec": { - "hidden": false, - "query": { - "datasource": { - "name": "grafanacloud-projairi-prom" - }, - "group": "prometheus", - "kind": "DataQuery", - "spec": { - "editorMode": "code", - "expr": "sum(increase(gen_ai_client_token_usage_output_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__range]))", - "legendFormat": "output", - "range": true - }, - "version": "v0" - }, - "refId": "B" - } - } - ], - "queryOptions": {}, - "transformations": [] - } - }, - "description": "Total input and output tokens billed over the dashboard range, from the upstream `usage` block (requests where the upstream omits usage are not counted). The cumulative counterpart to panel-71 throughput — use for \"how many tokens did we burn this window\" cost math.", - "id": 73, - "links": [], - "title": "Tokens Consumed (range)", - "vizConfig": { - "group": "stat", - "kind": "VizConfig", - "spec": { - "fieldConfig": { - "defaults": { - "color": { - "mode": "fixed", - "fixedColor": "blue" - }, - "fieldMinMax": false, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": 0 - } - ] - }, - "unit": "short", - "noValue": "0" - }, - "overrides": [] - }, - "options": { - "colorMode": "none", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": true, - "textMode": "value_and_name", - "wideLayout": true - } - }, - "version": "13.2.0-28666480772" - } - } - }, - "panel-71": { - "kind": "Panel", - "spec": { - "data": { - "kind": "QueryGroup", - "spec": { - "queries": [ - { - "kind": "PanelQuery", - "spec": { - "hidden": false, - "query": { - "datasource": { - "name": "grafanacloud-projairi-prom" - }, - "group": "prometheus", - "kind": "DataQuery", - "spec": { - "editorMode": "code", - "expr": "sum(rate(gen_ai_client_token_usage_input_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))", - "legendFormat": "input tokens/s", - "range": true - }, - "version": "v0" - }, - "refId": "A" - } - }, - { - "kind": "PanelQuery", - "spec": { - "hidden": false, - "query": { - "datasource": { - "name": "grafanacloud-projairi-prom" - }, - "group": "prometheus", - "kind": "DataQuery", - "spec": { - "editorMode": "code", - "expr": "sum(rate(gen_ai_client_token_usage_output_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))", - "legendFormat": "output tokens/s", - "range": true - }, - "version": "v0" - }, - "refId": "B" - } - } - ], - "queryOptions": {}, - "transformations": [] - } - }, - "description": "Input vs output token throughput across the LLM gateway (tokens/sec). Recorded per request from the upstream `usage` block. Use for capacity planning and cost estimation. input = prompt tokens consumed; output = completion tokens generated.", - "id": 71, - "links": [], - "title": "LLM Token Throughput", - "vizConfig": { - "group": "timeseries", - "kind": "VizConfig", - "spec": { - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "options": { - "annotations": { - "clustering": -1, - "multiLane": false - }, - "legend": { - "calcs": [ - "lastNotNull", - "max" - ], - "displayMode": "table", - "enableFacetedFilter": false, - "overflow": "ellipsis", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - } - }, - "version": "13.2.0-28666480772" - } - } - }, - "panel-43": { - "kind": "Panel", - "spec": { - "data": { - "kind": "QueryGroup", - "spec": { - "queries": [ - { - "kind": "PanelQuery", - "spec": { - "hidden": false, - "query": { - "datasource": { - "name": "grafanacloud-projairi-prom" - }, - "group": "prometheus", - "kind": "DataQuery", - "spec": { - "editorMode": "code", - "expr": "sum(increase(airi_billing_flux_unbilled_total{service_name=~\"$service\", deployment_environment=~\"$env\", reason!=\"partial_debit_drained\"}[$__range]))", - "legendFormat": "flux", - "range": true - }, - "version": "v0" - }, - "refId": "A" - } - } - ], - "queryOptions": {}, - "transformations": [] - } - }, - "description": "Flux owed by users but never debited for unexpected reasons (excludes `partial_debit_drained`, a known partial-balance drain path). Real revenue leak — DB latency and HTTP 5xx alerts do NOT cover this, because the response was 2xx and the catch path is silent.", - "id": 43, - "links": [], - "title": "⚠ Flux Unbilled (range)", - "vizConfig": { - "group": "stat", - "kind": "VizConfig", - "spec": { - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 1 - } - ] - }, - "unit": "short", - "noValue": "0" - }, - "overrides": [] - }, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - } - }, - "version": "13.2.0-28666480772" - } - } - }, - "panel-41": { - "kind": "Panel", - "spec": { - "data": { - "kind": "QueryGroup", - "spec": { - "queries": [ - { - "kind": "PanelQuery", - "spec": { - "hidden": false, - "query": { - "datasource": { - "name": "grafanacloud-projairi-prom" - }, - "group": "prometheus", - "kind": "DataQuery", - "spec": { - "editorMode": "code", - "expr": "sum(increase(airi_gen_ai_stream_interrupted_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__range]))", - "legendFormat": "interruptions", - "range": true - }, - "version": "v0" - }, - "refId": "A" - } - } - ], - "queryOptions": {}, - "transformations": [] - } - }, - "description": "LLM streams that died mid-flight over the dashboard range. before_first_chunk = upstream blew up; mid_stream = partial delivery (user saw a broken response).", - "id": 41, - "links": [], - "title": "Stream Interruptions (range)", - "vizConfig": { - "group": "stat", - "kind": "VizConfig", - "spec": { - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "yellow", - "value": 1 - }, - { - "color": "red", - "value": 10 - } - ] - }, - "unit": "short", - "noValue": "0" - }, - "overrides": [] - }, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - } - }, - "version": "13.2.0-28666480772" - } - } - }, - "panel-60": { - "kind": "Panel", - "spec": { - "data": { - "kind": "QueryGroup", - "spec": { - "queries": [ - { - "kind": "PanelQuery", - "spec": { - "hidden": false, - "query": { - "datasource": { - "name": "grafanacloud-projairi-prom" - }, - "group": "prometheus", - "kind": "DataQuery", - "spec": { - "editorMode": "code", - "expr": "sum(increase(airi_gen_ai_gateway_key_exhausted_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[5m]))", - "legendFormat": "events", - "range": true - }, - "version": "v0" - }, - "refId": "A" - } - } - ], - "queryOptions": {}, - "transformations": [] - } - }, - "description": "Number of (model, upstream) pairs that ran out of usable keys within one user request over the last 5 minutes. Sustained > 0 = a provider account is dead or every stored ciphertext is failing to decrypt — page on-call.", - "id": 60, - "links": [], - "title": "Key Exhausted (5m)", - "vizConfig": { - "group": "stat", - "kind": "VizConfig", - "spec": { - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 1 - } - ] - }, - "unit": "short", - "noValue": "0" - }, - "overrides": [] - }, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - } - }, - "version": "13.2.0-28666480772" - } - } - }, - "panel-61": { - "kind": "Panel", - "spec": { - "data": { - "kind": "QueryGroup", - "spec": { - "queries": [ - { - "kind": "PanelQuery", - "spec": { - "hidden": false, - "query": { - "datasource": { - "name": "grafanacloud-projairi-prom" - }, - "group": "prometheus", - "kind": "DataQuery", - "spec": { - "editorMode": "code", - "expr": "sum(increase(airi_gen_ai_gateway_decrypt_failures_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[5m]))", - "legendFormat": "events", - "range": true - }, - "version": "v0" - }, - "refId": "A" - } - } - ], - "queryOptions": {}, - "transformations": [] - } - }, - "description": "Envelope-crypto decrypt failures in the key rotator. Non-zero is security-relevant: either the master key was rotated without re-wrapping ciphertexts, or someone forged a config blob.", - "id": 61, - "links": [], - "title": "Decrypt Failures (5m)", - "vizConfig": { - "group": "stat", - "kind": "VizConfig", - "spec": { - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "red", - "value": 1 - } - ] - }, - "unit": "short", - "noValue": "0" - }, - "overrides": [] - }, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - } - }, - "version": "13.2.0-28666480772" - } - } - }, - "panel-62": { - "kind": "Panel", - "spec": { - "data": { - "kind": "QueryGroup", - "spec": { - "queries": [ - { - "kind": "PanelQuery", - "spec": { - "hidden": false, - "query": { - "datasource": { - "name": "grafanacloud-projairi-prom" - }, - "group": "prometheus", - "kind": "DataQuery", - "spec": { - "editorMode": "code", - "expr": "100 * sum(rate(airi_gen_ai_gateway_fallback_count_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[5m])) / sum(rate(gen_ai_client_operation_count_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[5m]))", - "legendFormat": "fallback %", - "range": true - }, - "version": "v0" - }, - "refId": "A" - } - } - ], - "queryOptions": {}, - "transformations": [] - } - }, - "description": "Fallback attempts ÷ total LLM operations over the last 5m. Sustained > 30% means one provider is degraded and the router is silently masking it for users while burning quota on the failing upstream.", - "id": 62, - "links": [], - "title": "Fallback Ratio % (5m)", - "vizConfig": { - "group": "gauge", - "kind": "VizConfig", - "spec": { - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - }, - { - "color": "yellow", - "value": 10 - }, - { - "color": "red", - "value": 30 - } - ] - }, - "unit": "percent", - "decimals": 1, - "noValue": "0", - "min": 0, - "max": 100 - }, - "overrides": [] - }, - "options": { - "minVizHeight": 75, - "minVizWidth": 75, - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "sizing": "auto" - } - }, - "version": "13.2.0-28666480772" - } - } - }, - "panel-65": { - "kind": "Panel", - "spec": { - "data": { - "kind": "QueryGroup", - "spec": { - "queries": [ - { - "kind": "PanelQuery", - "spec": { - "hidden": false, - "query": { - "datasource": { - "name": "grafanacloud-projairi-prom" - }, - "group": "prometheus", - "kind": "DataQuery", - "spec": { - "editorMode": "code", - "expr": "sum by (provider, status_code) (rate(airi_gen_ai_gateway_upstream_errors_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))", - "legendFormat": "{{provider}} · {{status_code}}", - "range": true - }, - "version": "v0" - }, - "refId": "A" - } - } - ], - "queryOptions": {}, - "transformations": [] - } - }, - "description": "Per-upstream non-2xx response rate split by status code. Only counts attempts where the upstream actually answered. 401/403 = bad key; 429 = quota; 5xx = upstream outage.", - "id": 65, - "links": [], - "title": "Upstream Errors by Status Code", - "vizConfig": { - "group": "timeseries", - "kind": "VizConfig", - "spec": { - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - } - ] - }, - "unit": "ops" - }, - "overrides": [] - }, - "options": { - "annotations": { - "clustering": -1, - "multiLane": false - }, - "legend": { - "calcs": [ - "lastNotNull", - "max" - ], - "displayMode": "table", - "enableFacetedFilter": false, - "overflow": "ellipsis", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "hideZeros": false, - "mode": "multi", - "sort": "desc" - } - } - }, - "version": "13.2.0-28666480772" - } - } - }, - "panel-30": { - "kind": "Panel", - "spec": { - "data": { - "kind": "QueryGroup", - "spec": { - "queries": [ - { - "kind": "PanelQuery", - "spec": { - "hidden": false, - "query": { - "datasource": { - "name": "grafanacloud-projairi-prom" - }, - "group": "prometheus", - "kind": "DataQuery", - "spec": { - "editorMode": "code", - "expr": "sum by (currency) (increase(airi_stripe_revenue_minor_unit_total{service_name=~\"$service\", deployment_environment=~\"$env\", currency!=\"\"}[$__range])) / 100", - "legendFormat": "{{currency}}", - "range": true - }, - "version": "v0" - }, - "refId": "A" - } - } - ], - "queryOptions": {}, - "transformations": [] - } - }, - "description": "Stripe revenue over dashboard range, in major currency unit (cents → dollars). Cross-currency sums are meaningless — always grouped by currency. Empty in dev / fresh deploys.", - "id": 30, - "links": [], - "title": "Revenue (range)", - "vizConfig": { - "group": "stat", - "kind": "VizConfig", - "spec": { - "fieldConfig": { - "defaults": { - "color": { - "mode": "fixed", - "fixedColor": "green" - }, - "fieldMinMax": false, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": 0 - } - ] - }, - "unit": "short", - "decimals": 2, - "noValue": "—" - }, - "overrides": [] - }, - "options": { - "colorMode": "none", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": true, - "textMode": "value_and_name", - "wideLayout": true - } - }, - "version": "13.2.0-28666480772" - } - } - }, - "panel-31": { - "kind": "Panel", - "spec": { - "data": { - "kind": "QueryGroup", - "spec": { - "queries": [ - { - "kind": "PanelQuery", - "spec": { - "hidden": false, - "query": { - "datasource": { - "name": "grafanacloud-projairi-prom" - }, - "group": "prometheus", - "kind": "DataQuery", - "spec": { - "editorMode": "code", - "expr": "100 * sum(increase(stripe_checkout_completed_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__range])) / clamp_min(sum(increase(stripe_checkout_created_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__range])), 1)", - "legendFormat": "completed %", - "range": true - }, - "version": "v0" - }, - "refId": "A" - } - } - ], - "queryOptions": {}, - "transformations": [] - } - }, - "description": "Completed checkouts ÷ created checkouts over dashboard range. Drops can flag price-page bugs or payment-method outages.", - "id": 31, - "links": [], - "title": "Checkout Conversion %", - "vizConfig": { - "group": "gauge", - "kind": "VizConfig", - "spec": { - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": 0 - }, - { - "color": "yellow", - "value": 30 - }, - { - "color": "green", - "value": 60 - } - ] - }, - "unit": "percent", - "decimals": 1, - "noValue": "—", - "min": 0, - "max": 100 - }, - "overrides": [] - }, - "options": { - "minVizHeight": 75, - "minVizWidth": 75, - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "sizing": "auto" - } - }, - "version": "13.2.0-28666480772" - } - } - }, - "panel-32": { - "kind": "Panel", - "spec": { - "data": { - "kind": "QueryGroup", - "spec": { - "queries": [ - { - "kind": "PanelQuery", - "spec": { - "hidden": false, - "query": { - "datasource": { - "name": "grafanacloud-projairi-prom" - }, - "group": "prometheus", - "kind": "DataQuery", - "spec": { - "editorMode": "code", - "expr": "sum by (event_type) (increase(stripe_events_total{service_name=~\"$service\", deployment_environment=~\"$env\", event_type!=\"\"}[$__range]))", - "legendFormat": "{{event_type}}", - "range": true - }, - "version": "v0" - }, - "refId": "A" - } - } - ], - "queryOptions": {}, - "transformations": [] - } - }, - "description": "Webhook events grouped by event.type. Pattern shifts (e.g. surge in invoice.payment_failed) indicate billing health.", - "id": 32, - "links": [], - "title": "Stripe Events (range)", - "vizConfig": { - "group": "stat", - "kind": "VizConfig", - "spec": { - "fieldConfig": { - "defaults": { - "color": { - "mode": "fixed", - "fixedColor": "blue" - }, - "fieldMinMax": false, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "blue", - "value": 0 - } - ] - }, - "unit": "short", - "noValue": "—" - }, - "overrides": [] - }, - "options": { - "colorMode": "none", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": true, - "textMode": "value_and_name", - "wideLayout": true - } - }, - "version": "13.2.0-28666480772" - } - } - }, - "panel-91": { - "kind": "Panel", - "spec": { - "data": { - "kind": "QueryGroup", - "spec": { - "queries": [ - { - "kind": "PanelQuery", - "spec": { - "hidden": false, - "query": { - "datasource": { - "name": "grafanacloud-projairi-logs" - }, - "group": "loki", - "kind": "DataQuery", - "spec": { - "editorMode": "code", - "expr": "{service_name=~\"$service\", deployment_environment=~\"$env\"} | detected_level=~\"warn|error\"", - "legendFormat": "", - "range": true - }, - "version": "v0" - }, - "refId": "A" - } - } - ], - "queryOptions": {}, - "transformations": [] - } - }, - "description": "Server-side warn and error logs from Loki structured metadata. Derived fields make `trace_id` and `req` clickable — `trace_id` jumps to Tempo for full request playback.", - "id": 91, - "links": [], - "title": "Warn / Error Logs", - "vizConfig": { - "group": "logs", - "kind": "VizConfig", - "spec": { - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "options": { - "dedupStrategy": "none", - "enableInfiniteScrolling": false, - "enableLogDetails": true, - "prettifyLogMessage": false, - "showCommonLabels": false, - "showControls": false, - "showFieldSelector": false, - "showLabels": true, - "showLevel": true, - "showLogAttributes": true, - "showTime": true, - "sortOrder": "Descending", - "timestampResolution": "ms", - "unwrappedColumns": false, - "wrapLogMessage": true - } - }, - "version": "13.2.0-28666480772" - } - } - }, - "panel-90": { - "kind": "Panel", - "spec": { - "data": { - "kind": "QueryGroup", - "spec": { - "queries": [ - { - "kind": "PanelQuery", - "spec": { - "hidden": false, - "query": { - "datasource": { - "name": "grafanacloud-projairi-logs" - }, - "group": "loki", - "kind": "DataQuery", - "spec": { - "editorMode": "code", - "expr": "{service_name=~\"$service\", deployment_environment=~\"$env\"} |= ``", - "legendFormat": "", - "range": true - }, - "version": "v0" - }, - "refId": "A" - } - } - ], - "queryOptions": {}, - "transformations": [] - } - }, - "description": "Live application logs from Loki. Filter via the panel UI; click trace_id to jump to Tempo.", - "id": 90, - "links": [], - "title": "Application Logs", - "vizConfig": { - "group": "logs", - "kind": "VizConfig", - "spec": { - "fieldConfig": { - "defaults": {}, - "overrides": [] - }, - "options": { - "dedupStrategy": "none", - "enableInfiniteScrolling": false, - "enableLogDetails": true, - "prettifyLogMessage": false, - "showCommonLabels": false, - "showControls": false, - "showFieldSelector": false, - "showLabels": true, - "showLevel": true, - "showLogAttributes": true, - "showTime": true, - "sortOrder": "Descending", - "timestampResolution": "ms", - "unwrappedColumns": false, - "wrapLogMessage": true - } - }, - "version": "13.2.0-28666480772" - } - } - } - }, - "layout": { - "kind": "RowsLayout", - "spec": { - "rows": [ - { - "kind": "RowsLayoutRow", - "spec": { - "collapse": false, - "layout": { - "kind": "GridLayout", - "spec": { - "items": [ - { - "kind": "GridLayoutItem", - "spec": { - "element": { - "kind": "ElementReference", - "name": "panel-3" - }, - "height": 5, - "width": 4, - "x": 0, - "y": 0 - } - }, - { - "kind": "GridLayoutItem", - "spec": { - "element": { - "kind": "ElementReference", - "name": "panel-5" - }, - "height": 5, - "width": 4, - "x": 4, - "y": 0 - } - }, - { - "kind": "GridLayoutItem", - "spec": { - "element": { - "kind": "ElementReference", - "name": "panel-40" - }, - "height": 6, - "width": 10, - "x": 8, - "y": 0 - } - }, - { - "kind": "GridLayoutItem", - "spec": { - "element": { - "kind": "ElementReference", - "name": "panel-90" - }, - "height": 38, - "width": 6, - "x": 18, - "y": 0 - } - }, - { - "kind": "GridLayoutItem", - "spec": { - "element": { - "kind": "ElementReference", - "name": "panel-93" - }, - "height": 5, - "width": 4, - "x": 0, - "y": 5 - } - }, - { - "kind": "GridLayoutItem", - "spec": { - "element": { - "kind": "ElementReference", - "name": "panel-4" - }, - "height": 5, - "width": 4, - "x": 4, - "y": 5 - } - }, - { - "kind": "GridLayoutItem", - "spec": { - "element": { - "kind": "ElementReference", - "name": "panel-20" - }, - "height": 15, - "width": 10, - "x": 8, - "y": 6 - } - }, - { - "kind": "GridLayoutItem", - "spec": { - "element": { - "kind": "ElementReference", - "name": "panel-92" - }, - "height": 10, - "width": 4, - "x": 0, - "y": 10 - } - }, - { - "kind": "GridLayoutItem", - "spec": { - "element": { - "kind": "ElementReference", - "name": "panel-73" - }, - "height": 10, - "width": 2, - "x": 4, - "y": 10 - } - }, - { - "kind": "GridLayoutItem", - "spec": { - "element": { - "kind": "ElementReference", - "name": "panel-11" - }, - "height": 10, - "width": 2, - "x": 6, - "y": 10 - } - }, - { - "kind": "GridLayoutItem", - "spec": { - "element": { - "kind": "ElementReference", - "name": "panel-16" - }, - "height": 18, - "width": 8, - "x": 0, - "y": 20 - } - }, - { - "kind": "GridLayoutItem", - "spec": { - "element": { - "kind": "ElementReference", - "name": "panel-71" - }, - "height": 5, - "width": 5, - "x": 8, - "y": 21 - } - }, - { - "kind": "GridLayoutItem", - "spec": { - "element": { - "kind": "ElementReference", - "name": "panel-21" - }, - "height": 5, - "width": 5, - "x": 13, - "y": 21 - } - }, - { - "kind": "GridLayoutItem", - "spec": { - "element": { - "kind": "ElementReference", - "name": "panel-69" - }, - "height": 6, - "width": 5, - "x": 8, - "y": 26 - } - }, - { - "kind": "GridLayoutItem", - "spec": { - "element": { - "kind": "ElementReference", - "name": "panel-67" - }, - "height": 6, - "width": 5, - "x": 13, - "y": 26 - } - }, - { - "kind": "GridLayoutItem", - "spec": { - "element": { - "kind": "ElementReference", - "name": "panel-66" - }, - "height": 6, - "width": 5, - "x": 8, - "y": 32 - } - }, - { - "kind": "GridLayoutItem", - "spec": { - "element": { - "kind": "ElementReference", - "name": "panel-68" - }, - "height": 6, - "width": 5, - "x": 13, - "y": 32 - } - } - ] - } - }, - "title": "Service Health" - } - }, - { - "kind": "RowsLayoutRow", - "spec": { - "collapse": false, - "layout": { - "kind": "GridLayout", - "spec": { - "items": [ - { - "kind": "GridLayoutItem", - "spec": { - "element": { - "kind": "ElementReference", - "name": "panel-1" - }, - "height": 5, - "width": 3, - "x": 0, - "y": 0 - } - }, - { - "kind": "GridLayoutItem", - "spec": { - "element": { - "kind": "ElementReference", - "name": "panel-15" - }, - "height": 5, - "width": 3, - "x": 3, - "y": 0 - } - }, - { - "kind": "GridLayoutItem", - "spec": { - "element": { - "kind": "ElementReference", - "name": "panel-99" - }, - "height": 11, - "width": 12, - "x": 6, - "y": 0 - } - }, - { - "kind": "GridLayoutItem", - "spec": { - "element": { - "kind": "ElementReference", - "name": "panel-98" - }, - "height": 11, - "width": 6, - "x": 18, - "y": 0 - } - }, - { - "kind": "GridLayoutItem", - "spec": { - "element": { - "kind": "ElementReference", - "name": "panel-81" - }, - "height": 3, - "width": 3, - "x": 0, - "y": 5 - } - }, - { - "kind": "GridLayoutItem", - "spec": { - "element": { - "kind": "ElementReference", - "name": "panel-82" - }, - "height": 3, - "width": 3, - "x": 3, - "y": 5 - } - }, - { - "kind": "GridLayoutItem", - "spec": { - "element": { - "kind": "ElementReference", - "name": "panel-95" - }, - "height": 9, - "width": 6, - "x": 0, - "y": 11 - } - }, - { - "kind": "GridLayoutItem", - "spec": { - "element": { - "kind": "ElementReference", - "name": "panel-96" - }, - "height": 9, - "width": 6, - "x": 6, - "y": 11 - } - }, - { - "kind": "GridLayoutItem", - "spec": { - "element": { - "kind": "ElementReference", - "name": "panel-97" - }, - "height": 9, - "width": 12, - "x": 12, - "y": 11 - } - } - ] - } - }, - "title": "User Engagement" - } - }, - { - "kind": "RowsLayoutRow", - "spec": { - "collapse": false, - "layout": { - "kind": "GridLayout", - "spec": { - "items": [] - } - }, - "title": "Product Analytics" - } - }, - { - "kind": "RowsLayoutRow", - "spec": { - "collapse": false, - "layout": { - "kind": "GridLayout", - "spec": { - "items": [ - { - "kind": "GridLayoutItem", - "spec": { - "element": { - "kind": "ElementReference", - "name": "panel-94" - }, - "height": 8, - "width": 24, - "x": 0, - "y": 0 - } - } - ] - } - }, - "title": "HTTP" - } - }, - { - "kind": "RowsLayoutRow", - "spec": { - "collapse": false, - "layout": { - "kind": "GridLayout", - "spec": { - "items": [ - { - "kind": "GridLayoutItem", - "spec": { - "element": { - "kind": "ElementReference", - "name": "panel-43" - }, - "height": 7, - "width": 6, - "x": 0, - "y": 0 - } - }, - { - "kind": "GridLayoutItem", - "spec": { - "element": { - "kind": "ElementReference", - "name": "panel-41" - }, - "height": 7, - "width": 6, - "x": 6, - "y": 0 - } - } - ] - } - }, - "title": "LLM Tokens & Quality" - } - }, - { - "kind": "RowsLayoutRow", - "spec": { - "collapse": false, - "layout": { - "kind": "GridLayout", - "spec": { - "items": [ - { - "kind": "GridLayoutItem", - "spec": { - "element": { - "kind": "ElementReference", - "name": "panel-60" - }, - "height": 6, - "width": 6, - "x": 0, - "y": 0 - } - }, - { - "kind": "GridLayoutItem", - "spec": { - "element": { - "kind": "ElementReference", - "name": "panel-61" - }, - "height": 6, - "width": 6, - "x": 6, - "y": 0 - } - }, - { - "kind": "GridLayoutItem", - "spec": { - "element": { - "kind": "ElementReference", - "name": "panel-62" - }, - "height": 6, - "width": 6, - "x": 12, - "y": 0 - } - }, - { - "kind": "GridLayoutItem", - "spec": { - "element": { - "kind": "ElementReference", - "name": "panel-65" - }, - "height": 6, - "width": 6, - "x": 18, - "y": 0 - } - } - ] - } - }, - "title": "LLM Router Health" - } - }, - { - "kind": "RowsLayoutRow", - "spec": { - "collapse": false, - "layout": { - "kind": "GridLayout", - "spec": { - "items": [ - { - "kind": "GridLayoutItem", - "spec": { - "element": { - "kind": "ElementReference", - "name": "panel-30" - }, - "height": 7, - "width": 8, - "x": 0, - "y": 0 - } - }, - { - "kind": "GridLayoutItem", - "spec": { - "element": { - "kind": "ElementReference", - "name": "panel-31" - }, - "height": 7, - "width": 8, - "x": 8, - "y": 0 - } - }, - { - "kind": "GridLayoutItem", - "spec": { - "element": { - "kind": "ElementReference", - "name": "panel-32" - }, - "height": 7, - "width": 8, - "x": 16, - "y": 0 - } - } - ] - } - }, - "title": "Business" - } - }, - { - "kind": "RowsLayoutRow", - "spec": { - "collapse": false, - "layout": { - "kind": "GridLayout", - "spec": { - "items": [ - { - "kind": "GridLayoutItem", - "spec": { - "element": { - "kind": "ElementReference", - "name": "panel-91" - }, - "height": 8, - "width": 24, - "x": 0, - "y": 0 - } - } - ] - } - }, - "title": "Logs" - } - } - ] - } - }, - "links": [], - "liveNow": false, - "preload": false, - "tags": [ - "airi", - "observability", - "grafana-cloud" - ], - "timeSettings": { - "autoRefresh": "30s", - "autoRefreshIntervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "fiscalYearStartMonth": 0, - "from": "now-6h", - "hideTimepicker": false, - "timezone": "browser", - "to": "now" - }, - "title": "AIRI Server Overview", - "variables": [ - { - "kind": "QueryVariable", - "spec": { - "allowCustomValue": true, - "current": { - "text": "All", - "value": "$__all" - }, - "definition": "label_values(target_info, deployment_environment)", - "hide": "dontHide", - "includeAll": true, - "multi": false, - "name": "env", - "options": [], - "query": { - "datasource": { - "name": "grafanacloud-projairi-prom" - }, - "group": "prometheus", - "kind": "DataQuery", - "spec": { - "__legacyStringValue": "label_values(target_info, deployment_environment)" - }, - "version": "v0" - }, - "refresh": "onDashboardLoad", - "regex": "", - "regexApplyTo": "value", - "skipUrlSync": false, - "sort": "disabled" - } - }, - { - "kind": "QueryVariable", - "spec": { - "allowCustomValue": true, - "current": { - "text": [ - "server" - ], - "value": [ - "server" - ] - }, - "definition": "label_values(target_info{deployment_environment=~\"$env\"}, service_name)", - "hide": "dontHide", - "includeAll": true, - "multi": true, - "name": "service", - "options": [], - "query": { - "datasource": { - "name": "grafanacloud-projairi-prom" - }, - "group": "prometheus", - "kind": "DataQuery", - "spec": { - "__legacyStringValue": "label_values(target_info{deployment_environment=~\"$env\"}, service_name)" - }, - "version": "v0" - }, - "refresh": "onDashboardLoad", - "regex": "", - "regexApplyTo": "value", - "skipUrlSync": false, - "sort": "disabled" - } - } - ] -} diff --git a/apps/server/otel/grafana/dashboards/build.test.ts b/apps/server/otel/grafana/dashboards/build.test.ts deleted file mode 100644 index a20e798ad..000000000 --- a/apps/server/otel/grafana/dashboards/build.test.ts +++ /dev/null @@ -1,156 +0,0 @@ -import { describe, expect, it } from 'vitest' - -import { checkDashboardLayoutReferences, dashboard } from './build' - -/** - * Narrows unknown dashboard nodes into indexable records for assertions. - */ -function asRecord(value: unknown, label: string): Record { - if (!value || typeof value !== 'object') - throw new TypeError(`${label} is not an object`) - - return value as Record -} - -/** - * Reads a generated panel title from the dashboard object. - */ -function panelTitle(panelName: string): string { - const panel = asRecord(dashboard.elements[panelName], panelName) - const spec = asRecord(panel.spec, `${panelName}.spec`) - if (typeof spec.title !== 'string') - throw new TypeError(`${panelName}.spec.title is not a string`) - - return spec.title -} - -/** - * Collects PromQL expression strings from nested Grafana panel objects. - */ -function collectQueryExpressions(value: unknown, expressions: string[] = []): string[] { - if (!value || typeof value !== 'object') - return expressions - - const record = value as Record - if (typeof record.expr === 'string') - expressions.push(record.expr) - - for (const nestedValue of Object.values(record)) - collectQueryExpressions(nestedValue, expressions) - - return expressions -} - -describe('grafana dashboard builder', () => { - /** - * @example - * const result = checkDashboardLayoutReferences(dashboard) - * expect(result.orphanRefs).toEqual([]) - */ - it('keeps every generated panel connected to the row layout', () => { - const result = checkDashboardLayoutReferences(dashboard) - - expect(result.orphanRefs).toEqual([]) - expect(result.unusedElems).toEqual([]) - }) - - it('keeps the product analytics row focused on Prometheus-safe engagement signals', () => { - expect(panelTitle('panel-95')).toBe('Product Events (range)') - expect(panelTitle('panel-96')).toBe('Product Failure %') - expect(panelTitle('panel-97')).toBe('Top Product Actions (range)') - expect(panelTitle('panel-98')).toBe('Product Event Rate') - expect(panelTitle('panel-99')).toBe('DAU Trend') - }) - - /** - * @example - * const rendered = JSON.stringify(dashboard.elements['panel-101']) - * expect(rendered).not.toContain('voice_id') - */ - it('keeps high-cardinality voice fields out of Prometheus queries', () => { - const productPanelExpressions = collectQueryExpressions([ - dashboard.elements['panel-95'], - dashboard.elements['panel-96'], - dashboard.elements['panel-97'], - dashboard.elements['panel-98'], - dashboard.elements['panel-99'], - ]).join('\n') - - expect(productPanelExpressions).not.toContain('voice_id') - expect(productPanelExpressions).not.toContain('voice_pack_id') - expect(productPanelExpressions).not.toContain('user_id') - expect(productPanelExpressions).not.toContain('session_id') - expect(productPanelExpressions).not.toContain('request_id') - }) - - it('preserves histogram buckets until HTTP and LLM latency quantiles are calculated', () => { - // ROOT CAUSE: - // - // The latency panels previously summed cumulative `_bucket` rates after - // dropping `le`. Grafana then labelled that request-rate-derived value as - // seconds, making millisecond HTTP routes appear to take 20+ seconds. - // - // Quantiles must retain `le` while replicas are merged, then call - // histogram_quantile over the merged histogram. - const httpLatency = collectQueryExpressions(dashboard.elements['panel-20']).join('\n') - const llmLatency = collectQueryExpressions(dashboard.elements['panel-21']).join('\n') - - expect(httpLatency).toContain('histogram_quantile(0.95') - expect(httpLatency).toContain('sum by (le, http_route)') - expect(llmLatency.match(/histogram_quantile\(0\.95/g)).toHaveLength(2) - expect(llmLatency).toContain('gen_ai_client_first_token_duration_seconds_bucket') - expect(llmLatency).toContain('sum by (le)') - }) - - it('does not clamp rate denominators to one request per second', () => { - // ROOT CAUSE: - // - // `clamp_min(rate, 1)` changes the denominator whenever traffic is below - // 1 req/s, so low-volume provider and fallback failures are underreported. - for (const panelName of ['panel-4', 'panel-62', 'panel-68']) { - const expressions = collectQueryExpressions(dashboard.elements[panelName]).join('\n') - - expect(expressions).not.toContain('clamp_min') - } - }) - - it('uses visualizations and reductions that match each query shape', () => { - const statusPanel = JSON.stringify(dashboard.elements['panel-40']) - const modelMixPanel = JSON.stringify(dashboard.elements['panel-11']) - - expect(statusPanel).toContain('"group":"timeseries"') - expect(statusPanel).not.toContain('"group":"heatmap"') - expect(modelMixPanel).toContain('increase(') - expect(modelMixPanel).toContain('"instant":true') - expect(modelMixPanel).toContain('"calcs":["lastNotNull"]') - }) - - it('filters structured Loki severity without parsing plain-text bodies as JSON', () => { - const errorLogsPanel = JSON.stringify(dashboard.elements['panel-91']) - - expect(panelTitle('panel-91')).toBe('Warn / Error Logs') - expect(errorLogsPanel).toContain('detected_level=~\\"warn|error\\"') - expect(errorLogsPanel).not.toContain('| json') - }) - - it('keeps one DAU visualization and refreshes the operations dashboard', () => { - expect(dashboard.elements['panel-80']).toBeUndefined() - expect(panelTitle('panel-99')).toBe('DAU Trend') - expect(dashboard.timeSettings.autoRefresh).toBe('30s') - }) - - it('queries cluster-wide distinct online websocket users as an instant value', () => { - // ROOT CAUSE: - // - // Counting WebSocket contexts measures tabs/connections, not people. The - // online-user gauge counts unique Redis broadcast channels cluster-wide, - // and every replica reports that same shared value. - const wsOnlinePanel = JSON.stringify(dashboard.elements['panel-93']) - - expect(panelTitle('panel-93')).toBe('Online Users') - expect(wsOnlinePanel).toContain('max(ws_users_online') - expect(wsOnlinePanel).not.toContain('ws_connections_active') - expect(wsOnlinePanel).toContain('"instant":true') - expect(wsOnlinePanel).toContain('"range":false') - }) -}) diff --git a/apps/server/otel/grafana/dashboards/build.ts b/apps/server/otel/grafana/dashboards/build.ts deleted file mode 100644 index ec89a68f6..000000000 --- a/apps/server/otel/grafana/dashboards/build.ts +++ /dev/null @@ -1,1117 +0,0 @@ -/** - * Dashboard generator for `airi-server-overview-cloud.json`. - * - * Run: `pnpm -F @proj-airi/server otel:dashboards` - * (or directly: `pnpm exec tsx apps/server/otel/grafana/dashboards/build.ts`) - * - * Why a generator instead of hand-edited JSON: the dashboard's Grafana v2 - * schema is verbose (~50 lines per panel). Rebuilding the file by hand every - * time we add a row guarantees drift between query expressions and the - * panel layout. A small DSL keeps each panel to one or two screen lines and - * cross-references panel ids → grid positions in one place. - * - * Scope: ONE core panel per metric. We intentionally do NOT keep the same - * metric in stat + trend + bar + pie forms — each metric gets the single - * visualisation that answers its question best (gauge for bounded ratios, - * bar gauge for top-N rankings, timeseries for trends, stat for range totals). - * - * Visual language: - * - stat — absolute counts / range totals - * - gauge — bounded ratios (%) where thresholds tell a story (5xx %, fallback %) - * - bargauge — top-N leaderboards (which route is hottest / slowest) - * - timeseries — trends over time, with rich legend calcs - * - * Counter queries follow strict semantics: - * - rate() for "right now" trends - * - increase($__range) for "total over visible window" - * - never raw sum() on a cumulative counter (resets on deploy distort it) - */ - -import { writeFileSync } from 'node:fs' -import { dirname, join } from 'node:path' -import { argv, exit } from 'node:process' -import { fileURLToPath, pathToFileURL } from 'node:url' - -const PROM = { name: 'grafanacloud-projairi-prom' } -const LOKI = { name: 'grafanacloud-projairi-logs' } -const SCHEMA_VERSION = '13.2.0-28666480772' - -// Service / env filter applied to every Prom query. Pulled into a helper so -// the variable name only appears once. -const SERVICE_FILTER = 'service_name=~"$service", deployment_environment=~"$env"' -const PRODUCT_EVENT_FILTER = `${SERVICE_FILTER}, feature!="", action!=""` - -// Build-script local types. Kept loose — Grafana owns the schema, and we -// validate the rendered JSON by re-importing it into Grafana, not by typing. -type DataSource = typeof PROM | typeof LOKI -interface ThresholdStep { color: string, value: number } -type PanelQuery = ReturnType -type LegendCalc = 'lastNotNull' | 'max' | 'min' | 'mean' | 'sum' - -interface QueryOpts { - instant?: boolean -} - -function query(expr: string, legend: string, refId = 'A', datasource: DataSource = PROM, opts: QueryOpts = {}) { - return { - kind: 'PanelQuery', - spec: { - hidden: false, - query: { - datasource, - group: datasource === LOKI ? 'loki' : 'prometheus', - kind: 'DataQuery', - spec: { - editorMode: 'code', - expr, - legendFormat: legend, - ...(opts.instant ? { instant: true, range: false } : { range: true }), - }, - version: 'v0', - }, - refId, - }, - } -} - -function thresholds(steps: ThresholdStep[]) { - return { mode: 'absolute', steps } -} - -interface DefaultsBlockOpts { - unit: string - steps: ThresholdStep[] - decimals?: number - noValue?: string - min?: number - max?: number -} - -interface StatPanelOpts { - unit?: string - steps?: ThresholdStep[] - decimals?: number - noValue?: string - graphMode?: 'area' | 'none' - /** - * Stat visual language: - * - 'health' (default) — traffic-light colour driven by `steps`, no trend - * delta. For numbers that are good or bad (req/s, 5xx, unbilled flux). - * - 'count' — neutral fixed colour + period-over-period % delta. For pure - * informational counts/totals with no good/bad threshold (active users, - * DAU/WAU, revenue, tokens consumed). - */ - variant?: 'health' | 'count' - /** Fixed colour for the 'count' variant. Ignored by 'health'. @default 'blue' */ - color?: string -} - -interface GaugePanelOpts { - unit?: string - steps: ThresholdStep[] - decimals?: number - min?: number - max?: number - noValue?: string -} - -interface BarGaugePanelOpts { - unit?: string - steps?: ThresholdStep[] - decimals?: number - min?: number - max?: number - noValue?: string -} - -interface TimeseriesPanelOpts { - unit?: string - stack?: boolean - fillOpacity?: number - legendCalcs?: LegendCalc[] - legendPlacement?: 'bottom' | 'right' - legendDisplayMode?: 'list' | 'table' -} - -// `noValue` shows a friendly placeholder instead of "No data" red text when -// the env genuinely has zero traffic (e.g. dev, fresh deploy). Empty-string -// fields are omitted from the JSON to keep diffs tidy. -function defaultsBlock({ unit, steps, decimals, noValue, min, max }: DefaultsBlockOpts) { - return { - color: { mode: 'thresholds' }, - thresholds: thresholds(steps), - unit, - ...(decimals != null && { decimals }), - ...(noValue != null && { noValue }), - ...(min != null && { min }), - ...(max != null && { max }), - } -} - -function statPanel(id: number, title: string, description: string, queries: PanelQuery[], opts: StatPanelOpts = {}) { - const { unit = 'short', steps = [{ color: 'green', value: 0 }], decimals, noValue, graphMode = 'area', variant = 'health', color = 'blue' } = opts - const isCount = variant === 'count' - - // 'count' stats drop the traffic-light colouring (the value is neither good - // nor bad) and instead surface a period-over-period % delta so the trend is - // readable at a glance. 'health' keeps threshold colouring and no delta. - const defaults = isCount - ? { - color: { mode: 'fixed', fixedColor: color }, - fieldMinMax: false, - thresholds: thresholds([{ color, value: 0 }]), - unit, - ...(decimals != null && { decimals }), - ...(noValue != null && { noValue }), - } - : defaultsBlock({ unit, steps, decimals, noValue }) - - return { - kind: 'Panel', - spec: { - data: { kind: 'QueryGroup', spec: { queries, queryOptions: {}, transformations: [] } }, - description, - id, - links: [], - title, - vizConfig: { - group: 'stat', - kind: 'VizConfig', - spec: { - fieldConfig: { defaults, overrides: [] }, - options: { - colorMode: isCount ? 'none' : 'value', - graphMode, - justifyMode: 'auto', - orientation: 'auto', - percentChangeColorMode: 'standard', - reduceOptions: { calcs: ['lastNotNull'], fields: '', values: false }, - showPercentChange: isCount, - textMode: isCount ? 'value_and_name' : 'auto', - wideLayout: true, - }, - }, - version: SCHEMA_VERSION, - }, - }, - } -} - -// Bounded ratio with traffic-light thresholds. Use for percent or capacity -// metrics; the radial fill instantly conveys "OK / warn / critical" without -// reading the number. -function gaugePanel(id: number, title: string, description: string, queries: PanelQuery[], opts: GaugePanelOpts) { - const { unit = 'percent', steps, decimals = 1, min = 0, max = 100, noValue } = opts - return { - kind: 'Panel', - spec: { - data: { kind: 'QueryGroup', spec: { queries, queryOptions: {}, transformations: [] } }, - description, - id, - links: [], - title, - vizConfig: { - group: 'gauge', - kind: 'VizConfig', - spec: { - fieldConfig: { defaults: defaultsBlock({ unit, steps, decimals, min, max, noValue }), overrides: [] }, - options: { - minVizHeight: 75, - minVizWidth: 75, - orientation: 'auto', - reduceOptions: { calcs: ['lastNotNull'], fields: '', values: false }, - showThresholdLabels: false, - showThresholdMarkers: true, - sizing: 'auto', - }, - }, - version: SCHEMA_VERSION, - }, - }, - } -} - -// Horizontal bar gauge for top-N leaderboards. Each series (one route) becomes -// one bar; bar length encodes the value and threshold colours flag severity. -// Use over a table when the question is "rank these and show relative -// magnitude" — it reads at a glance without scanning rows or a dead Time -// column. Feed it an INSTANT query (one point per series) so every route -// reduces to a single current value. -function barGaugePanel(id: number, title: string, description: string, queries: PanelQuery[], opts: BarGaugePanelOpts = {}) { - const { unit = 'short', steps = [{ color: 'green', value: 0 }], decimals, min, max, noValue } = opts - return { - kind: 'Panel', - spec: { - data: { kind: 'QueryGroup', spec: { queries, queryOptions: {}, transformations: [] } }, - description, - id, - links: [], - title, - vizConfig: { - group: 'bargauge', - kind: 'VizConfig', - spec: { - fieldConfig: { defaults: defaultsBlock({ unit, steps, decimals, min, max, noValue }), overrides: [] }, - options: { - displayMode: 'gradient', - maxVizHeight: 300, - minVizHeight: 12, - minVizWidth: 8, - namePlacement: 'auto', - orientation: 'horizontal', - reduceOptions: { calcs: ['lastNotNull'], fields: '', values: false }, - showUnfilled: true, - sizing: 'auto', - valueMode: 'color', - }, - }, - version: SCHEMA_VERSION, - }, - }, - } -} - -function timeseriesPanel(id: number, title: string, description: string, queries: PanelQuery[], opts: TimeseriesPanelOpts = {}) { - const { - unit = 'short', - stack = false, - fillOpacity = 20, - legendCalcs = ['lastNotNull', 'max'], - legendPlacement = 'right', - legendDisplayMode = 'table', - } = opts - return { - kind: 'Panel', - spec: { - data: { kind: 'QueryGroup', spec: { queries, queryOptions: {}, transformations: [] } }, - description, - id, - links: [], - title, - vizConfig: { - group: 'timeseries', - kind: 'VizConfig', - spec: { - fieldConfig: { - defaults: { - color: { mode: 'palette-classic' }, - custom: { - axisBorderShow: false, - axisCenteredZero: false, - axisColorMode: 'text', - axisLabel: '', - axisPlacement: 'auto', - barAlignment: 0, - barWidthFactor: 0.6, - drawStyle: 'line', - fillOpacity, - gradientMode: 'none', - hideFrom: { legend: false, tooltip: false, viz: false }, - insertNulls: false, - lineInterpolation: 'smooth', - lineWidth: 1, - pointSize: 5, - scaleDistribution: { type: 'linear' }, - showPoints: 'auto', - showValues: false, - spanNulls: false, - stacking: { group: 'A', mode: stack ? 'normal' : 'none' }, - thresholdsStyle: { mode: 'off' }, - }, - thresholds: thresholds([{ color: 'green', value: 0 }]), - unit, - }, - overrides: [], - }, - options: { - annotations: { clustering: -1, multiLane: false }, - // Show last + max in the legend table so viewers don't have to - // click each line to see numbers — same trick as Keycloak's - // "Login Errors" panel. - legend: { - calcs: legendCalcs, - displayMode: legendDisplayMode, - enableFacetedFilter: false, - overflow: 'ellipsis', - placement: legendPlacement, - showLegend: true, - }, - tooltip: { hideZeros: false, mode: 'multi', sort: 'desc' }, - }, - }, - version: SCHEMA_VERSION, - }, - }, - } -} - -function pieChartPanel(id: number, title: string, description: string, queries: PanelQuery[], unit = 'short') { - return { - kind: 'Panel', - spec: { - data: { kind: 'QueryGroup', spec: { queries, queryOptions: {}, transformations: [] } }, - description, - id, - links: [], - title, - vizConfig: { - group: 'piechart', - kind: 'VizConfig', - spec: { - fieldConfig: { - defaults: { - color: { fixedColor: '#73BF69', mode: 'palette-classic' }, - custom: { hideFrom: { legend: false, tooltip: false, viz: false } }, - unit, - }, - overrides: [], - }, - options: { - displayLabels: ['percent'], - legend: { displayMode: 'table', overflow: 'ellipsis', placement: 'bottom', showLegend: true }, - pieType: 'pie', - reduceOptions: { calcs: ['lastNotNull'], fields: '', values: false }, - sort: 'desc', - tooltip: { hideZeros: false, mode: 'single', sort: 'none' }, - }, - }, - version: SCHEMA_VERSION, - }, - }, - } -} - -// Keep the rolling 24-hour series as a trend panel; the point-in-time DAU stat -// would duplicate the same metric at the right edge of this chart. -function dailyActiveUsersTrendPanel() { - return timeseriesPanel( - 99, - 'DAU Trend', - 'Rolling 24-hour distinct active users over time. This is the trend view of `user.active_rolling`; the duplicate point-in-time DAU stat is intentionally omitted from this dashboard.', - [query(`max(user_active_rolling{${SERVICE_FILTER}, window="24h"})`, 'DAU')], - { unit: 'short', fillOpacity: 17, legendPlacement: 'bottom', legendCalcs: ['lastNotNull', 'max'] }, - ) -} - -function logsPanel(id: number, title: string, description: string, expr: string) { - return { - kind: 'Panel', - spec: { - data: { kind: 'QueryGroup', spec: { queries: [query(expr, '', 'A', LOKI)], queryOptions: {}, transformations: [] } }, - description, - id, - links: [], - title, - vizConfig: { - group: 'logs', - kind: 'VizConfig', - spec: { - fieldConfig: { defaults: {}, overrides: [] }, - options: { - dedupStrategy: 'none', - enableInfiniteScrolling: false, - enableLogDetails: true, - prettifyLogMessage: false, - showCommonLabels: false, - showControls: false, - showFieldSelector: false, - showLabels: true, - showLevel: true, - showLogAttributes: true, - showTime: true, - sortOrder: 'Descending', - timestampResolution: 'ms', - unwrappedColumns: false, - wrapLogMessage: true, - }, - }, - version: SCHEMA_VERSION, - }, - }, - } -} - -function item(name: string, x: number, y: number, width: number, height: number) { - return { kind: 'GridLayoutItem', spec: { element: { kind: 'ElementReference', name }, height, width, x, y } } -} - -function row(title: string, items: ReturnType[], { collapse = false }: { collapse?: boolean } = {}) { - return { - kind: 'RowsLayoutRow', - spec: { - collapse, - layout: { kind: 'GridLayout', spec: { items } }, - title, - }, - } -} - -// --------------------------------------------------------------------------- -// Panels -// --------------------------------------------------------------------------- - -// Grafana v2 element entries are opaque to us — each helper returns a Panel -// shape with deeply-nested fieldConfig/options that we don't statically type -// (Grafana owns that schema, and any drift would surface at dashboard import -// time, not compile time). Treat `elements` as a string-keyed bag of -// `unknown`-shaped panel JSON; the cross-check below catches mismatches -// between defined panel ids and layout references. -const elements: Record = {} - -// --- Row 1: Service Health — "is anything broken right now?" --------------- -// All ratios use a fixed [5m] window and DO NOT follow the time picker: this -// row is an on-call glance, the numbers should be stable regardless of which -// range the viewer picked. Trends live in their own rows below. -elements['panel-1'] = statPanel( - 1, - 'Total Users', - 'Current Better Auth user table size from `user.total` (cluster-wide DB gauge, aggregate with `max()`) plus rolling 24h signup delta from `increase(user.registered)`. Use the delta as today/new-user growth, and DAU / WAU / MAU below for returning-user engagement.', - [ - query(`max(user_total{${SERVICE_FILTER}})`, 'total users', 'A'), - query(`sum(increase(user_registered_total{${SERVICE_FILTER}}[24h]))`, 'new today', 'B'), - ], - { unit: 'short', variant: 'count' }, -) - -elements['panel-15'] = statPanel( - 15, - 'Active Sessions', - 'COUNT(*) over the Better Auth `session` table where `expires_at > now()`, aggregated with `avg()` (cluster-wide gauge). Counts session **rows**, not users — compare against DAU to spot session-row inflation.', - [query(`avg(user_active_sessions{${SERVICE_FILTER}})`, 'sessions')], - { unit: 'short', variant: 'count' }, -) - -elements['panel-3'] = statPanel( - 3, - 'Req/s (5m)', - '5-minute average inbound HTTP request rate. /livez and /readyz (K8s probes) are excluded at the @hono/otel middleware level so this reflects real user traffic.', - [query(`sum(rate(http_server_request_duration_seconds_count{${SERVICE_FILTER}, http_request_method!="OPTIONS"}[5m]))`, 'req/s')], - { unit: 'reqps', steps: [{ color: 'green', value: 0 }, { color: 'yellow', value: 100 }, { color: 'red', value: 500 }], decimals: 2 }, -) - -elements['panel-4'] = gaugePanel( - 4, - '5xx Rate %', - '5xx responses ÷ all responses over the last 5m. Fixed 5m window for an on-call glance ("is the service failing right now"). >1% warns, >5% pages.', - [query( - `100 * sum(rate(http_server_request_duration_seconds_count{${SERVICE_FILTER}, http_request_method!="OPTIONS", http_response_status_code=~"5.."}[5m])) / sum(rate(http_server_request_duration_seconds_count{${SERVICE_FILTER}, http_request_method!="OPTIONS"}[5m]))`, - 'fail %', - )], - { steps: [{ color: 'green', value: 0 }, { color: 'yellow', value: 1 }, { color: 'red', value: 5 }], max: 10, decimals: 2, noValue: '0' }, -) - -elements['panel-5'] = statPanel( - 5, - 'LLM Req/s (5m)', - '5-minute average LLM gateway request rate (chat + tts). For per-model trends see the LLM Gateway row.', - [query(`sum(rate(gen_ai_client_operation_count_total{${SERVICE_FILTER}}[5m]))`, 'req/s')], - { unit: 'reqps', decimals: 2 }, -) - -// --- Users & Engagement: DAU/WAU/MAU + sessions + live WebSocket presence --- -// DAU/WAU/MAU come from the `user.active_rolling` gauge (COUNT(*) over `user` -// filtered by last_seen_at; one series per window). Cluster-wide gauge — every -// replica reports the same value, so aggregate with max(), NOT sum(). -const ROLLING_USERS = [ - { id: 81, window: '7d', title: 'WAU', label: 'Weekly', span: 'last 7d' }, - { id: 82, window: '30d', title: 'MAU', label: 'Monthly', span: 'last 30d' }, -] as const -for (const { id, window, title, label, span } of ROLLING_USERS) { - elements[`panel-${id}`] = statPanel( - id, - title, - `${label} active users — distinct users with activity in the ${span}. Sourced from \`user.last_seen_at\` (touched on sign-in and every OIDC token refresh) via the \`user.active_rolling\` gauge. Cluster-wide gauge aggregated with \`max()\`.`, - [query(`max(user_active_rolling{${SERVICE_FILTER}, window="${window}"})`, title)], - { unit: 'short', variant: 'count', noValue: '0' }, - ) -} - -elements['panel-93'] = statPanel( - 93, - 'Online Users', - 'Cluster-wide distinct authenticated users with at least one active `/ws/chat` connection. Redis returns each per-user broadcast channel once even when that user has multiple tabs or connections across server replicas; every replica reports the same global value, so the query uses `max()`.', - [query(`max(ws_users_online{${SERVICE_FILTER}})`, 'users', 'A', PROM, { instant: true })], - { unit: 'short', variant: 'count', color: 'purple', noValue: '—' }, -) - -elements['panel-92'] = timeseriesPanel( - 92, - 'WS Connections', - 'Concurrent WebSocket connections over time (`sum` across replicas). A cliff to zero with no matching deploy = mass disconnect (LB drop, network blackhole); a slow ramp without disconnects = connection leak.', - [query(`sum(ws_connections_active{${SERVICE_FILTER}})`, 'connections')], - { unit: 'short', fillOpacity: 30 }, -) - -// --- Product Analytics — event volume + server-side TTS health ------------- -// Prometheus deliberately does not carry user_id. These panels answer -// "which product actions are happening and failing"; DB-side product_events -// queries answer "how many distinct users used each feature". -elements['panel-95'] = statPanel( - 95, - 'Product Events (range)', - 'Total first-party product analytics events over the dashboard range. This is event volume, not distinct users — distinct-user counts come from the Postgres `product_events` table.', - [ - query(`sum(increase(airi_product_events_total{${PRODUCT_EVENT_FILTER}}[$__range]))`, 'events'), - ], - { unit: 'short', variant: 'count', noValue: '0', graphMode: 'none' }, -) - -elements['panel-96'] = gaugePanel( - 96, - 'Product Failure %', - 'Failed product events ÷ all product events over the dashboard range. Uses only bounded labels (`feature`, `action`, `status`, `source`); no user/session/request identifiers are present in Prometheus.', - [query( - `100 * sum(increase(airi_product_events_total{${PRODUCT_EVENT_FILTER}, status="failed"}[$__range])) / clamp_min(sum(increase(airi_product_events_total{${PRODUCT_EVENT_FILTER}}[$__range])), 1)`, - 'failed %', - )], - { steps: [{ color: 'green', value: 0 }, { color: 'yellow', value: 2 }, { color: 'red', value: 10 }], max: 20, decimals: 2, noValue: '0' }, -) - -elements['panel-97'] = barGaugePanel( - 97, - 'Top Product Actions (range)', - 'Top product actions by event count over the dashboard range. Use this to see which features are actually being exercised after deployment; pair with DB `count(distinct user_id)` for user counts.', - [query( - `topk(12, sum by (feature, action, status) (increase(airi_product_events_total{${PRODUCT_EVENT_FILTER}}[$__range])))`, - '{{feature}} · {{action}} · {{status}}', - 'A', - PROM, - { instant: true }, - )], - { unit: 'short', noValue: '0' }, -) - -elements['panel-98'] = timeseriesPanel( - 98, - 'Product Event Rate', - 'Product event rate by feature/action/status. This is the Prometheus-safe trend view; user-level analysis remains in Postgres `product_events`.', - [query( - `sum by (feature, action, status) (rate(airi_product_events_total{${PRODUCT_EVENT_FILTER}}[$__rate_interval]))`, - '{{feature}} · {{action}} · {{status}}', - )], - { unit: 'eps', fillOpacity: 15 }, -) - -elements['panel-99'] = dailyActiveUsersTrendPanel() - -// --- Row 2: HTTP — traffic ranking, error trend, latency trend ------------- -elements['panel-16'] = barGaugePanel( - 16, - 'Top Routes by Requests (range)', - 'Top Hono-matched routes by request count over the dashboard range. The main traffic list: which API surfaces are hottest. Wildcard patterns like `/api/v1/openai/*` are requests that did not reach a concrete handler (404 / auth-rejected); concrete paths are successful routes.', - [query( - `topk(50, sum by (http_route) (increase(http_server_request_duration_seconds_count{${SERVICE_FILTER}, http_request_method!="OPTIONS", http_route!=""}[$__range])))`, - '{{http_route}}', - 'A', - PROM, - { instant: true }, - )], - { unit: 'short' }, -) - -elements['panel-40'] = timeseriesPanel( - 40, - 'HTTP Status Rate', - 'Stacked non-OPTIONS request rate by HTTP status code. A new or growing 4xx / 5xx band flags a traffic-quality or service-health change.', - [query( - `sum by (http_response_status_code) (rate(http_server_request_duration_seconds_count{${SERVICE_FILTER}, http_request_method!="OPTIONS"}[$__rate_interval]))`, - '{{http_response_status_code}}', - )], - { unit: 'reqps', stack: true, fillOpacity: 60, legendPlacement: 'bottom' }, -) - -elements['panel-20'] = timeseriesPanel( - 20, - 'Request Latency P95 by Route', - 'P95 Hono request duration by matched route. Histogram buckets are merged across replicas while preserving `le`, then interpolated by `histogram_quantile`; values are estimates bounded by the configured bucket widths.', - [query( - `histogram_quantile(0.95, sum by (le, http_route) ( - rate(http_server_request_duration_seconds_bucket{${SERVICE_FILTER}, http_request_method!="OPTIONS", http_route!~"/api/v1/openai/.*", http_response_status_code!="404"}[$__rate_interval]) -))`, - '{{http_route}}', - )], - { unit: 's', legendPlacement: 'bottom' }, -) - -elements['panel-94'] = timeseriesPanel( - 94, - 'Errors by Route', - 'Error responses per route, broken out by status code. Excludes success (2xx/3xx) and the expected-client-error codes 401/402/404 (auth-required / payment-required / not-found noise) so the curve isolates real failures: 4xx like 400/403/422/429 and all 5xx. The per-route companion to the aggregate Error Rate % stat.', - [query( - `sum by (http_route, http_response_status_code) (increase(http_server_request_duration_seconds_count{${SERVICE_FILTER}, http_request_method!="OPTIONS", http_response_status_code!~"2..|3..|401|402|404"}[$__rate_interval]))`, - '{{http_response_status_code}} {{http_route}}', - )], - { unit: 'short' }, -) - -// --- Row 3: LLM Gateway — request mix + latency ---------------------------- -elements['panel-11'] = pieChartPanel( - 11, - 'LLM Requests by Model (range)', - 'Per-model request count over the visible dashboard range (chat + tts). The pie shows each model\'s share without depending on Grafana sampling resolution.', - [query( - `sum by (gen_ai_request_model) (increase(gen_ai_client_operation_count_total{${SERVICE_FILTER}, gen_ai_request_model!=""}[$__range]))`, - '{{gen_ai_request_model}}', - 'A', - PROM, - { instant: true }, - )], - 'short', -) - -elements['panel-21'] = timeseriesPanel( - 21, - 'LLM Latency P95', - 'Two P95 latency signals for the LLM gateway, aggregated across models. TTFB = time to first streamed token (streaming chat UX). End-to-end = full operation duration — the only latency signal for non-streaming chat and TTS, which have no first-token event.', - [ - query(`histogram_quantile(0.95, sum by (le) (rate(gen_ai_client_first_token_duration_seconds_bucket{${SERVICE_FILTER}}[$__rate_interval])))`, 'TTFB p95', 'A'), - query(`histogram_quantile(0.95, sum by (le) (rate(gen_ai_client_operation_duration_seconds_bucket{${SERVICE_FILTER}}[$__rate_interval])))`, 'end-to-end p95', 'B'), - ], - { unit: 's' }, -) - -// --- Row: Provider Upstreams — our gateway's view of each upstream so the -// per-provider consoles (OpenRouter / Volcengine 豆包 / DashScope 阿里) don't -// have to be checked one by one. `provider` is the upstream the router -// actually used (winning upstream on success, last-tried on exhaustion); -// it's the URL hostname, so legends read e.g. `openrouter.ai`, -// `dashscope.aliyuncs.com`. Note: provider-only truths (real $ spend, account -// quota / balance) are NOT here — those need the provider billing APIs. -elements['panel-66'] = timeseriesPanel( - 66, - 'Requests/s by Provider', - 'Outbound request rate to each upstream provider (chat + tts), as our gateway sees it. The RPM / 调用次数 screens on the provider consoles, unified. provider = upstream hostname the router used.', - [query( - `sum by (provider) (rate(gen_ai_client_operation_count_total{${SERVICE_FILTER}, provider!=""}[$__rate_interval]))`, - '{{provider}}', - )], - { unit: 'reqps' }, -) - -elements['panel-67'] = timeseriesPanel( - 67, - 'Provider Latency P95', - 'P95 upstream call duration per provider (chat + tts), across models. Mirrors each provider console\'s 调用时长 p95/p99 panel — but here every provider is on one axis.', - [query( - `histogram_quantile(0.95, sum by (le, provider) (rate(gen_ai_client_operation_duration_seconds_bucket{${SERVICE_FILTER}, provider!=""}[$__rate_interval])))`, - '{{provider}}', - )], - { unit: 's' }, -) - -elements['panel-68'] = timeseriesPanel( - 68, - 'Provider Failure %', - '4xx + 5xx ÷ all requests per provider, our side of the call. Matches each provider 失败率 panel. Pair with Upstream Errors by Status Code (LLM Router Health) to see which codes drive it.', - [query( - `100 * sum by (provider) (rate(gen_ai_client_operation_count_total{${SERVICE_FILTER}, provider!="", http_response_status_code=~"4..|5.."}[$__rate_interval])) / sum by (provider) (rate(gen_ai_client_operation_count_total{${SERVICE_FILTER}, provider!=""}[$__rate_interval]))`, - '{{provider}}', - )], - { unit: 'percent' }, -) - -elements['panel-69'] = timeseriesPanel( - 69, - 'TTS Characters/s by Model', - 'Billed TTS characters per second by model (from `airi.billing.tts.chars`). The 用量统计「字数」screen on the TTS consoles (豆包 / 阿里), unified. Integrate over the range for a window total.', - [query( - `sum by (model) (rate(airi_billing_tts_chars_total{${SERVICE_FILTER}}[$__rate_interval]))`, - '{{model}}', - )], - { unit: 'short' }, -) - -// --- Row 4: LLM Tokens & Quality — usage totals + revenue-leak alerts ------ -elements['panel-73'] = statPanel( - 73, - 'Tokens Consumed (range)', - 'Total input and output tokens billed over the dashboard range, from the upstream `usage` block (requests where the upstream omits usage are not counted). The cumulative counterpart to panel-71 throughput — use for "how many tokens did we burn this window" cost math.', - [ - query(`sum(increase(gen_ai_client_token_usage_input_total{${SERVICE_FILTER}}[$__range]))`, 'input', 'A'), - query(`sum(increase(gen_ai_client_token_usage_output_total{${SERVICE_FILTER}}[$__range]))`, 'output', 'B'), - ], - { unit: 'short', variant: 'count', noValue: '0', graphMode: 'none' }, -) - -elements['panel-71'] = timeseriesPanel( - 71, - 'LLM Token Throughput', - 'Input vs output token throughput across the LLM gateway (tokens/sec). Recorded per request from the upstream `usage` block. Use for capacity planning and cost estimation. input = prompt tokens consumed; output = completion tokens generated.', - [ - query(`sum(rate(gen_ai_client_token_usage_input_total{${SERVICE_FILTER}}[$__rate_interval]))`, 'input tokens/s', 'A'), - query(`sum(rate(gen_ai_client_token_usage_output_total{${SERVICE_FILTER}}[$__rate_interval]))`, 'output tokens/s', 'B'), - ], - { unit: 'short' }, -) - -elements['panel-43'] = statPanel( - 43, - '⚠ Flux Unbilled (range)', - 'Flux owed by users but never debited for unexpected reasons (excludes `partial_debit_drained`, a known partial-balance drain path). Real revenue leak — DB latency and HTTP 5xx alerts do NOT cover this, because the response was 2xx and the catch path is silent.', - [query( - `sum(increase(airi_billing_flux_unbilled_total{${SERVICE_FILTER}, reason!="partial_debit_drained"}[$__range]))`, - 'flux', - )], - { unit: 'short', steps: [{ color: 'green', value: 0 }, { color: 'red', value: 1 }], noValue: '0', graphMode: 'none' }, -) - -elements['panel-41'] = statPanel( - 41, - 'Stream Interruptions (range)', - 'LLM streams that died mid-flight over the dashboard range. before_first_chunk = upstream blew up; mid_stream = partial delivery (user saw a broken response).', - [query( - `sum(increase(airi_gen_ai_stream_interrupted_total{${SERVICE_FILTER}}[$__range]))`, - 'interruptions', - )], - { unit: 'short', steps: [{ color: 'green', value: 0 }, { color: 'yellow', value: 1 }, { color: 'red', value: 10 }], noValue: '0', graphMode: 'none' }, -) - -// --- Row 5: LLM Router Health — "wake someone up" gateway indicators ------- -// Counters from `apps/server/src/services/llm-router/router.ts`, emitted for -// every chat AND tts dispatch attempt. Prom names (OTel dot → underscore, -// `_total` for counters): airi_gen_ai_gateway_{key_exhausted,decrypt_failures, -// fallback_count,upstream_errors}_total. -elements['panel-60'] = statPanel( - 60, - 'Key Exhausted (5m)', - 'Number of (model, upstream) pairs that ran out of usable keys within one user request over the last 5 minutes. Sustained > 0 = a provider account is dead or every stored ciphertext is failing to decrypt — page on-call.', - [query(`sum(increase(airi_gen_ai_gateway_key_exhausted_total{${SERVICE_FILTER}}[5m]))`, 'events')], - { unit: 'short', steps: [{ color: 'green', value: 0 }, { color: 'red', value: 1 }], noValue: '0', graphMode: 'none' }, -) - -elements['panel-61'] = statPanel( - 61, - 'Decrypt Failures (5m)', - 'Envelope-crypto decrypt failures in the key rotator. Non-zero is security-relevant: either the master key was rotated without re-wrapping ciphertexts, or someone forged a config blob.', - [query(`sum(increase(airi_gen_ai_gateway_decrypt_failures_total{${SERVICE_FILTER}}[5m]))`, 'events')], - { unit: 'short', steps: [{ color: 'green', value: 0 }, { color: 'red', value: 1 }], noValue: '0', graphMode: 'none' }, -) - -elements['panel-62'] = gaugePanel( - 62, - 'Fallback Ratio % (5m)', - 'Fallback attempts ÷ total LLM operations over the last 5m. Sustained > 30% means one provider is degraded and the router is silently masking it for users while burning quota on the failing upstream.', - [query( - `100 * sum(rate(airi_gen_ai_gateway_fallback_count_total{${SERVICE_FILTER}}[5m])) / sum(rate(gen_ai_client_operation_count_total{${SERVICE_FILTER}}[5m]))`, - 'fallback %', - )], - { steps: [{ color: 'green', value: 0 }, { color: 'yellow', value: 10 }, { color: 'red', value: 30 }], max: 100, decimals: 1, noValue: '0' }, -) - -elements['panel-65'] = timeseriesPanel( - 65, - 'Upstream Errors by Status Code', - 'Per-upstream non-2xx response rate split by status code. Only counts attempts where the upstream actually answered. 401/403 = bad key; 429 = quota; 5xx = upstream outage.', - [query( - `sum by (provider, status_code) (rate(airi_gen_ai_gateway_upstream_errors_total{${SERVICE_FILTER}}[$__rate_interval]))`, - '{{provider}} · {{status_code}}', - )], - { unit: 'ops' }, -) - -// --- Row 6: Business — money flow ------------------------------------------ -elements['panel-30'] = statPanel( - 30, - 'Revenue (range)', - 'Stripe revenue over dashboard range, in major currency unit (cents → dollars). Cross-currency sums are meaningless — always grouped by currency. Empty in dev / fresh deploys.', - [query( - `sum by (currency) (increase(airi_stripe_revenue_minor_unit_total{${SERVICE_FILTER}, currency!=""}[$__range])) / 100`, - '{{currency}}', - )], - { unit: 'short', variant: 'count', color: 'green', decimals: 2, noValue: '—' }, -) - -elements['panel-31'] = gaugePanel( - 31, - 'Checkout Conversion %', - 'Completed checkouts ÷ created checkouts over dashboard range. Drops can flag price-page bugs or payment-method outages.', - [query( - `100 * sum(increase(stripe_checkout_completed_total{${SERVICE_FILTER}}[$__range])) / clamp_min(sum(increase(stripe_checkout_created_total{${SERVICE_FILTER}}[$__range])), 1)`, - 'completed %', - )], - { steps: [{ color: 'red', value: 0 }, { color: 'yellow', value: 30 }, { color: 'green', value: 60 }], decimals: 1, noValue: '—' }, -) - -elements['panel-32'] = statPanel( - 32, - 'Stripe Events (range)', - 'Webhook events grouped by event.type. Pattern shifts (e.g. surge in invoice.payment_failed) indicate billing health.', - [query( - `sum by (event_type) (increase(stripe_events_total{${SERVICE_FILTER}, event_type!=""}[$__range]))`, - '{{event_type}}', - )], - { unit: 'short', variant: 'count', noValue: '—', graphMode: 'none' }, -) - -// --- Row 8: Logs ------------------------------------------------------------ -elements['panel-91'] = logsPanel( - 91, - 'Warn / Error Logs', - 'Server-side warn and error logs from Loki structured metadata. Derived fields make `trace_id` and `req` clickable — `trace_id` jumps to Tempo for full request playback.', - `{${SERVICE_FILTER}} | detected_level=~"warn|error"`, -) - -elements['panel-90'] = logsPanel( - 90, - 'Application Logs', - 'Live application logs from Loki. Filter via the panel UI; click trace_id to jump to Tempo.', - `{${SERVICE_FILTER}} |= \`\``, -) - -// --------------------------------------------------------------------------- -// Layout -// --------------------------------------------------------------------------- - -const rows = [ - // Row 1: Service Health — dense single-screen operations layout from the - // latest Grafana Cloud edit. Logs stay docked on the right while traffic, - // latency, LLM, token, and provider panels fill the left. - row('Service Health', [ - item('panel-3', 0, 0, 4, 5), - item('panel-5', 4, 0, 4, 5), - item('panel-40', 8, 0, 10, 6), - item('panel-90', 18, 0, 6, 38), - item('panel-93', 0, 5, 4, 5), - item('panel-4', 4, 5, 4, 5), - item('panel-20', 8, 6, 10, 15), - item('panel-92', 0, 10, 4, 10), - item('panel-73', 4, 10, 2, 10), - item('panel-11', 6, 10, 2, 10), - item('panel-16', 0, 20, 8, 18), - item('panel-71', 8, 21, 5, 5), - item('panel-21', 13, 21, 5, 5), - item('panel-69', 8, 26, 5, 6), - item('panel-67', 13, 26, 5, 6), - item('panel-66', 8, 32, 5, 6), - item('panel-68', 13, 32, 5, 6), - ]), - // Row 2: User Engagement — rolling-window active users and Prom-safe product - // analytics. DAU uses one trend panel; compact stats cover the longer - // WAU/MAU windows without duplicating the same 24-hour gauge. - row('User Engagement', [ - item('panel-1', 0, 0, 3, 5), - item('panel-15', 3, 0, 3, 5), - item('panel-99', 6, 0, 12, 11), - item('panel-98', 18, 0, 6, 11), - item('panel-81', 0, 5, 3, 3), - item('panel-82', 3, 5, 3, 3), - item('panel-95', 0, 11, 6, 9), - item('panel-96', 6, 11, 6, 9), - item('panel-97', 12, 11, 12, 9), - ]), - row('Product Analytics', []), - // Row 3: HTTP — full-width error breakdown; traffic ranking and latency moved - // into the dense Service Health screen above. - row('HTTP', [ - item('panel-94', 0, 0, 24, 8), - ]), - // Row 4: token totals + throughput + the two revenue/quality alert stats. - row('LLM Tokens & Quality', [ - item('panel-43', 0, 0, 6, 7), - item('panel-41', 6, 0, 6, 7), - ]), - // Row 5: router health — three "wake someone up" stats/gauge + upstream errors. - row('LLM Router Health', [ - item('panel-60', 0, 0, 6, 6), - item('panel-61', 6, 0, 6, 6), - item('panel-62', 12, 0, 6, 6), - item('panel-65', 18, 0, 6, 6), - ]), - // Row 6: business money flow. - row('Business', [ - item('panel-30', 0, 0, 8, 7), - item('panel-31', 8, 0, 8, 7), - item('panel-32', 16, 0, 8, 7), - ]), - // Row 8: focused error logs; the live application firehose is docked in - // Service Health for the latest cloud layout. - row('Logs', [ - item('panel-91', 0, 0, 24, 8), - ]), -] - -// --------------------------------------------------------------------------- -// Variables (use target_info — always present, owns service.name + deployment.environment labels) -// --------------------------------------------------------------------------- - -const variables = [ - { - kind: 'QueryVariable', - spec: { - allowCustomValue: true, - current: { text: 'All', value: '$__all' }, - definition: 'label_values(target_info, deployment_environment)', - hide: 'dontHide', - includeAll: true, - multi: false, - name: 'env', - options: [], - query: { - datasource: PROM, - group: 'prometheus', - kind: 'DataQuery', - spec: { __legacyStringValue: 'label_values(target_info, deployment_environment)' }, - version: 'v0', - }, - refresh: 'onDashboardLoad', - regex: '', - regexApplyTo: 'value', - skipUrlSync: false, - sort: 'disabled', - }, - }, - { - kind: 'QueryVariable', - spec: { - allowCustomValue: true, - current: { text: ['server'], value: ['server'] }, - definition: 'label_values(target_info{deployment_environment=~"$env"}, service_name)', - hide: 'dontHide', - includeAll: true, - multi: true, - name: 'service', - options: [], - query: { - datasource: PROM, - group: 'prometheus', - kind: 'DataQuery', - spec: { __legacyStringValue: 'label_values(target_info{deployment_environment=~"$env"}, service_name)' }, - version: 'v0', - }, - refresh: 'onDashboardLoad', - regex: '', - regexApplyTo: 'value', - skipUrlSync: false, - sort: 'disabled', - }, - }, -] - -// --------------------------------------------------------------------------- -// Top-level dashboard -// --------------------------------------------------------------------------- - -/** - * AIRI Server Overview dashboard. - * - * Reading order: - * 1. Service Health — dense operations screen with request, LLM, provider, - * token, WebSocket, status, and live application-log signals. - * 2. User Engagement — rolling DAU/WAU/MAU, total users, sessions, product - * event health, and the hand-tuned daily-active-user trend. - * 3. HTTP — full-width route error breakdown. - * 4. LLM Tokens & Quality — revenue-leak and stream-interruption alerts. - * 5. LLM Router Health — key/decrypt/fallback "wake someone up" signals. - * 6. Business — Stripe / Flux money flow. - * 7. Logs — Loki warning/error logs for live debugging. - * - * One metric, one panel: we deliberately do not duplicate a metric across - * stat/trend/bar/pie forms. Counter conventions: rate() for "now" trends, - * increase($__range) for "total over window", never raw sum() on a counter. - * - * Variables source from `target_info` (always present, no business-metric - * dependency) so the dashboard never goes blank when an app metric is renamed. - */ -export const dashboard = { - annotations: [ - { - kind: 'AnnotationQuery', - spec: { - builtIn: true, - enable: true, - hide: true, - iconColor: 'rgba(0, 211, 255, 1)', - name: 'Annotations & Alerts', - query: { - datasource: { name: '-- Grafana --' }, - group: 'grafana', - kind: 'DataQuery', - spec: {}, - version: 'v0', - }, - }, - }, - ], - cursorSync: 'Crosshair', - editable: true, - elements, - layout: { kind: 'RowsLayout', spec: { rows } }, - links: [], - liveNow: false, - preload: false, - tags: ['airi', 'observability', 'grafana-cloud'], - timeSettings: { - autoRefresh: '30s', - autoRefreshIntervals: ['5s', '10s', '30s', '1m', '5m', '15m', '30m', '1h', '2h', '1d'], - fiscalYearStartMonth: 0, - from: 'now-6h', - hideTimepicker: false, - timezone: 'browser', - to: 'now', - }, - title: 'AIRI Server Overview', - variables, -} - -export interface DashboardLayoutCheckResult { - orphanRefs: string[] - unusedElems: string[] -} - -/** - * Validates that every dashboard layout reference points to a defined element. - * - * Use when: - * - Regenerating the Grafana JSON from this dashboard builder. - * - Testing that row changes did not orphan panels or leave panels unused. - * - * Expects: - * - A Grafana dashboard object shaped like {@link dashboard}. - * - * Returns: - * - Orphan layout references and unused element names. - */ -export function checkDashboardLayoutReferences(targetDashboard: typeof dashboard): DashboardLayoutCheckResult { - const elementNames = new Set(Object.keys(targetDashboard.elements)) - const refs = new Set() - collectElementReferences(targetDashboard.layout, refs) - return { - orphanRefs: [...refs].filter(r => !elementNames.has(r)), - unusedElems: [...elementNames].filter(e => !refs.has(e)), - } -} - -/** - * Recursively collects Grafana row element references from the layout tree. - */ -function collectElementReferences(node: unknown, refs: Set): void { - if (!node || typeof node !== 'object') - return - const layoutNode = node as { kind?: unknown, name?: unknown } - if (layoutNode.kind === 'ElementReference' && typeof layoutNode.name === 'string') - refs.add(layoutNode.name) - for (const value of Object.values(node)) collectElementReferences(value, refs) -} - -/** - * Writes the generated dashboard JSON and fails the CLI on layout drift. - */ -function writeDashboard(): void { - const here = dirname(fileURLToPath(import.meta.url)) - const outPath = join(here, 'airi-server-overview-cloud.json') - writeFileSync(outPath, `${JSON.stringify(dashboard, null, 2)}\n`) - console.info(`wrote ${outPath}`) - - const { orphanRefs, unusedElems } = checkDashboardLayoutReferences(dashboard) - const definedCount = Object.keys(dashboard.elements).length - const referencedCount = definedCount - unusedElems.length + orphanRefs.length - console.info(`panels defined: ${definedCount}, referenced: ${referencedCount}, orphans: ${orphanRefs.length}, unused: ${unusedElems.length}`) - if (orphanRefs.length || unusedElems.length) { - console.error('orphans:', orphanRefs) - console.error('unused:', unusedElems) - exit(1) - } -} - -if (import.meta.url === pathToFileURL(argv[1] ?? '').href) - writeDashboard() diff --git a/apps/server/otel/grafana/provisioning/dashboards/dashboards.yaml b/apps/server/otel/grafana/provisioning/dashboards/dashboards.yaml deleted file mode 100644 index 5be851f16..000000000 --- a/apps/server/otel/grafana/provisioning/dashboards/dashboards.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: 1 - -providers: - - name: AIRI Dashboards - orgId: 1 - folder: AIRI - type: file - disableDeletion: false - editable: true - options: - path: /var/lib/grafana/dashboards - foldersFromFilesStructure: false diff --git a/apps/server/otel/grafana/provisioning/datasources/datasources.yaml b/apps/server/otel/grafana/provisioning/datasources/datasources.yaml deleted file mode 100644 index 2e4f7715d..000000000 --- a/apps/server/otel/grafana/provisioning/datasources/datasources.yaml +++ /dev/null @@ -1,45 +0,0 @@ -apiVersion: 1 - -datasources: - - name: Prometheus - type: prometheus - access: proxy - url: http://prometheus:9090 - isDefault: true - editable: true - jsonData: - httpMethod: POST - exemplarTraceIdDestinations: - - name: traceID - datasourceUid: tempo - - - name: Loki - type: loki - access: proxy - url: http://loki:3100 - editable: true - jsonData: - derivedFields: - - datasourceUid: tempo - matcherRegex: '"traceId":"(\w+)"' - name: TraceID - url: '$${__value.raw}' - - - name: Tempo - type: tempo - access: proxy - url: http://tempo:3200 - uid: tempo - editable: true - jsonData: - tracesToLogsV2: - datasourceUid: loki - filterByTraceID: true - tracesToMetrics: - datasourceUid: prometheus - serviceMap: - datasourceUid: prometheus - nodeGraph: - enabled: true - lokiSearch: - datasourceUid: loki diff --git a/apps/server/otel/loki/loki.yaml b/apps/server/otel/loki/loki.yaml deleted file mode 100644 index 7d1fedf54..000000000 --- a/apps/server/otel/loki/loki.yaml +++ /dev/null @@ -1,36 +0,0 @@ -auth_enabled: false - -server: - http_listen_port: 3100 - -common: - path_prefix: /loki - storage: - filesystem: - chunks_directory: /loki/chunks - rules_directory: /loki/rules - replication_factor: 1 - ring: - kvstore: - store: inmemory - -schema_config: - configs: - - from: '2024-01-01' - store: tsdb - object_store: filesystem - schema: v13 - index: - prefix: index_ - period: 24h - -limits_config: - allow_structured_metadata: true - volume_enabled: true - -query_range: - results_cache: - cache: - embedded_cache: - enabled: true - max_size_mb: 100 diff --git a/apps/server/otel/prometheus/prometheus.yaml b/apps/server/otel/prometheus/prometheus.yaml deleted file mode 100644 index a558479b5..000000000 --- a/apps/server/otel/prometheus/prometheus.yaml +++ /dev/null @@ -1,16 +0,0 @@ -global: - scrape_interval: 15s - evaluation_interval: 15s - -scrape_configs: - # Scrape OTEL Collector's Prometheus exporter - - job_name: otel-collector - static_configs: - - targets: ['otel-collector:8889'] - labels: - service: otel-collector - - # Scrape OTEL Collector's own metrics - - job_name: otel-collector-internal - static_configs: - - targets: ['otel-collector:8888'] diff --git a/apps/server/otel/tempo/tempo.yaml b/apps/server/otel/tempo/tempo.yaml deleted file mode 100644 index ae9520293..000000000 --- a/apps/server/otel/tempo/tempo.yaml +++ /dev/null @@ -1,47 +0,0 @@ -server: - http_listen_port: 3200 - -distributor: - receivers: - otlp: - protocols: - grpc: - endpoint: 0.0.0.0:4317 - http: - endpoint: 0.0.0.0:4318 - -storage: - trace: - backend: local - local: - path: /var/tempo/traces - wal: - path: /var/tempo/wal - -metrics_generator: - registry: - external_labels: - source: tempo - cluster: docker-compose - storage: - path: /var/tempo/generator/wal - remote_write: - - url: http://prometheus:9090/api/v1/write - send_exemplars: true - traces_storage: - path: /var/tempo/generator/traces - processor: - service_graphs: - dimensions: - - http.method - - http.target - span_metrics: - dimensions: - - http.method - - http.target - - http.status_code - -overrides: - defaults: - metrics_generator: - processors: [service-graphs, span-metrics]