From f31a37e7e4e0e69ad22a4ff966afbc331093d59a Mon Sep 17 00:00:00 2001 From: RainbowBird Date: Mon, 30 Mar 2026 20:10:50 +0800 Subject: [PATCH] refactor(server): prune otel dashboard and active http request --- .../dashboards/airi-engagement-cloud.json | 279 -- .../dashboards/airi-revenue-cloud.json | 319 --- .../airi-server-overview-cloud.json | 2268 +++++++++++------ .../dashboards/airi-server-overview.json | 503 ---- .../dashboards/airi-user-auth-cloud.json | 421 --- apps/server/src/middlewares/otel.ts | 14 +- 6 files changed, 1540 insertions(+), 2264 deletions(-) delete mode 100644 apps/server/otel/grafana/dashboards/airi-engagement-cloud.json delete mode 100644 apps/server/otel/grafana/dashboards/airi-revenue-cloud.json delete mode 100644 apps/server/otel/grafana/dashboards/airi-server-overview.json delete mode 100644 apps/server/otel/grafana/dashboards/airi-user-auth-cloud.json diff --git a/apps/server/otel/grafana/dashboards/airi-engagement-cloud.json b/apps/server/otel/grafana/dashboards/airi-engagement-cloud.json deleted file mode 100644 index 9680c08e6..000000000 --- a/apps/server/otel/grafana/dashboards/airi-engagement-cloud.json +++ /dev/null @@ -1,279 +0,0 @@ -{ - "annotations": { "list": [] }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "links": [], - "panels": [ - { - "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, - "id": 100, - "title": "Chat", - "type": "row" - }, - { - "datasource": { "type": "prometheus", "uid": "grafanacloud-projairi-prom" }, - "fieldConfig": { - "defaults": { - "color": { "mode": "palette-classic" }, - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 50 }, - { "color": "red", "value": 200 } - ] - }, - "unit": "short" - } - }, - "gridPos": { "h": 8, "w": 8, "x": 0, "y": 1 }, - "id": 1, - "options": { - "reduceOptions": { "calcs": ["lastNotNull"] }, - "colorMode": "value", - "graphMode": "area" - }, - "title": "Chat Syncs (1h)", - "type": "stat", - "targets": [ - { - "expr": "sum(increase(chat_sync_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[1h]))", - "legendFormat": "syncs" - } - ] - }, - { - "datasource": { "type": "prometheus", "uid": "grafanacloud-projairi-prom" }, - "fieldConfig": { - "defaults": { - "color": { "mode": "palette-classic" }, - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 100 }, - { "color": "red", "value": 500 } - ] - }, - "unit": "short" - } - }, - "gridPos": { "h": 8, "w": 8, "x": 8, "y": 1 }, - "id": 2, - "options": { - "reduceOptions": { "calcs": ["lastNotNull"] }, - "colorMode": "value", - "graphMode": "area" - }, - "title": "Messages (1h)", - "type": "stat", - "targets": [ - { - "expr": "sum(increase(chat_messages_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[1h]))", - "legendFormat": "messages" - } - ] - }, - { - "datasource": { "type": "prometheus", "uid": "grafanacloud-projairi-prom" }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 20, - "stacking": { "mode": "none" } - }, - "unit": "reqps" - } - }, - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 9 }, - "id": 3, - "title": "Chat Sync Rate by Type", - "type": "timeseries", - "targets": [ - { - "expr": "sum by (chat_type) (rate(chat_sync_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))", - "legendFormat": "{{chat_type}}" - } - ] - }, - { - "datasource": { "type": "prometheus", "uid": "grafanacloud-projairi-prom" }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 20, - "stacking": { "mode": "none" } - }, - "unit": "short" - } - }, - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 9 }, - "id": 4, - "title": "Message Volume Over Time", - "type": "timeseries", - "targets": [ - { - "expr": "sum(rate(chat_messages_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))", - "legendFormat": "messages/s" - } - ] - }, - { - "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 17 }, - "id": 101, - "title": "Characters", - "type": "row" - }, - { - "datasource": { "type": "prometheus", "uid": "grafanacloud-projairi-prom" }, - "fieldConfig": { - "defaults": { - "color": { "mode": "palette-classic" }, - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 10 }, - { "color": "red", "value": 50 } - ] - }, - "unit": "short" - } - }, - "gridPos": { "h": 8, "w": 8, "x": 0, "y": 18 }, - "id": 5, - "options": { - "reduceOptions": { "calcs": ["lastNotNull"] }, - "colorMode": "value", - "graphMode": "area" - }, - "title": "Characters Created (24h)", - "type": "stat", - "targets": [ - { - "expr": "sum(increase(character_created_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[24h]))", - "legendFormat": "created" - } - ] - }, - { - "datasource": { "type": "prometheus", "uid": "grafanacloud-projairi-prom" }, - "fieldConfig": { - "defaults": { - "color": { "mode": "palette-classic" }, - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 5 }, - { "color": "red", "value": 20 } - ] - }, - "unit": "short" - } - }, - "gridPos": { "h": 8, "w": 8, "x": 8, "y": 18 }, - "id": 6, - "options": { - "reduceOptions": { "calcs": ["lastNotNull"] }, - "colorMode": "value", - "graphMode": "area" - }, - "title": "Characters Deleted (24h)", - "type": "stat", - "targets": [ - { - "expr": "sum(increase(character_deleted_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[24h]))", - "legendFormat": "deleted" - } - ] - }, - { - "datasource": { "type": "prometheus", "uid": "grafanacloud-projairi-prom" }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "bars", - "fillOpacity": 80, - "stacking": { "mode": "normal" } - }, - "unit": "short" - } - }, - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 26 }, - "id": 7, - "title": "Engagement Actions", - "type": "timeseries", - "targets": [ - { - "expr": "sum by (action) (rate(character_engagement_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))", - "legendFormat": "{{action}}" - } - ] - }, - { - "datasource": { "type": "prometheus", "uid": "grafanacloud-projairi-prom" }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 20, - "stacking": { "mode": "none" } - }, - "unit": "short" - } - }, - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 26 }, - "id": 8, - "title": "Character Creation Over Time", - "type": "timeseries", - "targets": [ - { - "expr": "sum(rate(character_created_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))", - "legendFormat": "created/s" - } - ] - } - ], - "schemaVersion": 39, - "tags": ["airi", "engagement", "grafana-cloud"], - "templating": { - "list": [ - { - "current": { "selected": false, "text": "production", "value": "production" }, - "datasource": { "type": "prometheus", "uid": "grafanacloud-projairi-prom" }, - "definition": "label_values(http_server_request_duration_milliseconds_count, deployment_environment)", - "hide": 0, - "includeAll": true, - "multi": false, - "name": "env", - "query": "label_values(http_server_request_duration_milliseconds_count, deployment_environment)", - "refresh": 2, - "type": "query" - }, - { - "current": { "selected": false, "text": "server", "value": "server" }, - "datasource": { "type": "prometheus", "uid": "grafanacloud-projairi-prom" }, - "definition": "label_values(http_server_request_duration_milliseconds_count{deployment_environment=~\"$env\"}, service_name)", - "hide": 0, - "includeAll": true, - "multi": true, - "name": "service", - "query": "label_values(http_server_request_duration_milliseconds_count{deployment_environment=~\"$env\"}, service_name)", - "refresh": 2, - "type": "query" - } - ] - }, - "time": { "from": "now-24h", "to": "now" }, - "title": "AIRI Engagement (Cloud)", - "uid": "airi-engagement-cloud" -} diff --git a/apps/server/otel/grafana/dashboards/airi-revenue-cloud.json b/apps/server/otel/grafana/dashboards/airi-revenue-cloud.json deleted file mode 100644 index d25b31143..000000000 --- a/apps/server/otel/grafana/dashboards/airi-revenue-cloud.json +++ /dev/null @@ -1,319 +0,0 @@ -{ - "annotations": { "list": [] }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "links": [], - "panels": [ - { - "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, - "id": 100, - "title": "Stripe", - "type": "row" - }, - { - "datasource": { "type": "prometheus", "uid": "grafanacloud-projairi-prom" }, - "fieldConfig": { - "defaults": { - "color": { "mode": "palette-classic" }, - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 10 }, - { "color": "red", "value": 50 } - ] - }, - "unit": "short" - } - }, - "gridPos": { "h": 8, "w": 6, "x": 0, "y": 1 }, - "id": 1, - "options": { - "reduceOptions": { "calcs": ["lastNotNull"] }, - "colorMode": "value", - "graphMode": "area" - }, - "title": "Checkouts Created (24h)", - "type": "stat", - "targets": [ - { - "expr": "sum(increase(stripe_checkout_created_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[24h]))", - "legendFormat": "created" - } - ] - }, - { - "datasource": { "type": "prometheus", "uid": "grafanacloud-projairi-prom" }, - "fieldConfig": { - "defaults": { - "color": { "mode": "palette-classic" }, - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 5 }, - { "color": "red", "value": 30 } - ] - }, - "unit": "short" - } - }, - "gridPos": { "h": 8, "w": 6, "x": 6, "y": 1 }, - "id": 2, - "options": { - "reduceOptions": { "calcs": ["lastNotNull"] }, - "colorMode": "value", - "graphMode": "area" - }, - "title": "Checkouts Completed (24h)", - "type": "stat", - "targets": [ - { - "expr": "sum(increase(stripe_checkout_completed_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[24h]))", - "legendFormat": "completed" - } - ] - }, - { - "datasource": { "type": "prometheus", "uid": "grafanacloud-projairi-prom" }, - "fieldConfig": { - "defaults": { - "color": { "mode": "palette-classic" }, - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "red", "value": null }, - { "color": "yellow", "value": 1 }, - { "color": "red", "value": 5 } - ] - }, - "unit": "short" - } - }, - "gridPos": { "h": 8, "w": 6, "x": 12, "y": 1 }, - "id": 3, - "options": { - "reduceOptions": { "calcs": ["lastNotNull"] }, - "colorMode": "value", - "graphMode": "area" - }, - "title": "Payment Failures (24h)", - "type": "stat", - "targets": [ - { - "expr": "sum(increase(stripe_payment_failed_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[24h]))", - "legendFormat": "failures" - } - ] - }, - { - "datasource": { "type": "prometheus", "uid": "grafanacloud-projairi-prom" }, - "fieldConfig": { - "defaults": { - "color": { "mode": "palette-classic" }, - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "red", "value": null }, - { "color": "yellow", "value": 0.5 }, - { "color": "green", "value": 0.8 } - ] - }, - "unit": "percentunit" - } - }, - "gridPos": { "h": 8, "w": 6, "x": 18, "y": 1 }, - "id": 4, - "options": { - "reduceOptions": { "calcs": ["lastNotNull"] }, - "colorMode": "value", - "graphMode": "area" - }, - "title": "Conversion Rate", - "type": "stat", - "targets": [ - { - "expr": "sum(increase(stripe_checkout_completed_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[24h])) / sum(increase(stripe_checkout_created_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[24h]))", - "legendFormat": "conversion" - } - ] - }, - { - "datasource": { "type": "prometheus", "uid": "grafanacloud-projairi-prom" }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 20, - "stacking": { "mode": "none" } - }, - "unit": "short" - } - }, - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 9 }, - "id": 5, - "title": "Checkout Funnel Over Time", - "type": "timeseries", - "targets": [ - { - "expr": "sum(rate(stripe_checkout_created_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))", - "legendFormat": "created/s" - }, - { - "expr": "sum(rate(stripe_checkout_completed_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))", - "legendFormat": "completed/s" - }, - { - "expr": "sum(rate(stripe_payment_failed_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))", - "legendFormat": "failed/s" - } - ] - }, - { - "datasource": { "type": "prometheus", "uid": "grafanacloud-projairi-prom" }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "bars", - "fillOpacity": 80, - "stacking": { "mode": "normal" } - }, - "unit": "short" - } - }, - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 9 }, - "id": 6, - "title": "Subscription Events", - "type": "timeseries", - "targets": [ - { - "expr": "sum by (event_type) (rate(stripe_subscription_event_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))", - "legendFormat": "{{event_type}}" - } - ] - }, - { - "datasource": { "type": "prometheus", "uid": "grafanacloud-projairi-prom" }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 20, - "stacking": { "mode": "none" } - }, - "unit": "short" - } - }, - "gridPos": { "h": 8, "w": 24, "x": 0, "y": 17 }, - "id": 7, - "title": "Stripe Events by Type", - "type": "timeseries", - "targets": [ - { - "expr": "sum by (event_type) (rate(stripe_events_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))", - "legendFormat": "{{event_type}}" - } - ] - }, - { - "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 25 }, - "id": 101, - "title": "Flux", - "type": "row" - }, - { - "datasource": { "type": "prometheus", "uid": "grafanacloud-projairi-prom" }, - "fieldConfig": { - "defaults": { - "color": { "mode": "palette-classic" }, - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 5 }, - { "color": "red", "value": 20 } - ] - }, - "unit": "short" - } - }, - "gridPos": { "h": 8, "w": 8, "x": 0, "y": 26 }, - "id": 8, - "options": { - "reduceOptions": { "calcs": ["lastNotNull"] }, - "colorMode": "value", - "graphMode": "area" - }, - "title": "Insufficient Balance (1h)", - "type": "stat", - "targets": [ - { - "expr": "sum(increase(flux_insufficient_balance_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[1h]))", - "legendFormat": "insufficient" - } - ] - }, - { - "datasource": { "type": "prometheus", "uid": "grafanacloud-projairi-prom" }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 20, - "stacking": { "mode": "none" } - }, - "unit": "short" - } - }, - "gridPos": { "h": 8, "w": 16, "x": 8, "y": 26 }, - "id": 9, - "title": "Insufficient Balance Over Time", - "type": "timeseries", - "targets": [ - { - "expr": "sum(rate(flux_insufficient_balance_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))", - "legendFormat": "insufficient/s" - } - ] - } - ], - "schemaVersion": 39, - "tags": ["airi", "revenue", "stripe", "grafana-cloud"], - "templating": { - "list": [ - { - "current": { "selected": false, "text": "production", "value": "production" }, - "datasource": { "type": "prometheus", "uid": "grafanacloud-projairi-prom" }, - "definition": "label_values(http_server_request_duration_milliseconds_count, deployment_environment)", - "hide": 0, - "includeAll": true, - "multi": false, - "name": "env", - "query": "label_values(http_server_request_duration_milliseconds_count, deployment_environment)", - "refresh": 2, - "type": "query" - }, - { - "current": { "selected": false, "text": "server", "value": "server" }, - "datasource": { "type": "prometheus", "uid": "grafanacloud-projairi-prom" }, - "definition": "label_values(http_server_request_duration_milliseconds_count{deployment_environment=~\"$env\"}, service_name)", - "hide": 0, - "includeAll": true, - "multi": true, - "name": "service", - "query": "label_values(http_server_request_duration_milliseconds_count{deployment_environment=~\"$env\"}, service_name)", - "refresh": 2, - "type": "query" - } - ] - }, - "time": { "from": "now-7d", "to": "now" }, - "title": "AIRI Revenue (Cloud)", - "uid": "airi-revenue-cloud" -} diff --git a/apps/server/otel/grafana/dashboards/airi-server-overview-cloud.json b/apps/server/otel/grafana/dashboards/airi-server-overview-cloud.json index fd6df97b1..8c338cc7e 100644 --- a/apps/server/otel/grafana/dashboards/airi-server-overview-cloud.json +++ b/apps/server/otel/grafana/dashboards/airi-server-overview-cloud.json @@ -1,751 +1,1551 @@ { - "annotations": { - "list": [] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "links": [], - "panels": [ + "annotations": [ { - "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, - "id": 100, - "title": "HTTP Overview", - "type": "row" - }, - { - "datasource": { "type": "prometheus", "uid": "grafanacloud-projairi-prom" }, - "fieldConfig": { - "defaults": { - "color": { "mode": "palette-classic" }, - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 100 }, - { "color": "red", "value": 500 } - ] - }, - "unit": "reqps" + "kind": "AnnotationQuery", + "spec": { + "builtIn": true, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "query": { + "datasource": { "name": "-- Grafana --" }, + "group": "grafana", + "kind": "DataQuery", + "spec": {}, + "version": "v0" } - }, - "gridPos": { "h": 8, "w": 8, "x": 0, "y": 1 }, - "id": 1, - "options": { - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto" - }, - "title": "Request Rate", - "type": "stat", - "targets": [ - { - "expr": "sum(rate(http_server_request_duration_seconds_count{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))", - "legendFormat": "req/s" - } - ] - }, - { - "datasource": { "type": "prometheus", "uid": "grafanacloud-projairi-prom" }, - "fieldConfig": { - "defaults": { - "color": { "mode": "palette-classic" }, - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 0.01 }, - { "color": "red", "value": 0.05 } - ] - }, - "unit": "percentunit" - } - }, - "gridPos": { "h": 8, "w": 8, "x": 8, "y": 1 }, - "id": 2, - "options": { - "reduceOptions": { "calcs": ["lastNotNull"] }, - "colorMode": "value", - "graphMode": "area" - }, - "title": "Error Rate (5xx)", - "type": "stat", - "targets": [ - { - "expr": "sum(rate(http_server_request_duration_seconds_count{service_name=~\"$service\", deployment_environment=~\"$env\", http_status_code=~\"5..\"}[$__rate_interval])) / sum(rate(http_server_request_duration_seconds_count{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))", - "legendFormat": "error rate" - } - ] - }, - { - "datasource": { "type": "prometheus", "uid": "grafanacloud-projairi-prom" }, - "fieldConfig": { - "defaults": { - "color": { "mode": "palette-classic" }, - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 0.2 }, - { "color": "red", "value": 1.0 } - ] - }, - "unit": "s" - } - }, - "gridPos": { "h": 8, "w": 8, "x": 16, "y": 1 }, - "id": 3, - "options": { - "reduceOptions": { "calcs": ["lastNotNull"] }, - "colorMode": "value", - "graphMode": "area" - }, - "title": "P95 Latency (excl. LLM)", - "type": "stat", - "targets": [ - { - "expr": "histogram_quantile(0.95, sum(rate(http_server_request_duration_seconds_bucket{service_name=~\"$service\", deployment_environment=~\"$env\", http_route!~\"/api/v1/openai/.*\"}[$__rate_interval])) by (le))", - "legendFormat": "p95" - } - ] - }, - { - "datasource": { "type": "prometheus", "uid": "grafanacloud-projairi-prom" }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 20, - "stacking": { "mode": "none" } - }, - "unit": "reqps" - } - }, - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 9 }, - "id": 4, - "title": "Request Rate by Route", - "type": "timeseries", - "targets": [ - { - "expr": "sum by (http_route) (rate(http_server_request_duration_seconds_count{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))", - "legendFormat": "{{http_route}}" - } - ] - }, - { - "datasource": { "type": "prometheus", "uid": "grafanacloud-projairi-prom" }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 20 - }, - "unit": "s" - } - }, - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 9 }, - "id": 5, - "title": "Latency by Route (P95)", - "type": "timeseries", - "targets": [ - { - "expr": "histogram_quantile(0.95, sum by (le, http_route) (rate(http_server_request_duration_seconds_bucket{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval])))", - "legendFormat": "{{http_route}}" - } - ] - }, - { - "datasource": { "type": "prometheus", "uid": "grafanacloud-projairi-prom" }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "bars", - "fillOpacity": 80, - "stacking": { "mode": "normal" } - }, - "unit": "reqps" - } - }, - "gridPos": { "h": 8, "w": 24, "x": 0, "y": 17 }, - "id": 6, - "title": "Response Status Codes", - "type": "timeseries", - "targets": [ - { - "expr": "sum by (http_status_code) (rate(http_server_request_duration_seconds_count{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))", - "legendFormat": "{{http_status_code}}" - } - ] - }, - { - "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 25 }, - "id": 101, - "title": "Node.js Runtime", - "type": "row" - }, - { - "datasource": { "type": "prometheus", "uid": "grafanacloud-projairi-prom" }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 20 - }, - "unit": "bytes" - } - }, - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 26 }, - "id": 7, - "title": "Heap Memory Usage", - "type": "timeseries", - "targets": [ - { - "expr": "process_runtime_nodejs_memory_heap_total{service_name=~\"$service\", deployment_environment=~\"$env\"}", - "legendFormat": "heap total" - }, - { - "expr": "process_runtime_nodejs_memory_heap_used{service_name=~\"$service\", deployment_environment=~\"$env\"}", - "legendFormat": "heap used" - } - ] - }, - { - "datasource": { "type": "prometheus", "uid": "grafanacloud-projairi-prom" }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 20 - }, - "unit": "bytes" - } - }, - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 26 }, - "id": 8, - "title": "RSS & External Memory", - "type": "timeseries", - "targets": [ - { - "expr": "process_runtime_nodejs_memory_rss{service_name=~\"$service\", deployment_environment=~\"$env\"}", - "legendFormat": "RSS" - }, - { - "expr": "process_runtime_nodejs_memory_array_buffers{service_name=~\"$service\", deployment_environment=~\"$env\"}", - "legendFormat": "array buffers" - } - ] - }, - { - "datasource": { "type": "prometheus", "uid": "grafanacloud-projairi-prom" }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 20 - }, - "unit": "short" - } - }, - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 34 }, - "id": 9, - "title": "Event Loop Lag", - "type": "timeseries", - "targets": [ - { - "expr": "process_runtime_nodejs_event_loop_lag_p99{service_name=~\"$service\", deployment_environment=~\"$env\"}", - "legendFormat": "p99 lag" - }, - { - "expr": "process_runtime_nodejs_event_loop_lag_p50{service_name=~\"$service\", deployment_environment=~\"$env\"}", - "legendFormat": "p50 lag" - } - ] - }, - { - "datasource": { "type": "prometheus", "uid": "grafanacloud-projairi-prom" }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 20 - }, - "unit": "short" - } - }, - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 34 }, - "id": 10, - "title": "Active Handles & Requests", - "type": "timeseries", - "targets": [ - { - "expr": "process_runtime_nodejs_active_handles_total{service_name=~\"$service\", deployment_environment=~\"$env\"}", - "legendFormat": "active handles" - }, - { - "expr": "process_runtime_nodejs_active_requests_total{service_name=~\"$service\", deployment_environment=~\"$env\"}", - "legendFormat": "active requests" - } - ] - }, - { - "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 42 }, - "id": 102, - "title": "PostgreSQL", - "type": "row" - }, - { - "datasource": { "type": "prometheus", "uid": "grafanacloud-projairi-prom" }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 20 - }, - "unit": "s" - } - }, - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 43 }, - "id": 11, - "title": "DB Query Duration (P95)", - "type": "timeseries", - "targets": [ - { - "expr": "histogram_quantile(0.95, sum by (le, db_operation) (rate(db_client_operation_duration_seconds_bucket{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval])))", - "legendFormat": "{{db_operation}}" - } - ] - }, - { - "datasource": { "type": "prometheus", "uid": "grafanacloud-projairi-prom" }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 20 - }, - "unit": "ops" - } - }, - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 43 }, - "id": 12, - "title": "DB Query Rate", - "type": "timeseries", - "targets": [ - { - "expr": "sum by (db_operation) (rate(db_client_operation_duration_seconds_count{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))", - "legendFormat": "{{db_operation}}" - } - ] - }, - { - "datasource": { "type": "prometheus", "uid": "grafanacloud-projairi-prom" }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 20 - }, - "unit": "short" - } - }, - "gridPos": { "h": 8, "w": 24, "x": 0, "y": 51 }, - "id": 13, - "title": "DB Connection Pool", - "type": "timeseries", - "targets": [ - { - "expr": "db_client_connections_usage{service_name=~\"$service\", deployment_environment=~\"$env\", state=\"used\"}", - "legendFormat": "used" - }, - { - "expr": "db_client_connections_usage{service_name=~\"$service\", deployment_environment=~\"$env\", state=\"idle\"}", - "legendFormat": "idle" - }, - { - "expr": "db_client_connections_pending_requests{service_name=~\"$service\", deployment_environment=~\"$env\"}", - "legendFormat": "pending" - } - ] - }, - { - "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 59 }, - "id": 103, - "title": "Redis", - "type": "row" - }, - { - "datasource": { "type": "prometheus", "uid": "grafanacloud-projairi-prom" }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 20 - }, - "unit": "s" - } - }, - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 60 }, - "id": 14, - "title": "Redis Command Duration (P95)", - "type": "timeseries", - "targets": [ - { - "expr": "histogram_quantile(0.95, sum by (le, db_operation) (rate(db_client_operation_duration_seconds_bucket{service_name=~\"$service\", deployment_environment=~\"$env\", db_system=\"redis\"}[$__rate_interval])))", - "legendFormat": "{{db_operation}}" - } - ] - }, - { - "datasource": { "type": "prometheus", "uid": "grafanacloud-projairi-prom" }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 20 - }, - "unit": "ops" - } - }, - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 60 }, - "id": 15, - "title": "Redis Command Rate", - "type": "timeseries", - "targets": [ - { - "expr": "sum by (db_operation) (rate(db_client_operation_duration_seconds_count{service_name=~\"$service\", deployment_environment=~\"$env\", db_system=\"redis\"}[$__rate_interval]))", - "legendFormat": "{{db_operation}}" - } - ] - }, - { - "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 68 }, - "id": 105, - "title": "LLM Gateway", - "type": "row" - }, - { - "datasource": { "type": "prometheus", "uid": "grafanacloud-projairi-prom" }, - "fieldConfig": { - "defaults": { - "color": { "mode": "palette-classic" }, - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 50 }, - { "color": "red", "value": 200 } - ] - }, - "unit": "reqps" - } - }, - "gridPos": { "h": 8, "w": 8, "x": 0, "y": 69 }, - "id": 17, - "options": { - "reduceOptions": { "calcs": ["lastNotNull"] }, - "colorMode": "value", - "graphMode": "area" - }, - "title": "LLM Request Rate", - "type": "stat", - "targets": [ - { - "expr": "sum(rate(gen_ai_client_operation_count_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))", - "legendFormat": "req/s" - } - ] - }, - { - "datasource": { "type": "prometheus", "uid": "grafanacloud-projairi-prom" }, - "fieldConfig": { - "defaults": { - "color": { "mode": "palette-classic" }, - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 0.5 }, - { "color": "red", "value": 2.0 } - ] - }, - "unit": "s" - } - }, - "gridPos": { "h": 8, "w": 8, "x": 8, "y": 69 }, - "id": 18, - "options": { - "reduceOptions": { "calcs": ["lastNotNull"] }, - "colorMode": "value", - "graphMode": "area" - }, - "title": "LLM P95 Latency", - "type": "stat", - "targets": [ - { - "expr": "histogram_quantile(0.95, sum(rate(gen_ai_client_operation_duration_seconds_bucket{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval])) by (le))", - "legendFormat": "p95" - } - ] - }, - { - "datasource": { "type": "prometheus", "uid": "grafanacloud-projairi-prom" }, - "fieldConfig": { - "defaults": { - "color": { "mode": "palette-classic" }, - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 100 }, - { "color": "red", "value": 1000 } - ] - }, - "unit": "short" - } - }, - "gridPos": { "h": 8, "w": 8, "x": 16, "y": 69 }, - "id": 19, - "options": { - "reduceOptions": { "calcs": ["lastNotNull"] }, - "colorMode": "value", - "graphMode": "area" - }, - "title": "Total Flux Consumed", - "type": "stat", - "targets": [ - { - "expr": "sum(rate(airi_billing_flux_consumed_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))", - "legendFormat": "flux/s" - } - ] - }, - { - "datasource": { "type": "prometheus", "uid": "grafanacloud-projairi-prom" }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 20, - "stacking": { "mode": "none" } - }, - "unit": "reqps" - } - }, - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 77 }, - "id": 20, - "title": "LLM Request Rate by Model", - "type": "timeseries", - "targets": [ - { - "expr": "sum by (gen_ai_request_model) (rate(gen_ai_client_operation_count_total{service_name=~\"$service\", deployment_environment=~\"$env\", gen_ai_request_model!=\"\"}[$__rate_interval]))", - "legendFormat": "{{gen_ai_request_model}}" - } - ] - }, - { - "datasource": { "type": "prometheus", "uid": "grafanacloud-projairi-prom" }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 20 - }, - "unit": "reqps" - } - }, - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 77 }, - "id": 21, - "title": "LLM Request Rate by Type", - "type": "timeseries", - "targets": [ - { - "expr": "sum by (gen_ai_operation_name) (rate(gen_ai_client_operation_count_total{service_name=~\"$service\", deployment_environment=~\"$env\", gen_ai_operation_name!=\"\"}[$__rate_interval]))", - "legendFormat": "{{gen_ai_operation_name}}" - }, - { - "expr": "sum by (airi_gen_ai_operation_kind) (rate(gen_ai_client_operation_count_total{service_name=~\"$service\", deployment_environment=~\"$env\", airi_gen_ai_operation_kind!=\"\"}[$__rate_interval]))", - "legendFormat": "{{airi_gen_ai_operation_kind}}" - } - ] - }, - { - "datasource": { "type": "prometheus", "uid": "grafanacloud-projairi-prom" }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 20 - }, - "unit": "s" - } - }, - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 85 }, - "id": 22, - "title": "LLM Latency by Model (P95)", - "type": "timeseries", - "targets": [ - { - "expr": "histogram_quantile(0.95, sum by (le, gen_ai_request_model) (rate(gen_ai_client_operation_duration_seconds_bucket{service_name=~\"$service\", deployment_environment=~\"$env\", gen_ai_request_model!=\"\"}[$__rate_interval])))", - "legendFormat": "{{gen_ai_request_model}}" - } - ] - }, - { - "datasource": { "type": "prometheus", "uid": "grafanacloud-projairi-prom" }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 20 - }, - "unit": "short" - } - }, - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 85 }, - "id": 23, - "title": "Flux Consumed by Model", - "type": "timeseries", - "targets": [ - { - "expr": "sum by (gen_ai_request_model) (rate(airi_billing_flux_consumed_total{service_name=~\"$service\", deployment_environment=~\"$env\", gen_ai_request_model!=\"\"}[$__rate_interval]))", - "legendFormat": "{{gen_ai_request_model}}" - } - ] - }, - { - "datasource": { "type": "prometheus", "uid": "grafanacloud-projairi-prom" }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 20 - }, - "unit": "short" - } - }, - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 93 }, - "id": 24, - "title": "Prompt Tokens by Model", - "type": "timeseries", - "targets": [ - { - "expr": "sum by (gen_ai_request_model) (rate(gen_ai_client_token_usage_input_total{service_name=~\"$service\", deployment_environment=~\"$env\", gen_ai_request_model!=\"\"}[$__rate_interval]))", - "legendFormat": "{{gen_ai_request_model}}" - } - ] - }, - { - "datasource": { "type": "prometheus", "uid": "grafanacloud-projairi-prom" }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 20 - }, - "unit": "short" - } - }, - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 93 }, - "id": 25, - "title": "Completion Tokens by Model", - "type": "timeseries", - "targets": [ - { - "expr": "sum by (gen_ai_request_model) (rate(gen_ai_client_token_usage_output_total{service_name=~\"$service\", deployment_environment=~\"$env\", gen_ai_request_model!=\"\"}[$__rate_interval]))", - "legendFormat": "{{gen_ai_request_model}}" - } - ] - }, - { - "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 101 }, - "id": 104, - "title": "Application Logs", - "type": "row" - }, - { - "datasource": { "type": "loki", "uid": "grafanacloud-projairi-logs" }, - "gridPos": { "h": 10, "w": 24, "x": 0, "y": 102 }, - "id": 16, - "title": "Application Logs", - "type": "logs", - "targets": [ - { - "expr": "{service_name=~\"$service\", deployment_environment=~\"$env\"} |= ``", - "legendFormat": "" - } - ], - "options": { - "showTime": true, - "showLabels": true, - "showCommonLabels": false, - "wrapLogMessage": true, - "prettifyLogMessage": false, - "enableLogDetails": true, - "sortOrder": "Descending" } } ], - "schemaVersion": 39, + "cursorSync": "Crosshair", + "editable": true, + "elements": { + "panel-1": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { "name": "grafanacloud-projairi-prom" }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum(rate(http_server_request_duration_seconds_count{service_name=~\"$service\", deployment_environment=~\"$env\", http_request_method!=\"OPTIONS\"}[$__rate_interval]))", + "legendFormat": "req/s" + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "", + "id": 1, + "links": [], + "title": "Request Rate", + "vizConfig": { + "group": "stat", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": 0 }, + { "color": "yellow", "value": 100 }, + { "color": "red", "value": 500 } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + } + }, + "version": "13.0.0-23630096546" + } + } + }, + "panel-2": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { "name": "grafanacloud-projairi-prom" }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum(rate(http_server_request_duration_seconds_count{service_name=~\"$service\", deployment_environment=~\"$env\", http_request_method!=\"OPTIONS\", http_response_status_code=~\"5..\"}[$__rate_interval])) / sum(rate(http_server_request_duration_seconds_count{service_name=~\"$service\", deployment_environment=~\"$env\", http_request_method!=\"OPTIONS\"}[$__rate_interval]))", + "legendFormat": "error rate" + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "", + "id": 2, + "links": [], + "title": "Error Rate (5xx)", + "vizConfig": { + "group": "stat", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": 0 }, + { "color": "yellow", "value": 0.01 }, + { "color": "red", "value": 0.05 } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + } + }, + "version": "13.0.0-23630096546" + } + } + }, + "panel-3": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { "name": "grafanacloud-projairi-prom" }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "histogram_quantile(0.95, sum(rate(http_server_request_duration_seconds_bucket{service_name=~\"$service\", deployment_environment=~\"$env\", http_request_method!=\"OPTIONS\", http_route!~\"/api/v1/openai/.*\"}[$__rate_interval])) by (le))", + "legendFormat": "p95" + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "", + "id": 3, + "links": [], + "title": "P95 Latency (excl. LLM)", + "vizConfig": { + "group": "stat", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": 0 }, + { "color": "yellow", "value": 0.2 }, + { "color": "red", "value": 1 } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + } + }, + "version": "13.0.0-23630096546" + } + } + }, + "panel-4": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { "name": "grafanacloud-projairi-prom" }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum by (http_route, http_request_method) (rate(http_server_request_duration_seconds_count{service_name=~\"$service\", deployment_environment=~\"$env\", http_request_method!=\"OPTIONS\", http_response_status_code!=\"404\"}[$__rate_interval]))", + "legendFormat": "{{http_request_method}} {{http_route}}" + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { "name": "grafanacloud-projairi-prom" }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum(rate(http_server_request_duration_seconds_count{service_name=~\"$service\", deployment_environment=~\"$env\", http_request_method!=\"OPTIONS\", http_response_status_code=\"404\"}[$__rate_interval]))", + "legendFormat": "404 (all paths)" + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Excludes 404 responses to avoid high-cardinality route explosion", + "id": 4, + "links": [], + "title": "Request Rate by Route + Method", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "red", "value": 80 }] }, + "unit": "reqps" + }, + "overrides": [] + }, + "options": { + "annotations": { "clustering": -1, "multiLane": false }, + "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, + "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } + } + }, + "version": "13.0.0-23630096546" + } + } + }, + "panel-5": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { "name": "grafanacloud-projairi-prom" }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "histogram_quantile(0.95, sum by (le, http_route) (rate(http_server_request_duration_seconds_bucket{service_name=~\"$service\", deployment_environment=~\"$env\", http_request_method!=\"OPTIONS\", http_response_status_code!=\"404\"}[$__rate_interval])))", + "legendFormat": "{{http_route}}" + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { "name": "grafanacloud-projairi-prom" }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{service_name=~\"$service\", deployment_environment=~\"$env\", http_request_method!=\"OPTIONS\", http_response_status_code=\"404\"}[$__rate_interval])))", + "legendFormat": "404 (all paths)" + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Excludes 404 responses to avoid high-cardinality route explosion", + "id": 5, + "links": [], + "title": "Latency by Route (P95)", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "red", "value": 80 }] }, + "unit": "s" + }, + "overrides": [] + }, + "options": { + "annotations": { "clustering": -1, "multiLane": false }, + "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, + "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } + } + }, + "version": "13.0.0-23630096546" + } + } + }, + "panel-6": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { "name": "grafanacloud-projairi-prom" }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum by (http_response_status_code) (rate(http_server_request_duration_seconds_count{service_name=~\"$service\", deployment_environment=~\"$env\", http_request_method!=\"OPTIONS\"}[$__rate_interval]))", + "legendFormat": "{{http_response_status_code}}" + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "", + "id": 6, + "links": [], + "title": "Response Status Codes", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { "group": "A", "mode": "normal" }, + "thresholdsStyle": { "mode": "off" } + }, + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "red", "value": 80 }] }, + "unit": "reqps" + }, + "overrides": [] + }, + "options": { + "annotations": { "clustering": -1, "multiLane": false }, + "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, + "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } + } + }, + "version": "13.0.0-23630096546" + } + } + }, + "panel-30": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { "name": "grafanacloud-projairi-prom" }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum(rate(http_server_request_duration_seconds_count{service_name=~\"$service\", deployment_environment=~\"$env\", http_request_method!=\"OPTIONS\", http_response_status_code=\"404\"}[$__rate_interval]))", + "legendFormat": "404 total" + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "All 404 responses aggregated into a single series to avoid high-cardinality route explosion", + "id": 30, + "links": [], + "title": "404 Request Rate (aggregated)", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "red", "value": 80 }] }, + "unit": "reqps" + }, + "overrides": [] + }, + "options": { + "annotations": { "clustering": -1, "multiLane": false }, + "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, + "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } + } + }, + "version": "13.0.0-23630096546" + } + } + }, + "panel-31": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { "name": "grafanacloud-projairi-prom" }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum(rate(auth_attempts_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))", + "legendFormat": "attempts/s" + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "", + "id": 31, + "links": [], + "title": "Auth Attempts", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "red", "value": 80 }] }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "annotations": { "clustering": -1, "multiLane": false }, + "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, + "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } + } + }, + "version": "13.0.0-23630096546" + } + } + }, + "panel-32": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { "name": "grafanacloud-projairi-prom" }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum(rate(user_login_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))", + "legendFormat": "logins/s" + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { "name": "grafanacloud-projairi-prom" }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum(rate(user_registered_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))", + "legendFormat": "registrations/s" + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "", + "id": 32, + "links": [], + "title": "User Logins & Registrations", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "red", "value": 80 }] }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "annotations": { "clustering": -1, "multiLane": false }, + "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, + "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } + } + }, + "version": "13.0.0-23630096546" + } + } + }, + "panel-33": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { "name": "grafanacloud-projairi-prom" }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "user_active_sessions{service_name=~\"$service\", deployment_environment=~\"$env\"}", + "legendFormat": "active sessions" + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "", + "id": 33, + "links": [], + "title": "Active Sessions", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "red", "value": 80 }] }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "annotations": { "clustering": -1, "multiLane": false }, + "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, + "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } + } + }, + "version": "13.0.0-23630096546" + } + } + }, + "panel-7": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { "name": "grafanacloud-projairi-prom" }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "v8js_memory_heap_limit_bytes{service_name=~\"$service\", deployment_environment=~\"$env\"}", + "legendFormat": "heap limit" + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { "name": "grafanacloud-projairi-prom" }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "v8js_memory_heap_used_bytes{service_name=~\"$service\", deployment_environment=~\"$env\"}", + "legendFormat": "heap used" + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "", + "id": 7, + "links": [], + "title": "Heap Memory Usage", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "red", "value": 80 }] }, + "unit": "bytes" + }, + "overrides": [] + }, + "options": { + "annotations": { "clustering": -1, "multiLane": false }, + "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, + "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } + } + }, + "version": "13.0.0-23630096546" + } + } + }, + "panel-8": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { "name": "grafanacloud-projairi-prom" }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "v8js_memory_heap_space_physical_size_bytes{service_name=~\"$service\", deployment_environment=~\"$env\"}", + "legendFormat": "physical size" + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { "name": "grafanacloud-projairi-prom" }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "v8js_memory_heap_space_available_size_bytes{service_name=~\"$service\", deployment_environment=~\"$env\"}", + "legendFormat": "available size" + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "", + "id": 8, + "links": [], + "title": "Heap Space Details", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "red", "value": 80 }] }, + "unit": "bytes" + }, + "overrides": [] + }, + "options": { + "annotations": { "clustering": -1, "multiLane": false }, + "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, + "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } + } + }, + "version": "13.0.0-23630096546" + } + } + }, + "panel-9": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { "name": "grafanacloud-projairi-prom" }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "nodejs_eventloop_delay_p99_seconds{service_name=~\"$service\", deployment_environment=~\"$env\"}", + "legendFormat": "p99" + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { "name": "grafanacloud-projairi-prom" }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "nodejs_eventloop_delay_p50_seconds{service_name=~\"$service\", deployment_environment=~\"$env\"}", + "legendFormat": "p50" + }, + "version": "v0" + }, + "refId": "B" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { "name": "grafanacloud-projairi-prom" }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "nodejs_eventloop_delay_mean_seconds{service_name=~\"$service\", deployment_environment=~\"$env\"}", + "legendFormat": "mean" + }, + "version": "v0" + }, + "refId": "C" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "", + "id": 9, + "links": [], + "title": "Event Loop Delay", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "red", "value": 80 }] }, + "unit": "s" + }, + "overrides": [] + }, + "options": { + "annotations": { "clustering": -1, "multiLane": false }, + "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, + "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } + } + }, + "version": "13.0.0-23630096546" + } + } + }, + "panel-10": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { "name": "grafanacloud-projairi-prom" }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "nodejs_eventloop_utilization_ratio{service_name=~\"$service\", deployment_environment=~\"$env\"}", + "legendFormat": "utilization" + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { "name": "grafanacloud-projairi-prom" }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "http_server_active_requests{service_name=~\"$service\", deployment_environment=~\"$env\"}", + "legendFormat": "active HTTP requests" + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "", + "id": 10, + "links": [], + "title": "Event Loop Utilization & Active Requests", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "red", "value": 80 }] }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "annotations": { "clustering": -1, "multiLane": false }, + "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, + "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } + } + }, + "version": "13.0.0-23630096546" + } + } + }, + "panel-16": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { "name": "grafanacloud-projairi-logs" }, + "group": "loki", + "kind": "DataQuery", + "spec": { + "expr": "{service_name=~\"$service\", deployment_environment=~\"$env\"} |= ``", + "legendFormat": "" + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "", + "id": 16, + "links": [], + "title": "Application Logs", + "vizConfig": { + "group": "logs", + "kind": "VizConfig", + "spec": { + "fieldConfig": { "defaults": {}, "overrides": [] }, + "options": { + "dedupStrategy": "none", + "enableInfiniteScrolling": false, + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showControls": false, + "showFieldSelector": false, + "showLabels": true, + "showLevel": true, + "showLogAttributes": true, + "showTime": true, + "sortOrder": "Descending", + "timestampResolution": "ms", + "unwrappedColumns": false, + "wrapLogMessage": true + } + }, + "version": "13.0.0-23630096546" + } + } + }, + "panel-17": { + "kind": "Panel", + "spec": { + "data": { "kind": "QueryGroup", "spec": { "queries": [{ "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(rate(gen_ai_client_operation_count_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))", "legendFormat": "req/s" }, "version": "v0" }, "refId": "A" } }], "queryOptions": {}, "transformations": [] } }, + "description": "", + "id": 17, + "links": [], + "title": "LLM Request Rate", + "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "yellow", "value": 50 }, { "color": "red", "value": 200 }] }, "unit": "reqps" }, "overrides": [] }, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true } }, "version": "13.0.0-23630096546" } + } + }, + "panel-18": { + "kind": "Panel", + "spec": { + "data": { "kind": "QueryGroup", "spec": { "queries": [{ "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "histogram_quantile(0.95, sum(rate(gen_ai_client_operation_duration_seconds_bucket{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval])) by (le))", "legendFormat": "p95" }, "version": "v0" }, "refId": "A" } }], "queryOptions": {}, "transformations": [] } }, + "description": "", + "id": 18, + "links": [], + "title": "LLM P95 Latency", + "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "yellow", "value": 0.5 }, { "color": "red", "value": 2 }] }, "unit": "s" }, "overrides": [] }, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true } }, "version": "13.0.0-23630096546" } + } + }, + "panel-19": { + "kind": "Panel", + "spec": { + "data": { "kind": "QueryGroup", "spec": { "queries": [{ "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(rate(airi_billing_flux_consumed_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))", "legendFormat": "flux/s" }, "version": "v0" }, "refId": "A" } }], "queryOptions": {}, "transformations": [] } }, + "description": "", + "id": 19, + "links": [], + "title": "Total Flux Consumed", + "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "yellow", "value": 100 }, { "color": "red", "value": 1000 }] }, "unit": "short" }, "overrides": [] }, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true } }, "version": "13.0.0-23630096546" } + } + }, + "panel-20": { + "kind": "Panel", + "spec": { + "data": { "kind": "QueryGroup", "spec": { "queries": [{ "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (gen_ai_request_model) (rate(gen_ai_client_operation_count_total{service_name=~\"$service\", deployment_environment=~\"$env\", gen_ai_request_model!=\"\"}[$__rate_interval]))", "legendFormat": "{{gen_ai_request_model}}" }, "version": "v0" }, "refId": "A" } }], "queryOptions": {}, "transformations": [] } }, + "description": "", + "id": 20, + "links": [], + "title": "LLM Request Rate by Model", + "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "red", "value": 80 }] }, "unit": "reqps" }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.0.0-23630096546" } + } + }, + "panel-21": { + "kind": "Panel", + "spec": { + "data": { "kind": "QueryGroup", "spec": { "queries": [{ "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (gen_ai_operation_name) (rate(gen_ai_client_operation_count_total{service_name=~\"$service\", deployment_environment=~\"$env\", gen_ai_operation_name!=\"\"}[$__rate_interval]))", "legendFormat": "{{gen_ai_operation_name}}" }, "version": "v0" }, "refId": "A" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (airi_gen_ai_operation_kind) (rate(gen_ai_client_operation_count_total{service_name=~\"$service\", deployment_environment=~\"$env\", airi_gen_ai_operation_kind!=\"\"}[$__rate_interval]))", "legendFormat": "{{airi_gen_ai_operation_kind}}" }, "version": "v0" }, "refId": "B" } }], "queryOptions": {}, "transformations": [] } }, + "description": "", + "id": 21, + "links": [], + "title": "LLM Request Rate by Type", + "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "red", "value": 80 }] }, "unit": "reqps" }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.0.0-23630096546" } + } + }, + "panel-22": { + "kind": "Panel", + "spec": { + "data": { "kind": "QueryGroup", "spec": { "queries": [{ "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "histogram_quantile(0.95, sum by (le, gen_ai_request_model) (rate(gen_ai_client_operation_duration_seconds_bucket{service_name=~\"$service\", deployment_environment=~\"$env\", gen_ai_request_model!=\"\"}[$__rate_interval])))", "legendFormat": "{{gen_ai_request_model}}" }, "version": "v0" }, "refId": "A" } }], "queryOptions": {}, "transformations": [] } }, + "description": "", + "id": 22, + "links": [], + "title": "LLM Latency by Model (P95)", + "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "red", "value": 80 }] }, "unit": "s" }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.0.0-23630096546" } + } + }, + "panel-23": { + "kind": "Panel", + "spec": { + "data": { "kind": "QueryGroup", "spec": { "queries": [{ "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (gen_ai_request_model) (rate(airi_billing_flux_consumed_total{service_name=~\"$service\", deployment_environment=~\"$env\", gen_ai_request_model!=\"\"}[$__rate_interval]))", "legendFormat": "{{gen_ai_request_model}}" }, "version": "v0" }, "refId": "A" } }], "queryOptions": {}, "transformations": [] } }, + "description": "", + "id": 23, + "links": [], + "title": "Flux Consumed by Model", + "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "red", "value": 80 }] }, "unit": "short" }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.0.0-23630096546" } + } + }, + "panel-24": { + "kind": "Panel", + "spec": { + "data": { "kind": "QueryGroup", "spec": { "queries": [{ "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (gen_ai_request_model) (rate(gen_ai_client_token_usage_input_total{service_name=~\"$service\", deployment_environment=~\"$env\", gen_ai_request_model!=\"\"}[$__rate_interval]))", "legendFormat": "{{gen_ai_request_model}}" }, "version": "v0" }, "refId": "A" } }], "queryOptions": {}, "transformations": [] } }, + "description": "", + "id": 24, + "links": [], + "title": "Prompt Tokens by Model", + "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "red", "value": 80 }] }, "unit": "short" }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.0.0-23630096546" } + } + }, + "panel-25": { + "kind": "Panel", + "spec": { + "data": { "kind": "QueryGroup", "spec": { "queries": [{ "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (gen_ai_request_model) (rate(gen_ai_client_token_usage_output_total{service_name=~\"$service\", deployment_environment=~\"$env\", gen_ai_request_model!=\"\"}[$__rate_interval]))", "legendFormat": "{{gen_ai_request_model}}" }, "version": "v0" }, "refId": "A" } }], "queryOptions": {}, "transformations": [] } }, + "description": "", + "id": 25, + "links": [], + "title": "Completion Tokens by Model", + "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "red", "value": 80 }] }, "unit": "short" }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.0.0-23630096546" } + } + } + }, + "layout": { + "kind": "RowsLayout", + "spec": { + "rows": [ + { + "kind": "RowsLayoutRow", + "spec": { + "collapse": false, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-1" }, "height": 8, "width": 8, "x": 0, "y": 0 } }, + { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-2" }, "height": 8, "width": 8, "x": 8, "y": 0 } }, + { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-3" }, "height": 8, "width": 8, "x": 16, "y": 0 } }, + { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-4" }, "height": 8, "width": 12, "x": 0, "y": 8 } }, + { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-5" }, "height": 8, "width": 12, "x": 12, "y": 8 } }, + { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-6" }, "height": 8, "width": 12, "x": 0, "y": 16 } }, + { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-30" }, "height": 8, "width": 12, "x": 12, "y": 16 } } + ] + } + }, + "title": "HTTP Overview" + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "collapse": false, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-31" }, "height": 8, "width": 8, "x": 0, "y": 0 } }, + { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-32" }, "height": 8, "width": 8, "x": 8, "y": 0 } }, + { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-33" }, "height": 8, "width": 8, "x": 16, "y": 0 } } + ] + } + }, + "title": "Auth & Users" + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "collapse": false, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-7" }, "height": 8, "width": 12, "x": 0, "y": 0 } }, + { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-8" }, "height": 8, "width": 12, "x": 12, "y": 0 } }, + { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-9" }, "height": 8, "width": 12, "x": 0, "y": 8 } }, + { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-10" }, "height": 8, "width": 12, "x": 12, "y": 8 } } + ] + } + }, + "title": "Node.js Runtime" + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "collapse": false, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-17" }, "height": 8, "width": 8, "x": 0, "y": 0 } }, + { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-18" }, "height": 8, "width": 8, "x": 8, "y": 0 } }, + { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-19" }, "height": 8, "width": 8, "x": 16, "y": 0 } }, + { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-20" }, "height": 8, "width": 12, "x": 0, "y": 8 } }, + { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-21" }, "height": 8, "width": 12, "x": 12, "y": 8 } }, + { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-22" }, "height": 8, "width": 12, "x": 0, "y": 16 } }, + { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-23" }, "height": 8, "width": 12, "x": 12, "y": 16 } }, + { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-24" }, "height": 8, "width": 12, "x": 0, "y": 24 } }, + { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-25" }, "height": 8, "width": 12, "x": 12, "y": 24 } } + ] + } + }, + "title": "LLM Gateway" + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "collapse": false, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-16" }, "height": 10, "width": 24, "x": 0, "y": 0 } } + ] + } + }, + "title": "Application Logs" + } + } + ] + } + }, + "links": [], + "liveNow": false, + "preload": false, "tags": ["airi", "observability", "grafana-cloud"], - "templating": { - "list": [ - { - "current": { "selected": false, "text": "production", "value": "production" }, - "datasource": { "type": "prometheus", "uid": "grafanacloud-projairi-prom" }, + "timeSettings": { + "autoRefresh": "", + "autoRefreshIntervals": ["5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d"], + "fiscalYearStartMonth": 0, + "from": "now-1h", + "hideTimepicker": false, + "timezone": "browser", + "to": "now" + }, + "title": "AIRI Server Overview (Cloud)", + "variables": [ + { + "kind": "QueryVariable", + "spec": { + "allowCustomValue": true, + "current": { "text": "All", "value": "$__all" }, "definition": "label_values(http_server_request_duration_seconds_count, deployment_environment)", - "hide": 0, + "hide": "dontHide", "includeAll": true, "multi": false, "name": "env", - "query": "label_values(http_server_request_duration_seconds_count, deployment_environment)", - "refresh": 2, - "type": "query" - }, - { - "current": { "selected": false, "text": "server", "value": "server" }, - "datasource": { "type": "prometheus", "uid": "grafanacloud-projairi-prom" }, + "options": [], + "query": { + "datasource": { "name": "grafanacloud-projairi-prom" }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { "__legacyStringValue": "label_values(http_server_request_duration_seconds_count, deployment_environment)" }, + "version": "v0" + }, + "refresh": "onTimeRangeChanged", + "regex": "", + "regexApplyTo": "value", + "skipUrlSync": false, + "sort": "disabled" + } + }, + { + "kind": "QueryVariable", + "spec": { + "allowCustomValue": true, + "current": { "text": ["server"], "value": ["server"] }, "definition": "label_values(http_server_request_duration_seconds_count{deployment_environment=~\"$env\"}, service_name)", - "hide": 0, + "hide": "dontHide", "includeAll": true, "multi": true, "name": "service", - "query": "label_values(http_server_request_duration_seconds_count{deployment_environment=~\"$env\"}, service_name)", - "refresh": 2, - "type": "query" + "options": [], + "query": { + "datasource": { "name": "grafanacloud-projairi-prom" }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { "__legacyStringValue": "label_values(http_server_request_duration_seconds_count{deployment_environment=~\"$env\"}, service_name)" }, + "version": "v0" + }, + "refresh": "onTimeRangeChanged", + "regex": "", + "regexApplyTo": "value", + "skipUrlSync": false, + "sort": "disabled" } - ] - }, - "time": { "from": "now-1h", "to": "now" }, - "title": "AIRI Server Overview (Cloud)", - "uid": "airi-server-overview-cloud" + } + ] } diff --git a/apps/server/otel/grafana/dashboards/airi-server-overview.json b/apps/server/otel/grafana/dashboards/airi-server-overview.json deleted file mode 100644 index 8e894a3a7..000000000 --- a/apps/server/otel/grafana/dashboards/airi-server-overview.json +++ /dev/null @@ -1,503 +0,0 @@ -{ - "annotations": { - "list": [] - }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "links": [], - "panels": [ - { - "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, - "id": 100, - "title": "HTTP Overview", - "type": "row" - }, - { - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "fieldConfig": { - "defaults": { - "color": { "mode": "palette-classic" }, - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 100 }, - { "color": "red", "value": 500 } - ] - }, - "unit": "reqps" - } - }, - "gridPos": { "h": 8, "w": 8, "x": 0, "y": 1 }, - "id": 1, - "options": { - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto" - }, - "title": "Request Rate", - "type": "stat", - "targets": [ - { - "expr": "sum(rate(airi_http_server_request_duration_seconds_count{service_name=~\"$service\"}[$__rate_interval]))", - "legendFormat": "req/s" - } - ] - }, - { - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "fieldConfig": { - "defaults": { - "color": { "mode": "palette-classic" }, - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 0.01 }, - { "color": "red", "value": 0.05 } - ] - }, - "unit": "percentunit" - } - }, - "gridPos": { "h": 8, "w": 8, "x": 8, "y": 1 }, - "id": 2, - "options": { - "reduceOptions": { "calcs": ["lastNotNull"] }, - "colorMode": "value", - "graphMode": "area" - }, - "title": "Error Rate (5xx)", - "type": "stat", - "targets": [ - { - "expr": "sum(rate(airi_http_server_request_duration_seconds_count{service_name=~\"$service\", http_status_code=~\"5..\"}[$__rate_interval])) / sum(rate(airi_http_server_request_duration_seconds_count{service_name=~\"$service\"}[$__rate_interval]))", - "legendFormat": "error rate" - } - ] - }, - { - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "fieldConfig": { - "defaults": { - "color": { "mode": "palette-classic" }, - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 0.2 }, - { "color": "red", "value": 1.0 } - ] - }, - "unit": "s" - } - }, - "gridPos": { "h": 8, "w": 8, "x": 16, "y": 1 }, - "id": 3, - "options": { - "reduceOptions": { "calcs": ["lastNotNull"] }, - "colorMode": "value", - "graphMode": "area" - }, - "title": "P95 Latency (excl. LLM)", - "type": "stat", - "targets": [ - { - "expr": "histogram_quantile(0.95, sum(rate(airi_http_server_request_duration_seconds_bucket{service_name=~\"$service\", http_route!~\"/api/v1/openai/.*\"}[$__rate_interval])) by (le))", - "legendFormat": "p95" - } - ] - }, - { - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 20, - "stacking": { "mode": "none" } - }, - "unit": "reqps" - } - }, - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 9 }, - "id": 4, - "title": "Request Rate by Route", - "type": "timeseries", - "targets": [ - { - "expr": "sum by (http_route) (rate(airi_http_server_request_duration_seconds_count{service_name=~\"$service\"}[$__rate_interval]))", - "legendFormat": "{{http_route}}" - } - ] - }, - { - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 20 - }, - "unit": "s" - } - }, - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 9 }, - "id": 5, - "title": "Latency by Route (P95)", - "type": "timeseries", - "targets": [ - { - "expr": "histogram_quantile(0.95, sum by (le, http_route) (rate(airi_http_server_request_duration_seconds_bucket{service_name=~\"$service\"}[$__rate_interval])))", - "legendFormat": "{{http_route}}" - } - ] - }, - { - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "bars", - "fillOpacity": 80, - "stacking": { "mode": "normal" } - }, - "unit": "reqps" - } - }, - "gridPos": { "h": 8, "w": 24, "x": 0, "y": 17 }, - "id": 6, - "title": "Response Status Codes", - "type": "timeseries", - "targets": [ - { - "expr": "sum by (http_status_code) (rate(airi_http_server_request_duration_seconds_count{service_name=~\"$service\"}[$__rate_interval]))", - "legendFormat": "{{http_status_code}}" - } - ] - }, - { - "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 25 }, - "id": 101, - "title": "Node.js Runtime", - "type": "row" - }, - { - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 20 - }, - "unit": "bytes" - } - }, - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 26 }, - "id": 7, - "title": "Heap Memory Usage", - "type": "timeseries", - "targets": [ - { - "expr": "airi_process_runtime_nodejs_memory_heap_total{service_name=~\"$service\"}", - "legendFormat": "heap total" - }, - { - "expr": "airi_process_runtime_nodejs_memory_heap_used{service_name=~\"$service\"}", - "legendFormat": "heap used" - } - ] - }, - { - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 20 - }, - "unit": "bytes" - } - }, - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 26 }, - "id": 8, - "title": "RSS & External Memory", - "type": "timeseries", - "targets": [ - { - "expr": "airi_process_runtime_nodejs_memory_rss{service_name=~\"$service\"}", - "legendFormat": "RSS" - }, - { - "expr": "airi_process_runtime_nodejs_memory_array_buffers{service_name=~\"$service\"}", - "legendFormat": "array buffers" - } - ] - }, - { - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 20 - }, - "unit": "short" - } - }, - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 34 }, - "id": 9, - "title": "Event Loop Lag", - "type": "timeseries", - "targets": [ - { - "expr": "airi_process_runtime_nodejs_event_loop_lag_p99{service_name=~\"$service\"}", - "legendFormat": "p99 lag" - }, - { - "expr": "airi_process_runtime_nodejs_event_loop_lag_p50{service_name=~\"$service\"}", - "legendFormat": "p50 lag" - } - ] - }, - { - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 20 - }, - "unit": "short" - } - }, - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 34 }, - "id": 10, - "title": "Active Handles & Requests", - "type": "timeseries", - "targets": [ - { - "expr": "airi_process_runtime_nodejs_active_handles_total{service_name=~\"$service\"}", - "legendFormat": "active handles" - }, - { - "expr": "airi_process_runtime_nodejs_active_requests_total{service_name=~\"$service\"}", - "legendFormat": "active requests" - } - ] - }, - { - "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 42 }, - "id": 102, - "title": "PostgreSQL", - "type": "row" - }, - { - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 20 - }, - "unit": "s" - } - }, - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 43 }, - "id": 11, - "title": "DB Query Duration (P95)", - "type": "timeseries", - "targets": [ - { - "expr": "histogram_quantile(0.95, sum by (le, db_operation) (rate(airi_db_client_operation_duration_seconds_bucket{service_name=~\"$service\"}[$__rate_interval])))", - "legendFormat": "{{db_operation}}" - } - ] - }, - { - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 20 - }, - "unit": "ops" - } - }, - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 43 }, - "id": 12, - "title": "DB Query Rate", - "type": "timeseries", - "targets": [ - { - "expr": "sum by (db_operation) (rate(airi_db_client_operation_duration_seconds_count{service_name=~\"$service\"}[$__rate_interval]))", - "legendFormat": "{{db_operation}}" - } - ] - }, - { - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 20 - }, - "unit": "short" - } - }, - "gridPos": { "h": 8, "w": 24, "x": 0, "y": 51 }, - "id": 13, - "title": "DB Connection Pool", - "type": "timeseries", - "targets": [ - { - "expr": "airi_db_client_connections_usage{service_name=~\"$service\", state=\"used\"}", - "legendFormat": "used" - }, - { - "expr": "airi_db_client_connections_usage{service_name=~\"$service\", state=\"idle\"}", - "legendFormat": "idle" - }, - { - "expr": "airi_db_client_connections_pending_requests{service_name=~\"$service\"}", - "legendFormat": "pending" - } - ] - }, - { - "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 59 }, - "id": 103, - "title": "Redis", - "type": "row" - }, - { - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 20 - }, - "unit": "s" - } - }, - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 60 }, - "id": 14, - "title": "Redis Command Duration (P95)", - "type": "timeseries", - "targets": [ - { - "expr": "histogram_quantile(0.95, sum by (le, db_operation) (rate(airi_db_client_operation_duration_seconds_bucket{service_name=~\"$service\", db_system=\"redis\"}[$__rate_interval])))", - "legendFormat": "{{db_operation}}" - } - ] - }, - { - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 20 - }, - "unit": "ops" - } - }, - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 60 }, - "id": 15, - "title": "Redis Command Rate", - "type": "timeseries", - "targets": [ - { - "expr": "sum by (db_operation) (rate(airi_db_client_operation_duration_seconds_count{service_name=~\"$service\", db_system=\"redis\"}[$__rate_interval]))", - "legendFormat": "{{db_operation}}" - } - ] - }, - { - "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 68 }, - "id": 104, - "title": "Application Logs", - "type": "row" - }, - { - "datasource": { "type": "loki", "uid": "loki" }, - "gridPos": { "h": 10, "w": 24, "x": 0, "y": 69 }, - "id": 16, - "title": "Application Logs", - "type": "logs", - "targets": [ - { - "expr": "{service_name=~\"$service\"} |= ``", - "legendFormat": "" - } - ], - "options": { - "showTime": true, - "showLabels": true, - "showCommonLabels": false, - "wrapLogMessage": true, - "prettifyLogMessage": false, - "enableLogDetails": true, - "sortOrder": "Descending" - } - } - ], - "schemaVersion": 39, - "tags": ["airi", "observability"], - "templating": { - "list": [ - { - "current": { "selected": false, "text": "Prometheus", "value": "Prometheus" }, - "hide": 0, - "includeAll": false, - "name": "datasource", - "options": [], - "query": "prometheus", - "refresh": 1, - "type": "datasource" - }, - { - "current": { "selected": false, "text": "airi-server", "value": "airi-server" }, - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "definition": "label_values(airi_http_server_request_duration_seconds_count, service_name)", - "hide": 0, - "includeAll": true, - "multi": true, - "name": "service", - "query": "label_values(airi_http_server_request_duration_seconds_count, service_name)", - "refresh": 2, - "type": "query" - } - ] - }, - "time": { "from": "now-1h", "to": "now" }, - "title": "AIRI Server Overview", - "uid": "airi-server-overview" -} diff --git a/apps/server/otel/grafana/dashboards/airi-user-auth-cloud.json b/apps/server/otel/grafana/dashboards/airi-user-auth-cloud.json deleted file mode 100644 index 11a34af48..000000000 --- a/apps/server/otel/grafana/dashboards/airi-user-auth-cloud.json +++ /dev/null @@ -1,421 +0,0 @@ -{ - "annotations": { "list": [] }, - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "links": [], - "panels": [ - { - "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, - "id": 100, - "title": "Registration", - "type": "row" - }, - { - "datasource": { "type": "prometheus", "uid": "grafanacloud-projairi-prom" }, - "fieldConfig": { - "defaults": { - "color": { "mode": "palette-classic" }, - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 10 }, - { "color": "red", "value": 50 } - ] - }, - "unit": "short" - } - }, - "gridPos": { "h": 8, "w": 8, "x": 0, "y": 1 }, - "id": 1, - "options": { - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, - "orientation": "auto", - "textMode": "auto", - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto" - }, - "title": "New Users (24h)", - "type": "stat", - "targets": [ - { - "expr": "sum(increase(user_registered_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[24h]))", - "legendFormat": "new users" - } - ] - }, - { - "datasource": { "type": "prometheus", "uid": "grafanacloud-projairi-prom" }, - "fieldConfig": { - "defaults": { - "color": { "mode": "palette-classic" }, - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 5 }, - { "color": "red", "value": 20 } - ] - }, - "unit": "short" - } - }, - "gridPos": { "h": 8, "w": 8, "x": 8, "y": 1 }, - "id": 2, - "options": { - "reduceOptions": { "calcs": ["lastNotNull"] }, - "colorMode": "value", - "graphMode": "area" - }, - "title": "New Users (1h)", - "type": "stat", - "targets": [ - { - "expr": "sum(increase(user_registered_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[1h]))", - "legendFormat": "new users" - } - ] - }, - { - "datasource": { "type": "prometheus", "uid": "grafanacloud-projairi-prom" }, - "fieldConfig": { - "defaults": { - "color": { "mode": "palette-classic" }, - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 0.1 }, - { "color": "red", "value": 1 } - ] - }, - "unit": "reqps" - } - }, - "gridPos": { "h": 8, "w": 8, "x": 16, "y": 1 }, - "id": 3, - "options": { - "reduceOptions": { "calcs": ["lastNotNull"] }, - "colorMode": "value", - "graphMode": "area" - }, - "title": "Registration Rate", - "type": "stat", - "targets": [ - { - "expr": "sum(rate(user_registered_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))", - "legendFormat": "reg/s" - } - ] - }, - { - "datasource": { "type": "prometheus", "uid": "grafanacloud-projairi-prom" }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "bars", - "fillOpacity": 80, - "stacking": { "mode": "none" } - }, - "unit": "short" - } - }, - "gridPos": { "h": 8, "w": 24, "x": 0, "y": 9 }, - "id": 4, - "title": "Registrations Over Time", - "type": "timeseries", - "targets": [ - { - "expr": "sum(increase(user_registered_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))", - "legendFormat": "new users" - } - ] - }, - { - "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 17 }, - "id": 101, - "title": "Sessions", - "type": "row" - }, - { - "datasource": { "type": "prometheus", "uid": "grafanacloud-projairi-prom" }, - "fieldConfig": { - "defaults": { - "color": { "mode": "palette-classic" }, - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 50 }, - { "color": "red", "value": 200 } - ] - }, - "unit": "short" - } - }, - "gridPos": { "h": 8, "w": 8, "x": 0, "y": 18 }, - "id": 5, - "options": { - "reduceOptions": { "calcs": ["lastNotNull"] }, - "colorMode": "value", - "graphMode": "area" - }, - "title": "Active Sessions", - "type": "stat", - "targets": [ - { - "expr": "sum(user_active_sessions{service_name=~\"$service\", deployment_environment=~\"$env\"})", - "legendFormat": "active" - } - ] - }, - { - "datasource": { "type": "prometheus", "uid": "grafanacloud-projairi-prom" }, - "fieldConfig": { - "defaults": { - "color": { "mode": "palette-classic" }, - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 50 }, - { "color": "red", "value": 200 } - ] - }, - "unit": "short" - } - }, - "gridPos": { "h": 8, "w": 8, "x": 8, "y": 18 }, - "id": 6, - "options": { - "reduceOptions": { "calcs": ["lastNotNull"] }, - "colorMode": "value", - "graphMode": "area" - }, - "title": "Logins (1h)", - "type": "stat", - "targets": [ - { - "expr": "sum(increase(user_login_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[1h]))", - "legendFormat": "logins" - } - ] - }, - { - "datasource": { "type": "prometheus", "uid": "grafanacloud-projairi-prom" }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 20, - "stacking": { "mode": "none" } - }, - "unit": "short" - } - }, - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 26 }, - "id": 7, - "title": "Login Rate Over Time", - "type": "timeseries", - "targets": [ - { - "expr": "sum(rate(user_login_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))", - "legendFormat": "logins/s" - } - ] - }, - { - "datasource": { "type": "prometheus", "uid": "grafanacloud-projairi-prom" }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 20, - "stacking": { "mode": "none" } - }, - "unit": "short" - } - }, - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 26 }, - "id": 8, - "title": "Active Sessions Over Time", - "type": "timeseries", - "targets": [ - { - "expr": "sum(user_active_sessions{service_name=~\"$service\", deployment_environment=~\"$env\"})", - "legendFormat": "active sessions" - } - ] - }, - { - "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 34 }, - "id": 102, - "title": "Auth Health", - "type": "row" - }, - { - "datasource": { "type": "prometheus", "uid": "grafanacloud-projairi-prom" }, - "fieldConfig": { - "defaults": { - "color": { "mode": "palette-classic" }, - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 100 }, - { "color": "red", "value": 500 } - ] - }, - "unit": "short" - } - }, - "gridPos": { "h": 8, "w": 8, "x": 0, "y": 35 }, - "id": 9, - "options": { - "reduceOptions": { "calcs": ["lastNotNull"] }, - "colorMode": "value", - "graphMode": "area" - }, - "title": "Auth Attempts (1h)", - "type": "stat", - "targets": [ - { - "expr": "sum(increase(auth_attempts_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[1h]))", - "legendFormat": "attempts" - } - ] - }, - { - "datasource": { "type": "prometheus", "uid": "grafanacloud-projairi-prom" }, - "fieldConfig": { - "defaults": { - "color": { "mode": "palette-classic" }, - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 5 }, - { "color": "red", "value": 20 } - ] - }, - "unit": "short" - } - }, - "gridPos": { "h": 8, "w": 8, "x": 8, "y": 35 }, - "id": 10, - "options": { - "reduceOptions": { "calcs": ["lastNotNull"] }, - "colorMode": "value", - "graphMode": "area" - }, - "title": "Auth Failures (1h)", - "type": "stat", - "targets": [ - { - "expr": "sum(increase(auth_failures_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[1h]))", - "legendFormat": "failures" - } - ] - }, - { - "datasource": { "type": "prometheus", "uid": "grafanacloud-projairi-prom" }, - "fieldConfig": { - "defaults": { - "color": { "mode": "palette-classic" }, - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "green", "value": null }, - { "color": "yellow", "value": 0.05 }, - { "color": "red", "value": 0.2 } - ] - }, - "unit": "percentunit" - } - }, - "gridPos": { "h": 8, "w": 8, "x": 16, "y": 35 }, - "id": 11, - "options": { - "reduceOptions": { "calcs": ["lastNotNull"] }, - "colorMode": "value", - "graphMode": "area" - }, - "title": "Auth Failure Rate", - "type": "stat", - "targets": [ - { - "expr": "sum(rate(auth_failures_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval])) / sum(rate(auth_attempts_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))", - "legendFormat": "failure rate" - } - ] - }, - { - "datasource": { "type": "prometheus", "uid": "grafanacloud-projairi-prom" }, - "fieldConfig": { - "defaults": { - "custom": { - "drawStyle": "line", - "lineInterpolation": "smooth", - "fillOpacity": 20, - "stacking": { "mode": "none" } - }, - "unit": "short" - } - }, - "gridPos": { "h": 8, "w": 24, "x": 0, "y": 43 }, - "id": 12, - "title": "Auth Trends", - "type": "timeseries", - "targets": [ - { - "expr": "sum(rate(auth_attempts_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))", - "legendFormat": "attempts/s" - }, - { - "expr": "sum(rate(auth_failures_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))", - "legendFormat": "failures/s" - } - ] - } - ], - "schemaVersion": 39, - "tags": ["airi", "users", "auth", "grafana-cloud"], - "templating": { - "list": [ - { - "current": { "selected": false, "text": "production", "value": "production" }, - "datasource": { "type": "prometheus", "uid": "grafanacloud-projairi-prom" }, - "definition": "label_values(http_server_request_duration_milliseconds_count, deployment_environment)", - "hide": 0, - "includeAll": true, - "multi": false, - "name": "env", - "query": "label_values(http_server_request_duration_milliseconds_count, deployment_environment)", - "refresh": 2, - "type": "query" - }, - { - "current": { "selected": false, "text": "server", "value": "server" }, - "datasource": { "type": "prometheus", "uid": "grafanacloud-projairi-prom" }, - "definition": "label_values(http_server_request_duration_milliseconds_count{deployment_environment=~\"$env\"}, service_name)", - "hide": 0, - "includeAll": true, - "multi": true, - "name": "service", - "query": "label_values(http_server_request_duration_milliseconds_count{deployment_environment=~\"$env\"}, service_name)", - "refresh": 2, - "type": "query" - } - ] - }, - "time": { "from": "now-24h", "to": "now" }, - "title": "AIRI User & Auth (Cloud)", - "uid": "airi-user-auth-cloud" -} diff --git a/apps/server/src/middlewares/otel.ts b/apps/server/src/middlewares/otel.ts index 4dc6761d9..b16d527b5 100644 --- a/apps/server/src/middlewares/otel.ts +++ b/apps/server/src/middlewares/otel.ts @@ -11,11 +11,15 @@ const tracer = trace.getTracer('airi-server-hono') /** * Hono middleware that creates spans for each request and records - * custom HTTP metrics (duration, active requests, status codes). + * active request counts. + * + * NOTICE: Request duration is intentionally NOT recorded here. The + * Node HTTP instrumentation already emits `http.server.request.duration`, + * and recording the same metric here would double-count every request in + * Grafana panels that read the histogram `_count` series as request rate. */ export function otelMiddleware(http: HttpMetrics): MiddlewareHandler { return async (c, next) => { - const startTime = performance.now() const method = c.req.method const path = c.req.path @@ -38,12 +42,6 @@ export function otelMiddleware(http: HttpMetrics): MiddlewareHandler { if (status >= 500) { span.setStatus({ code: SpanStatusCode.ERROR, message: `HTTP ${status}` }) } - - http.requestDuration.record((performance.now() - startTime) / 1000, { - 'http.request.method': method, - 'http.route': path, - 'http.response.status_code': status, - }) } catch (err) { const errorMessage = errorMessageFromUnknown(err)