From 9aef35948c4f1ec83ed5c9ec4139f2f45dd29cc3 Mon Sep 17 00:00:00 2001 From: RainbowBird Date: Sat, 16 May 2026 03:59:19 +0800 Subject: [PATCH] refactor(server): llm router config sync subscriber --- .../ai-context/verifications/llm-router.md | 19 +- .../airi-server-overview-cloud.json | 947 ++++++++++++++++++ apps/server/otel/grafana/dashboards/build.ts | 138 +++ apps/server/src/app.ts | 35 +- .../llm-router/config-sync-subscriber.ts | 117 +++ apps/server/src/services/llm-router/index.ts | 3 + 6 files changed, 1229 insertions(+), 30 deletions(-) create mode 100644 apps/server/src/services/llm-router/config-sync-subscriber.ts diff --git a/apps/server/docs/ai-context/verifications/llm-router.md b/apps/server/docs/ai-context/verifications/llm-router.md index 8ee1d2326..24a88da99 100644 --- a/apps/server/docs/ai-context/verifications/llm-router.md +++ b/apps/server/docs/ai-context/verifications/llm-router.md @@ -113,10 +113,21 @@ stops asserting completion ahead of measurement. 2026-05-15: env entry removed, all routes go through `llmRouter.route` / `routeTts` / `listTtsVoices`. The `LLM_ROUTER_MASTER_KEY` env var is now required (no graceful skip). -- **Grafana dashboard JSON updates**: the new `airi.gen_ai.gateway.*` - counters are emitted from `apps/server/src/otel/index.ts` but the - Grafana dashboard JSON in `otel/grafana/dashboards/` does not yet have - panels for them. Plan U8 panel + alert work is deferred to a follow-up. +- ~~**Grafana dashboard JSON updates**: the new `airi.gen_ai.gateway.*` + counters … do not yet have panels for them~~. Partially resolved + 2026-05-16: `otel/grafana/dashboards/build.ts` generates three router + rows (Health / Trends / Admin Plane) covering the 7 gateway counters + that have live producers: `fallback_count`, `upstream_errors`, + `key_exhausted`, `same_status_exhaustion`, `decrypt_failures`, + `config_reload`, and `subscriber_state` (producer added in the same + PR — `app.ts` now emits `connected` / `error` / `reconnecting` from the + `configkv:invalidate` subscriber). The remaining two counters + (`config_write`, `config_invalid_hmac`) intentionally have no panels + because their producer is the Plan U9 admin HTTP endpoint that has + not shipped; they will rejoin Rows 6.5 / 6.7 alongside the U9 PR. + Alert rules (key.exhausted > 0, fallback ratio > 30%, single-key + error ratio > 80%) are still configured through Grafana UI, not + build.ts — IaC-ifying them is a separate follow-up. - **knoway compose retention**: keep `/Users/luoling8192/Git/proj-airi/airi-railway/knoway/` + the corresponding container entry in `airi-railway/docker-compose.yml` for **at least 14 days without a P1+ incident** before removing per plan R18. diff --git a/apps/server/otel/grafana/dashboards/airi-server-overview-cloud.json b/apps/server/otel/grafana/dashboards/airi-server-overview-cloud.json index a2d4d1b66..6aba7db4c 100644 --- a/apps/server/otel/grafana/dashboards/airi-server-overview-cloud.json +++ b/apps/server/otel/grafana/dashboards/airi-server-overview-cloud.json @@ -2231,6 +2231,807 @@ } } }, + "panel-60": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "grafanacloud-projairi-prom" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum(increase(airi_gen_ai_gateway_key_exhausted_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[5m]))", + "legendFormat": "events" + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Number of (model, upstream) pairs that ran out of usable keys within one user request over the last 5 minutes. Any non-zero value means at least one user request walked an entire upstream's key list without a 2xx. Sustained > 0 = a provider account is dead or every stored ciphertext is failing to decrypt — page on-call.", + "id": 60, + "links": [], + "title": "Key Exhausted (5m)", + "vizConfig": { + "group": "stat", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "short", + "noValue": "0" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + } + }, + "version": "13.0.0-23630096546" + } + } + }, + "panel-61": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "grafanacloud-projairi-prom" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum(increase(airi_gen_ai_gateway_decrypt_failures_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[5m]))", + "legendFormat": "events" + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Envelope-crypto decrypt failures in the key rotator. A non-zero value here is security-relevant: either the master key was rotated without re-wrapping ciphertexts, or someone forged a config blob. Triages straight to the seed script + master-key audit trail.", + "id": 61, + "links": [], + "title": "Decrypt Failures (5m)", + "vizConfig": { + "group": "stat", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "short", + "noValue": "0" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + } + }, + "version": "13.0.0-23630096546" + } + } + }, + "panel-62": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "grafanacloud-projairi-prom" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "100 * sum(rate(airi_gen_ai_gateway_fallback_count_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[5m])) / clamp_min(sum(rate(gen_ai_client_operation_count_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[5m])), 1)", + "legendFormat": "fallback %" + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Fallback attempts ÷ total LLM operations over the last 5m. Sustained > 30% means one provider is degraded and the router is silently masking it for users (but burning quota on the failing upstream). chat + tts share the operation counter so this is a cluster-wide health gauge.", + "id": 62, + "links": [], + "title": "Fallback Ratio % (5m)", + "vizConfig": { + "group": "gauge", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "yellow", + "value": 10 + }, + { + "color": "red", + "value": 30 + } + ] + }, + "unit": "percent", + "decimals": 1, + "noValue": "0", + "min": 0, + "max": 100 + }, + "overrides": [] + }, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + } + }, + "version": "13.0.0-23630096546" + } + } + }, + "panel-64": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "grafanacloud-projairi-prom" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum by (provider, reason) (rate(airi_gen_ai_gateway_fallback_count_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))", + "legendFormat": "{{provider}} · {{reason}}" + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Per-provider fallback events broken down by failure reason (HTTP status or `timeout`). A wide spread of reasons under one provider = transient upstream; a single reason dominating = systematic issue (e.g. 429 quota cap, 401 expired key).", + "id": 64, + "links": [], + "title": "Fallback Count by Provider + Reason", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + } + }, + "version": "13.0.0-23630096546" + } + } + }, + "panel-65": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "grafanacloud-projairi-prom" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum by (provider, status_code) (rate(airi_gen_ai_gateway_upstream_errors_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))", + "legendFormat": "{{provider}} · {{status_code}}" + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Per-upstream non-2xx response rate split by status code. Only counts attempts where the upstream actually answered (network timeouts and adapter aborts go to the fallback counter under `reason=timeout`). 401/403 = bad key; 429 = quota; 5xx = upstream outage.", + "id": 65, + "links": [], + "title": "Upstream Errors by Status Code", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + } + }, + "version": "13.0.0-23630096546" + } + } + }, + "panel-66": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "grafanacloud-projairi-prom" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum by (provider, status_code) (rate(airi_gen_ai_gateway_same_status_exhaustion_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))", + "legendFormat": "{{provider}} · {{status_code}}" + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Full-chain exhaustions where every attempt returned the same status code (or all timed out). A strong signal that ordinary key fallback cannot recover — points at an account-level cap or a shared backend brownout on the provider side. Each spike is one user request.", + "id": 66, + "links": [], + "title": "Same-Status Exhaustion by Provider/Status", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + } + }, + "version": "13.0.0-23630096546" + } + } + }, + "panel-67": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "grafanacloud-projairi-prom" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum by (source) (rate(airi_gen_ai_gateway_config_reload_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))", + "legendFormat": "{{source}}" + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Cache invalidations of `LLM_ROUTER_CONFIG` split by source (`pubsub` = Redis fan-out from a peer's write; `boot` = first load on startup). Steady low rate is normal. A sudden burst aligned with a deploy = expected. An unexplained burst = someone is writing to the config without auditing.", + "id": 67, + "links": [], + "title": "Config Reload Events", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + } + }, + "version": "13.0.0-23630096546" + } + } + }, + "panel-69": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "grafanacloud-projairi-prom" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum by (state, service_instance_id) (increase(airi_gen_ai_gateway_subscriber_state_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[5m]))", + "legendFormat": "{{state}} ({{service_instance_id}})" + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Lifecycle events for the cross-instance Redis Pub/Sub subscriber (`connected` on initial subscribe, `error` on connection error or subscribe failure, `reconnecting` while ioredis is re-establishing the connection). One `connected` per replica per deploy is normal. Sustained `error` or `reconnecting` without a matching `connected` means an instance is desynced from `configkv:invalidate` — its in-memory router config will drift up to the configCacheTtlMs fallback window.", + "id": 69, + "links": [], + "title": "Subscriber State Events (5m)", + "vizConfig": { + "group": "stat", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "short", + "noValue": "0" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + } + }, + "version": "13.0.0-23630096546" + } + } + }, "panel-50": { "kind": "Panel", "spec": { @@ -3200,6 +4001,152 @@ "title": "Business" } }, + { + "kind": "RowsLayoutRow", + "spec": { + "collapse": false, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-60" + }, + "height": 5, + "width": 8, + "x": 0, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-61" + }, + "height": 5, + "width": 8, + "x": 8, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-62" + }, + "height": 5, + "width": 8, + "x": 16, + "y": 0 + } + } + ] + } + }, + "title": "LLM Router Health" + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "collapse": false, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-64" + }, + "height": 8, + "width": 6, + "x": 0, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-65" + }, + "height": 8, + "width": 6, + "x": 6, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-66" + }, + "height": 8, + "width": 6, + "x": 12, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-67" + }, + "height": 8, + "width": 6, + "x": 18, + "y": 0 + } + } + ] + } + }, + "title": "LLM Router Trends" + } + }, + { + "kind": "RowsLayoutRow", + "spec": { + "collapse": true, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-69" + }, + "height": 5, + "width": 24, + "x": 0, + "y": 0 + } + } + ] + } + }, + "title": "Gateway Admin Plane" + } + }, { "kind": "RowsLayoutRow", "spec": { diff --git a/apps/server/otel/grafana/dashboards/build.ts b/apps/server/otel/grafana/dashboards/build.ts index 789683883..03ddcfca8 100644 --- a/apps/server/otel/grafana/dashboards/build.ts +++ b/apps/server/otel/grafana/dashboards/build.ts @@ -729,6 +729,118 @@ elements['panel-32'] = piePanel( { noValue: '—' }, ) +// Rows 6.5 / 6.6 / 6.7: LLM/TTS in-process router (KTD-5 / KTD-6). +// These counters are emitted from `apps/server/src/services/llm-router/router.ts` +// for every chat AND tts dispatch attempt. fallback_count / upstream_errors +// track per-attempt failures inside one user request; key_exhausted / +// same_status_exhaustion fire only on full chain exhaustion. config_* counters +// belong to the admin-plane subscriber and the seed/admin writers — separated +// into a collapsed row because operators only look at them during config +// rollout or post-incident. +// +// Counter → prom name mapping (OTel dot → underscore + `_total` for counters): +// airi.gen_ai.gateway.fallback.count → airi_gen_ai_gateway_fallback_count_total +// airi.gen_ai.gateway.upstream.errors → airi_gen_ai_gateway_upstream_errors_total +// airi.gen_ai.gateway.key.exhausted → airi_gen_ai_gateway_key_exhausted_total +// airi.gen_ai.gateway.same_status_exhaustion → airi_gen_ai_gateway_same_status_exhaustion_total +// airi.gen_ai.gateway.config.reload → airi_gen_ai_gateway_config_reload_total +// airi.gen_ai.gateway.decrypt.failures → airi_gen_ai_gateway_decrypt_failures_total +// airi.gen_ai.gateway.subscriber_state → airi_gen_ai_gateway_subscriber_state_total +// airi.gen_ai.gateway.config.write → airi_gen_ai_gateway_config_write_total +// airi.gen_ai.gateway.config.invalid_hmac → airi_gen_ai_gateway_config_invalid_hmac_total +elements['panel-60'] = statPanel( + 60, + 'Key Exhausted (5m)', + 'Number of (model, upstream) pairs that ran out of usable keys within one user request over the last 5 minutes. Any non-zero value means at least one user request walked an entire upstream\'s key list without a 2xx. Sustained > 0 = a provider account is dead or every stored ciphertext is failing to decrypt — page on-call.', + [query(`sum(increase(airi_gen_ai_gateway_key_exhausted_total{${SERVICE_FILTER}}[5m]))`, 'events')], + { unit: 'short', steps: [{ color: 'green', value: 0 }, { color: 'red', value: 1 }], noValue: '0' }, +) + +elements['panel-61'] = statPanel( + 61, + 'Decrypt Failures (5m)', + 'Envelope-crypto decrypt failures in the key rotator. A non-zero value here is security-relevant: either the master key was rotated without re-wrapping ciphertexts, or someone forged a config blob. Triages straight to the seed script + master-key audit trail.', + [query(`sum(increase(airi_gen_ai_gateway_decrypt_failures_total{${SERVICE_FILTER}}[5m]))`, 'events')], + { unit: 'short', steps: [{ color: 'green', value: 0 }, { color: 'red', value: 1 }], noValue: '0' }, +) + +elements['panel-62'] = gaugePanel( + 62, + 'Fallback Ratio % (5m)', + 'Fallback attempts ÷ total LLM operations over the last 5m. Sustained > 30% means one provider is degraded and the router is silently masking it for users (but burning quota on the failing upstream). chat + tts share the operation counter so this is a cluster-wide health gauge.', + [query( + `100 * sum(rate(airi_gen_ai_gateway_fallback_count_total{${SERVICE_FILTER}}[5m])) / clamp_min(sum(rate(gen_ai_client_operation_count_total{${SERVICE_FILTER}}[5m])), 1)`, + 'fallback %', + )], + { steps: [{ color: 'green', value: 0 }, { color: 'yellow', value: 10 }, { color: 'red', value: 30 }], max: 100, decimals: 1, noValue: '0' }, +) + +// panel-63 (Invalid HMAC Writes) intentionally not built: the producer +// (`config_invalid_hmac` counter) lives in the Plan U9 admin HTTP endpoint +// that hasn't shipped yet. Adding the panel now would surface a permanent +// zero that misleads readers into thinking "no attack" when it actually +// means "no producer". Re-add together with the U9 endpoint PR. + +elements['panel-64'] = timeseriesPanel( + 64, + 'Fallback Count by Provider + Reason', + 'Per-provider fallback events broken down by failure reason (HTTP status or `timeout`). A wide spread of reasons under one provider = transient upstream; a single reason dominating = systematic issue (e.g. 429 quota cap, 401 expired key).', + [query( + `sum by (provider, reason) (rate(airi_gen_ai_gateway_fallback_count_total{${SERVICE_FILTER}}[$__rate_interval]))`, + '{{provider}} · {{reason}}', + )], + { unit: 'ops' }, +) + +elements['panel-65'] = timeseriesPanel( + 65, + 'Upstream Errors by Status Code', + 'Per-upstream non-2xx response rate split by status code. Only counts attempts where the upstream actually answered (network timeouts and adapter aborts go to the fallback counter under `reason=timeout`). 401/403 = bad key; 429 = quota; 5xx = upstream outage.', + [query( + `sum by (provider, status_code) (rate(airi_gen_ai_gateway_upstream_errors_total{${SERVICE_FILTER}}[$__rate_interval]))`, + '{{provider}} · {{status_code}}', + )], + { unit: 'ops' }, +) + +elements['panel-66'] = timeseriesPanel( + 66, + 'Same-Status Exhaustion by Provider/Status', + 'Full-chain exhaustions where every attempt returned the same status code (or all timed out). A strong signal that ordinary key fallback cannot recover — points at an account-level cap or a shared backend brownout on the provider side. Each spike is one user request.', + [query( + `sum by (provider, status_code) (rate(airi_gen_ai_gateway_same_status_exhaustion_total{${SERVICE_FILTER}}[$__rate_interval]))`, + '{{provider}} · {{status_code}}', + )], + { unit: 'ops' }, +) + +elements['panel-67'] = timeseriesPanel( + 67, + 'Config Reload Events', + 'Cache invalidations of `LLM_ROUTER_CONFIG` split by source (`pubsub` = Redis fan-out from a peer\'s write; `boot` = first load on startup). Steady low rate is normal. A sudden burst aligned with a deploy = expected. An unexplained burst = someone is writing to the config without auditing.', + [query( + `sum by (source) (rate(airi_gen_ai_gateway_config_reload_total{${SERVICE_FILTER}}[$__rate_interval]))`, + '{{source}}', + )], + { unit: 'ops' }, +) + +// panel-68 (Config Writes) intentionally not built: same reasoning as panel-63 +// — the `config_write` counter has no producer until the Plan U9 admin +// endpoint ships. Re-add together with the U9 endpoint PR so the `by result` +// label set (success / etag_mismatch / validation_failed) is meaningful. + +elements['panel-69'] = statPanel( + 69, + 'Subscriber State Events (5m)', + 'Lifecycle events for the cross-instance Redis Pub/Sub subscriber (`connected` on initial subscribe, `error` on connection error or subscribe failure, `reconnecting` while ioredis is re-establishing the connection). One `connected` per replica per deploy is normal. Sustained `error` or `reconnecting` without a matching `connected` means an instance is desynced from `configkv:invalidate` — its in-memory router config will drift up to the configCacheTtlMs fallback window.', + [query( + `sum by (state, service_instance_id) (increase(airi_gen_ai_gateway_subscriber_state_total{${SERVICE_FILTER}}[5m]))`, + '{{state}} ({{service_instance_id}})', + )], + { unit: 'short', noValue: '0' }, +) + // Row 7: Infrastructure — process / DB health (collapsed by default) elements['panel-50'] = statPanel( 50, @@ -867,6 +979,32 @@ const rows = [ item('panel-31', 8, 0, 8, 7), item('panel-32', 16, 0, 8, 7), ]), + // Row 6.5: LLM router health — 3 stat/gauge × 8 wide × 5 high. + // These are the "wake someone up" indicators (key exhausted, decrypt + // failures, fallback storms). Always visible, no collapse, because they + // answer "is the in-process router healthy right now". The Invalid HMAC + // panel will join this row when the Plan U9 admin endpoint lands. + row('LLM Router Health', [ + item('panel-60', 0, 0, 8, 5), + item('panel-61', 8, 0, 8, 5), + item('panel-62', 16, 0, 8, 5), + ]), + // Row 6.6: LLM router trends — 4 timeseries × 6 wide × 8 high. + // Same dimensions as Row 6.5 but answers "how is it changing / which + // provider is contributing". Wider time context for triage. + row('LLM Router Trends', [ + item('panel-64', 0, 0, 6, 8), + item('panel-65', 6, 0, 6, 8), + item('panel-66', 12, 0, 6, 8), + item('panel-67', 18, 0, 6, 8), + ]), + // Row 6.7: Gateway admin plane — 1 stat × 24 wide × 5 high, collapsed by + // default. Tracks per-instance subscriber lifecycle for cross-instance + // config consistency triage. Config Writes panel rejoins this row when the + // Plan U9 admin endpoint lands. + row('Gateway Admin Plane', [ + item('panel-69', 0, 0, 24, 5), + ], { collapse: true }), // Row 7: 1 stat + 3 by-instance timeseries × 6 wide × 6 high (collapsed by // default — only relevant when triaging. By-instance breakdowns catch // single-replica issues that cluster aggregates would average away.) diff --git a/apps/server/src/app.ts b/apps/server/src/app.ts index 70db21869..e6d171d6d 100644 --- a/apps/server/src/app.ts +++ b/apps/server/src/app.ts @@ -63,7 +63,7 @@ import { createConfigKVService } from './services/config-kv' import { createEmailService } from './services/email' import { createFluxService } from './services/flux' import { createFluxTransactionService } from './services/flux-transaction' -import { createLlmRouterService } from './services/llm-router' +import { createConfigSyncSubscriber, createLlmRouterService } from './services/llm-router' import { createPostHogClient } from './services/posthog' import { createProviderService } from './services/providers' import { createRequestLogService } from './services/request-log' @@ -164,31 +164,14 @@ export async function buildApp(deps: AppDeps) { return chatWsSetup(session.user.id) })) - // Subscribe to the cross-instance config invalidation channel so admin - // writes to LLM_ROUTER_CONFIG propagate within ≤5s across the cluster - // (R16 / KTD-4). Falls back to the in-memory cache TTL on missed messages. - // Uses a dedicated ioredis subscriber connection (subscribe mode requires - // a separate connection per ioredis docs). - const configSub = deps.redis.duplicate() - configSub.on('message', (channel, message) => { - if (channel !== 'configkv:invalidate') - return - try { - const payload = JSON.parse(message) as { key?: unknown } - if (payload?.key !== 'LLM_ROUTER_CONFIG') - return - deps.llmRouter.invalidateConfig() - deps.otel?.gateway?.configReload.add(1, { - source: 'pubsub', - service_instance_id: deps.env.OTEL_SERVICE_NAME, - }) - } - catch (err) { - logger.withError(err).warn('Failed to parse configkv:invalidate payload') - } - }) - configSub.subscribe('configkv:invalidate').catch((err: unknown) => { - logger.withError(err).warn('Failed to subscribe to configkv:invalidate channel') + // Cross-instance config invalidation. The subscriber owns its own + // connection + lifecycle metrics; see services/llm-router/config-sync-subscriber.ts. + createConfigSyncSubscriber({ + redis: deps.redis, + llmRouter: deps.llmRouter, + gatewayMetrics: deps.otel?.gateway ?? null, + instanceId: deps.env.OTEL_SERVICE_NAME, + logger: useLogger('config-sync').useGlobalConfig(), }) const builtApp = app diff --git a/apps/server/src/services/llm-router/config-sync-subscriber.ts b/apps/server/src/services/llm-router/config-sync-subscriber.ts new file mode 100644 index 000000000..67862c18a --- /dev/null +++ b/apps/server/src/services/llm-router/config-sync-subscriber.ts @@ -0,0 +1,117 @@ +import type { useLogger } from '@guiiai/logg' +import type Redis from 'ioredis' + +import type { GatewayMetrics } from '../../otel' +import type { LlmRouterService } from './router' + +/** + * Dependencies needed to wire the cross-instance config invalidation + * subscriber. + */ +export interface ConfigSyncSubscriberOptions { + /** + * Primary Redis client. The subscriber takes its own connection via + * `.duplicate()` because ioredis forbids non-pubsub commands on a + * connection in subscribe mode. + */ + redis: Redis + /** Router service whose in-memory `LLM_ROUTER_CONFIG` cache we invalidate. */ + llmRouter: LlmRouterService + /** + * OTel gateway metric bundle. `null` when OTel is disabled — emit calls + * become no-ops. + */ + gatewayMetrics: GatewayMetrics | null + /** Value attached to the `service_instance_id` label on emitted metrics. */ + instanceId: string + /** Logger handle. Caller supplies a scoped logger so namespacing is theirs. */ + logger: ReturnType +} + +/** + * Per-call shape returned to the caller. Kept narrow so the caller can hold + * the subscriber handle for graceful shutdown or tests without leaking the + * internal emit closure. + */ +export interface ConfigSyncSubscriber { + /** Underlying ioredis subscriber connection. */ + subscriber: Redis +} + +/** + * Wires the cross-instance `configkv:invalidate` subscriber to the router's + * cache and OTel gateway metrics. + * + * Use when: + * - Booting a server replica that has a live `LlmRouterService` and needs + * to react to peer-instance config writes within the Pub/Sub propagation + * window (R16 / KTD-4, ≤5s under healthy Redis). + * + * Expects: + * - `redis` is the application's primary client. We `.duplicate()` it here + * because ioredis forbids non-pubsub commands on a subscribed connection. + * - `llmRouter` is already constructed. The caller owns its lifecycle. + * + * Returns: + * - `subscriber` — the dedicated ioredis subscriber connection, so the + * caller can `await subscriber.quit()` during graceful shutdown. + * + * Emits the following `airi.gen_ai.gateway.*` metrics: + * - `config_reload` (source = `pubsub`) once per accepted invalidation msg + * - `subscriber_state` with `state` = `connected` / `error` / `reconnecting` + * + * The router's in-memory cache reloads on either a pub/sub message OR the + * `configCacheTtlMs` fallback (default 5s); a silently-disconnected + * subscriber means the instance drifts inside that window. `subscriber_state` + * is the only direct signal for that drift. + */ +export function createConfigSyncSubscriber(opts: ConfigSyncSubscriberOptions): ConfigSyncSubscriber { + const subscriber = opts.redis.duplicate() + + function recordSubscriberState(state: 'connected' | 'error' | 'reconnecting') { + opts.gatewayMetrics?.subscriberState.add(1, { + state, + service_instance_id: opts.instanceId, + }) + } + + subscriber.on('message', (channel, message) => { + if (channel !== 'configkv:invalidate') + return + try { + const payload = JSON.parse(message) as { key?: unknown } + // Only LLM_ROUTER_CONFIG drives a router invalidation right now; + // future configKV entries with their own subscribers should route + // through additional channels, not this branch. + if (payload?.key !== 'LLM_ROUTER_CONFIG') + return + opts.llmRouter.invalidateConfig() + opts.gatewayMetrics?.configReload.add(1, { + source: 'pubsub', + service_instance_id: opts.instanceId, + }) + } + catch (err) { + opts.logger.withError(err).warn('Failed to parse configkv:invalidate payload') + } + }) + + subscriber.on('error', (err: Error) => { + opts.logger.withError(err).warn('configkv:invalidate subscriber connection error') + recordSubscriberState('error') + }) + + // ioredis emits `reconnecting` before each reconnect attempt; the + // subscription itself is restored automatically because `autoResubscribe` + // defaults to true. + subscriber.on('reconnecting', () => recordSubscriberState('reconnecting')) + + subscriber.subscribe('configkv:invalidate') + .then(() => recordSubscriberState('connected')) + .catch((err: unknown) => { + opts.logger.withError(err).warn('Failed to subscribe to configkv:invalidate channel') + recordSubscriberState('error') + }) + + return { subscriber } +} diff --git a/apps/server/src/services/llm-router/index.ts b/apps/server/src/services/llm-router/index.ts index 19a00d2e4..2eeef7253 100644 --- a/apps/server/src/services/llm-router/index.ts +++ b/apps/server/src/services/llm-router/index.ts @@ -1,6 +1,9 @@ export { createConfigLoader } from './config-loader' export type { ConfigLoader, ConfigLoaderOptions, ModelConfigSlice } from './config-loader' +export { createConfigSyncSubscriber } from './config-sync-subscriber' +export type { ConfigSyncSubscriber, ConfigSyncSubscriberOptions } from './config-sync-subscriber' + export { mapUpstreamError } from './error-mapping' export type { UpstreamErrorContext } from './error-mapping'