refactor(server): llm router config sync subscriber
This commit is contained in:
RainbowBird
@@ -113,10 +113,21 @@ stops asserting completion ahead of measurement.
|
||||
2026-05-15: env entry removed, all routes go through `llmRouter.route` /
|
||||
`routeTts` / `listTtsVoices`. The `LLM_ROUTER_MASTER_KEY` env var is
|
||||
now required (no graceful skip).
|
||||
- **Grafana dashboard JSON updates**: the new `airi.gen_ai.gateway.*`
|
||||
counters are emitted from `apps/server/src/otel/index.ts` but the
|
||||
Grafana dashboard JSON in `otel/grafana/dashboards/` does not yet have
|
||||
panels for them. Plan U8 panel + alert work is deferred to a follow-up.
|
||||
- ~~**Grafana dashboard JSON updates**: the new `airi.gen_ai.gateway.*`
|
||||
counters … do not yet have panels for them~~. Partially resolved
|
||||
2026-05-16: `otel/grafana/dashboards/build.ts` generates three router
|
||||
rows (Health / Trends / Admin Plane) covering the 7 gateway counters
|
||||
that have live producers: `fallback_count`, `upstream_errors`,
|
||||
`key_exhausted`, `same_status_exhaustion`, `decrypt_failures`,
|
||||
`config_reload`, and `subscriber_state` (producer added in the same
|
||||
PR — `app.ts` now emits `connected` / `error` / `reconnecting` from the
|
||||
`configkv:invalidate` subscriber). The remaining two counters
|
||||
(`config_write`, `config_invalid_hmac`) intentionally have no panels
|
||||
because their producer is the Plan U9 admin HTTP endpoint that has
|
||||
not shipped; they will rejoin Rows 6.5 / 6.7 alongside the U9 PR.
|
||||
Alert rules (key.exhausted > 0, fallback ratio > 30%, single-key
|
||||
error ratio > 80%) are still configured through Grafana UI, not
|
||||
build.ts — IaC-ifying them is a separate follow-up.
|
||||
- **knoway compose retention**: keep `/Users/luoling8192/Git/proj-airi/airi-railway/knoway/`
|
||||
+ the corresponding container entry in `airi-railway/docker-compose.yml`
|
||||
for **at least 14 days without a P1+ incident** before removing per plan R18.
|
||||
|
||||
@@ -2231,6 +2231,807 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"panel-60": {
|
||||
"kind": "Panel",
|
||||
"spec": {
|
||||
"data": {
|
||||
"kind": "QueryGroup",
|
||||
"spec": {
|
||||
"queries": [
|
||||
{
|
||||
"kind": "PanelQuery",
|
||||
"spec": {
|
||||
"hidden": false,
|
||||
"query": {
|
||||
"datasource": {
|
||||
"name": "grafanacloud-projairi-prom"
|
||||
},
|
||||
"group": "prometheus",
|
||||
"kind": "DataQuery",
|
||||
"spec": {
|
||||
"expr": "sum(increase(airi_gen_ai_gateway_key_exhausted_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[5m]))",
|
||||
"legendFormat": "events"
|
||||
},
|
||||
"version": "v0"
|
||||
},
|
||||
"refId": "A"
|
||||
}
|
||||
}
|
||||
],
|
||||
"queryOptions": {},
|
||||
"transformations": []
|
||||
}
|
||||
},
|
||||
"description": "Number of (model, upstream) pairs that ran out of usable keys within one user request over the last 5 minutes. Any non-zero value means at least one user request walked an entire upstream's key list without a 2xx. Sustained > 0 = a provider account is dead or every stored ciphertext is failing to decrypt — page on-call.",
|
||||
"id": 60,
|
||||
"links": [],
|
||||
"title": "Key Exhausted (5m)",
|
||||
"vizConfig": {
|
||||
"group": "stat",
|
||||
"kind": "VizConfig",
|
||||
"spec": {
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": 0
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 1
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "short",
|
||||
"noValue": "0"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"colorMode": "value",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto",
|
||||
"orientation": "auto",
|
||||
"percentChangeColorMode": "standard",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"showPercentChange": false,
|
||||
"textMode": "auto",
|
||||
"wideLayout": true
|
||||
}
|
||||
},
|
||||
"version": "13.0.0-23630096546"
|
||||
}
|
||||
}
|
||||
},
|
||||
"panel-61": {
|
||||
"kind": "Panel",
|
||||
"spec": {
|
||||
"data": {
|
||||
"kind": "QueryGroup",
|
||||
"spec": {
|
||||
"queries": [
|
||||
{
|
||||
"kind": "PanelQuery",
|
||||
"spec": {
|
||||
"hidden": false,
|
||||
"query": {
|
||||
"datasource": {
|
||||
"name": "grafanacloud-projairi-prom"
|
||||
},
|
||||
"group": "prometheus",
|
||||
"kind": "DataQuery",
|
||||
"spec": {
|
||||
"expr": "sum(increase(airi_gen_ai_gateway_decrypt_failures_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[5m]))",
|
||||
"legendFormat": "events"
|
||||
},
|
||||
"version": "v0"
|
||||
},
|
||||
"refId": "A"
|
||||
}
|
||||
}
|
||||
],
|
||||
"queryOptions": {},
|
||||
"transformations": []
|
||||
}
|
||||
},
|
||||
"description": "Envelope-crypto decrypt failures in the key rotator. A non-zero value here is security-relevant: either the master key was rotated without re-wrapping ciphertexts, or someone forged a config blob. Triages straight to the seed script + master-key audit trail.",
|
||||
"id": 61,
|
||||
"links": [],
|
||||
"title": "Decrypt Failures (5m)",
|
||||
"vizConfig": {
|
||||
"group": "stat",
|
||||
"kind": "VizConfig",
|
||||
"spec": {
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": 0
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 1
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "short",
|
||||
"noValue": "0"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"colorMode": "value",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto",
|
||||
"orientation": "auto",
|
||||
"percentChangeColorMode": "standard",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"showPercentChange": false,
|
||||
"textMode": "auto",
|
||||
"wideLayout": true
|
||||
}
|
||||
},
|
||||
"version": "13.0.0-23630096546"
|
||||
}
|
||||
}
|
||||
},
|
||||
"panel-62": {
|
||||
"kind": "Panel",
|
||||
"spec": {
|
||||
"data": {
|
||||
"kind": "QueryGroup",
|
||||
"spec": {
|
||||
"queries": [
|
||||
{
|
||||
"kind": "PanelQuery",
|
||||
"spec": {
|
||||
"hidden": false,
|
||||
"query": {
|
||||
"datasource": {
|
||||
"name": "grafanacloud-projairi-prom"
|
||||
},
|
||||
"group": "prometheus",
|
||||
"kind": "DataQuery",
|
||||
"spec": {
|
||||
"expr": "100 * sum(rate(airi_gen_ai_gateway_fallback_count_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[5m])) / clamp_min(sum(rate(gen_ai_client_operation_count_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[5m])), 1)",
|
||||
"legendFormat": "fallback %"
|
||||
},
|
||||
"version": "v0"
|
||||
},
|
||||
"refId": "A"
|
||||
}
|
||||
}
|
||||
],
|
||||
"queryOptions": {},
|
||||
"transformations": []
|
||||
}
|
||||
},
|
||||
"description": "Fallback attempts ÷ total LLM operations over the last 5m. Sustained > 30% means one provider is degraded and the router is silently masking it for users (but burning quota on the failing upstream). chat + tts share the operation counter so this is a cluster-wide health gauge.",
|
||||
"id": 62,
|
||||
"links": [],
|
||||
"title": "Fallback Ratio % (5m)",
|
||||
"vizConfig": {
|
||||
"group": "gauge",
|
||||
"kind": "VizConfig",
|
||||
"spec": {
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": 0
|
||||
},
|
||||
{
|
||||
"color": "yellow",
|
||||
"value": 10
|
||||
},
|
||||
{
|
||||
"color": "red",
|
||||
"value": 30
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "percent",
|
||||
"decimals": 1,
|
||||
"noValue": "0",
|
||||
"min": 0,
|
||||
"max": 100
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"minVizHeight": 75,
|
||||
"minVizWidth": 75,
|
||||
"orientation": "auto",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"showThresholdLabels": false,
|
||||
"showThresholdMarkers": true,
|
||||
"sizing": "auto"
|
||||
}
|
||||
},
|
||||
"version": "13.0.0-23630096546"
|
||||
}
|
||||
}
|
||||
},
|
||||
"panel-64": {
|
||||
"kind": "Panel",
|
||||
"spec": {
|
||||
"data": {
|
||||
"kind": "QueryGroup",
|
||||
"spec": {
|
||||
"queries": [
|
||||
{
|
||||
"kind": "PanelQuery",
|
||||
"spec": {
|
||||
"hidden": false,
|
||||
"query": {
|
||||
"datasource": {
|
||||
"name": "grafanacloud-projairi-prom"
|
||||
},
|
||||
"group": "prometheus",
|
||||
"kind": "DataQuery",
|
||||
"spec": {
|
||||
"expr": "sum by (provider, reason) (rate(airi_gen_ai_gateway_fallback_count_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))",
|
||||
"legendFormat": "{{provider}} · {{reason}}"
|
||||
},
|
||||
"version": "v0"
|
||||
},
|
||||
"refId": "A"
|
||||
}
|
||||
}
|
||||
],
|
||||
"queryOptions": {},
|
||||
"transformations": []
|
||||
}
|
||||
},
|
||||
"description": "Per-provider fallback events broken down by failure reason (HTTP status or `timeout`). A wide spread of reasons under one provider = transient upstream; a single reason dominating = systematic issue (e.g. 429 quota cap, 401 expired key).",
|
||||
"id": 64,
|
||||
"links": [],
|
||||
"title": "Fallback Count by Provider + Reason",
|
||||
"vizConfig": {
|
||||
"group": "timeseries",
|
||||
"kind": "VizConfig",
|
||||
"spec": {
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"barWidthFactor": 0.6,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 20,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "auto",
|
||||
"showValues": false,
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": 0
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "ops"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"annotations": {
|
||||
"clustering": -1,
|
||||
"multiLane": false
|
||||
},
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"lastNotNull",
|
||||
"max"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"placement": "right",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"hideZeros": false,
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
}
|
||||
},
|
||||
"version": "13.0.0-23630096546"
|
||||
}
|
||||
}
|
||||
},
|
||||
"panel-65": {
|
||||
"kind": "Panel",
|
||||
"spec": {
|
||||
"data": {
|
||||
"kind": "QueryGroup",
|
||||
"spec": {
|
||||
"queries": [
|
||||
{
|
||||
"kind": "PanelQuery",
|
||||
"spec": {
|
||||
"hidden": false,
|
||||
"query": {
|
||||
"datasource": {
|
||||
"name": "grafanacloud-projairi-prom"
|
||||
},
|
||||
"group": "prometheus",
|
||||
"kind": "DataQuery",
|
||||
"spec": {
|
||||
"expr": "sum by (provider, status_code) (rate(airi_gen_ai_gateway_upstream_errors_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))",
|
||||
"legendFormat": "{{provider}} · {{status_code}}"
|
||||
},
|
||||
"version": "v0"
|
||||
},
|
||||
"refId": "A"
|
||||
}
|
||||
}
|
||||
],
|
||||
"queryOptions": {},
|
||||
"transformations": []
|
||||
}
|
||||
},
|
||||
"description": "Per-upstream non-2xx response rate split by status code. Only counts attempts where the upstream actually answered (network timeouts and adapter aborts go to the fallback counter under `reason=timeout`). 401/403 = bad key; 429 = quota; 5xx = upstream outage.",
|
||||
"id": 65,
|
||||
"links": [],
|
||||
"title": "Upstream Errors by Status Code",
|
||||
"vizConfig": {
|
||||
"group": "timeseries",
|
||||
"kind": "VizConfig",
|
||||
"spec": {
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"barWidthFactor": 0.6,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 20,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "auto",
|
||||
"showValues": false,
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": 0
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "ops"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"annotations": {
|
||||
"clustering": -1,
|
||||
"multiLane": false
|
||||
},
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"lastNotNull",
|
||||
"max"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"placement": "right",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"hideZeros": false,
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
}
|
||||
},
|
||||
"version": "13.0.0-23630096546"
|
||||
}
|
||||
}
|
||||
},
|
||||
"panel-66": {
|
||||
"kind": "Panel",
|
||||
"spec": {
|
||||
"data": {
|
||||
"kind": "QueryGroup",
|
||||
"spec": {
|
||||
"queries": [
|
||||
{
|
||||
"kind": "PanelQuery",
|
||||
"spec": {
|
||||
"hidden": false,
|
||||
"query": {
|
||||
"datasource": {
|
||||
"name": "grafanacloud-projairi-prom"
|
||||
},
|
||||
"group": "prometheus",
|
||||
"kind": "DataQuery",
|
||||
"spec": {
|
||||
"expr": "sum by (provider, status_code) (rate(airi_gen_ai_gateway_same_status_exhaustion_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))",
|
||||
"legendFormat": "{{provider}} · {{status_code}}"
|
||||
},
|
||||
"version": "v0"
|
||||
},
|
||||
"refId": "A"
|
||||
}
|
||||
}
|
||||
],
|
||||
"queryOptions": {},
|
||||
"transformations": []
|
||||
}
|
||||
},
|
||||
"description": "Full-chain exhaustions where every attempt returned the same status code (or all timed out). A strong signal that ordinary key fallback cannot recover — points at an account-level cap or a shared backend brownout on the provider side. Each spike is one user request.",
|
||||
"id": 66,
|
||||
"links": [],
|
||||
"title": "Same-Status Exhaustion by Provider/Status",
|
||||
"vizConfig": {
|
||||
"group": "timeseries",
|
||||
"kind": "VizConfig",
|
||||
"spec": {
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"barWidthFactor": 0.6,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 20,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "auto",
|
||||
"showValues": false,
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": 0
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "ops"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"annotations": {
|
||||
"clustering": -1,
|
||||
"multiLane": false
|
||||
},
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"lastNotNull",
|
||||
"max"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"placement": "right",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"hideZeros": false,
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
}
|
||||
},
|
||||
"version": "13.0.0-23630096546"
|
||||
}
|
||||
}
|
||||
},
|
||||
"panel-67": {
|
||||
"kind": "Panel",
|
||||
"spec": {
|
||||
"data": {
|
||||
"kind": "QueryGroup",
|
||||
"spec": {
|
||||
"queries": [
|
||||
{
|
||||
"kind": "PanelQuery",
|
||||
"spec": {
|
||||
"hidden": false,
|
||||
"query": {
|
||||
"datasource": {
|
||||
"name": "grafanacloud-projairi-prom"
|
||||
},
|
||||
"group": "prometheus",
|
||||
"kind": "DataQuery",
|
||||
"spec": {
|
||||
"expr": "sum by (source) (rate(airi_gen_ai_gateway_config_reload_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))",
|
||||
"legendFormat": "{{source}}"
|
||||
},
|
||||
"version": "v0"
|
||||
},
|
||||
"refId": "A"
|
||||
}
|
||||
}
|
||||
],
|
||||
"queryOptions": {},
|
||||
"transformations": []
|
||||
}
|
||||
},
|
||||
"description": "Cache invalidations of `LLM_ROUTER_CONFIG` split by source (`pubsub` = Redis fan-out from a peer's write; `boot` = first load on startup). Steady low rate is normal. A sudden burst aligned with a deploy = expected. An unexplained burst = someone is writing to the config without auditing.",
|
||||
"id": 67,
|
||||
"links": [],
|
||||
"title": "Config Reload Events",
|
||||
"vizConfig": {
|
||||
"group": "timeseries",
|
||||
"kind": "VizConfig",
|
||||
"spec": {
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "palette-classic"
|
||||
},
|
||||
"custom": {
|
||||
"axisBorderShow": false,
|
||||
"axisCenteredZero": false,
|
||||
"axisColorMode": "text",
|
||||
"axisLabel": "",
|
||||
"axisPlacement": "auto",
|
||||
"barAlignment": 0,
|
||||
"barWidthFactor": 0.6,
|
||||
"drawStyle": "line",
|
||||
"fillOpacity": 20,
|
||||
"gradientMode": "none",
|
||||
"hideFrom": {
|
||||
"legend": false,
|
||||
"tooltip": false,
|
||||
"viz": false
|
||||
},
|
||||
"insertNulls": false,
|
||||
"lineInterpolation": "smooth",
|
||||
"lineWidth": 1,
|
||||
"pointSize": 5,
|
||||
"scaleDistribution": {
|
||||
"type": "linear"
|
||||
},
|
||||
"showPoints": "auto",
|
||||
"showValues": false,
|
||||
"spanNulls": false,
|
||||
"stacking": {
|
||||
"group": "A",
|
||||
"mode": "none"
|
||||
},
|
||||
"thresholdsStyle": {
|
||||
"mode": "off"
|
||||
}
|
||||
},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": 0
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "ops"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"annotations": {
|
||||
"clustering": -1,
|
||||
"multiLane": false
|
||||
},
|
||||
"legend": {
|
||||
"calcs": [
|
||||
"lastNotNull",
|
||||
"max"
|
||||
],
|
||||
"displayMode": "table",
|
||||
"placement": "right",
|
||||
"showLegend": true
|
||||
},
|
||||
"tooltip": {
|
||||
"hideZeros": false,
|
||||
"mode": "multi",
|
||||
"sort": "desc"
|
||||
}
|
||||
}
|
||||
},
|
||||
"version": "13.0.0-23630096546"
|
||||
}
|
||||
}
|
||||
},
|
||||
"panel-69": {
|
||||
"kind": "Panel",
|
||||
"spec": {
|
||||
"data": {
|
||||
"kind": "QueryGroup",
|
||||
"spec": {
|
||||
"queries": [
|
||||
{
|
||||
"kind": "PanelQuery",
|
||||
"spec": {
|
||||
"hidden": false,
|
||||
"query": {
|
||||
"datasource": {
|
||||
"name": "grafanacloud-projairi-prom"
|
||||
},
|
||||
"group": "prometheus",
|
||||
"kind": "DataQuery",
|
||||
"spec": {
|
||||
"expr": "sum by (state, service_instance_id) (increase(airi_gen_ai_gateway_subscriber_state_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[5m]))",
|
||||
"legendFormat": "{{state}} ({{service_instance_id}})"
|
||||
},
|
||||
"version": "v0"
|
||||
},
|
||||
"refId": "A"
|
||||
}
|
||||
}
|
||||
],
|
||||
"queryOptions": {},
|
||||
"transformations": []
|
||||
}
|
||||
},
|
||||
"description": "Lifecycle events for the cross-instance Redis Pub/Sub subscriber (`connected` on initial subscribe, `error` on connection error or subscribe failure, `reconnecting` while ioredis is re-establishing the connection). One `connected` per replica per deploy is normal. Sustained `error` or `reconnecting` without a matching `connected` means an instance is desynced from `configkv:invalidate` — its in-memory router config will drift up to the configCacheTtlMs fallback window.",
|
||||
"id": 69,
|
||||
"links": [],
|
||||
"title": "Subscriber State Events (5m)",
|
||||
"vizConfig": {
|
||||
"group": "stat",
|
||||
"kind": "VizConfig",
|
||||
"spec": {
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": 0
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "short",
|
||||
"noValue": "0"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"colorMode": "value",
|
||||
"graphMode": "area",
|
||||
"justifyMode": "auto",
|
||||
"orientation": "auto",
|
||||
"percentChangeColorMode": "standard",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"showPercentChange": false,
|
||||
"textMode": "auto",
|
||||
"wideLayout": true
|
||||
}
|
||||
},
|
||||
"version": "13.0.0-23630096546"
|
||||
}
|
||||
}
|
||||
},
|
||||
"panel-50": {
|
||||
"kind": "Panel",
|
||||
"spec": {
|
||||
@@ -3200,6 +4001,152 @@
|
||||
"title": "Business"
|
||||
}
|
||||
},
|
||||
{
|
||||
"kind": "RowsLayoutRow",
|
||||
"spec": {
|
||||
"collapse": false,
|
||||
"layout": {
|
||||
"kind": "GridLayout",
|
||||
"spec": {
|
||||
"items": [
|
||||
{
|
||||
"kind": "GridLayoutItem",
|
||||
"spec": {
|
||||
"element": {
|
||||
"kind": "ElementReference",
|
||||
"name": "panel-60"
|
||||
},
|
||||
"height": 5,
|
||||
"width": 8,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
}
|
||||
},
|
||||
{
|
||||
"kind": "GridLayoutItem",
|
||||
"spec": {
|
||||
"element": {
|
||||
"kind": "ElementReference",
|
||||
"name": "panel-61"
|
||||
},
|
||||
"height": 5,
|
||||
"width": 8,
|
||||
"x": 8,
|
||||
"y": 0
|
||||
}
|
||||
},
|
||||
{
|
||||
"kind": "GridLayoutItem",
|
||||
"spec": {
|
||||
"element": {
|
||||
"kind": "ElementReference",
|
||||
"name": "panel-62"
|
||||
},
|
||||
"height": 5,
|
||||
"width": 8,
|
||||
"x": 16,
|
||||
"y": 0
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"title": "LLM Router Health"
|
||||
}
|
||||
},
|
||||
{
|
||||
"kind": "RowsLayoutRow",
|
||||
"spec": {
|
||||
"collapse": false,
|
||||
"layout": {
|
||||
"kind": "GridLayout",
|
||||
"spec": {
|
||||
"items": [
|
||||
{
|
||||
"kind": "GridLayoutItem",
|
||||
"spec": {
|
||||
"element": {
|
||||
"kind": "ElementReference",
|
||||
"name": "panel-64"
|
||||
},
|
||||
"height": 8,
|
||||
"width": 6,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
}
|
||||
},
|
||||
{
|
||||
"kind": "GridLayoutItem",
|
||||
"spec": {
|
||||
"element": {
|
||||
"kind": "ElementReference",
|
||||
"name": "panel-65"
|
||||
},
|
||||
"height": 8,
|
||||
"width": 6,
|
||||
"x": 6,
|
||||
"y": 0
|
||||
}
|
||||
},
|
||||
{
|
||||
"kind": "GridLayoutItem",
|
||||
"spec": {
|
||||
"element": {
|
||||
"kind": "ElementReference",
|
||||
"name": "panel-66"
|
||||
},
|
||||
"height": 8,
|
||||
"width": 6,
|
||||
"x": 12,
|
||||
"y": 0
|
||||
}
|
||||
},
|
||||
{
|
||||
"kind": "GridLayoutItem",
|
||||
"spec": {
|
||||
"element": {
|
||||
"kind": "ElementReference",
|
||||
"name": "panel-67"
|
||||
},
|
||||
"height": 8,
|
||||
"width": 6,
|
||||
"x": 18,
|
||||
"y": 0
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"title": "LLM Router Trends"
|
||||
}
|
||||
},
|
||||
{
|
||||
"kind": "RowsLayoutRow",
|
||||
"spec": {
|
||||
"collapse": true,
|
||||
"layout": {
|
||||
"kind": "GridLayout",
|
||||
"spec": {
|
||||
"items": [
|
||||
{
|
||||
"kind": "GridLayoutItem",
|
||||
"spec": {
|
||||
"element": {
|
||||
"kind": "ElementReference",
|
||||
"name": "panel-69"
|
||||
},
|
||||
"height": 5,
|
||||
"width": 24,
|
||||
"x": 0,
|
||||
"y": 0
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"title": "Gateway Admin Plane"
|
||||
}
|
||||
},
|
||||
{
|
||||
"kind": "RowsLayoutRow",
|
||||
"spec": {
|
||||
|
||||
@@ -729,6 +729,118 @@ elements['panel-32'] = piePanel(
|
||||
{ noValue: '—' },
|
||||
)
|
||||
|
||||
// Rows 6.5 / 6.6 / 6.7: LLM/TTS in-process router (KTD-5 / KTD-6).
|
||||
// These counters are emitted from `apps/server/src/services/llm-router/router.ts`
|
||||
// for every chat AND tts dispatch attempt. fallback_count / upstream_errors
|
||||
// track per-attempt failures inside one user request; key_exhausted /
|
||||
// same_status_exhaustion fire only on full chain exhaustion. config_* counters
|
||||
// belong to the admin-plane subscriber and the seed/admin writers — separated
|
||||
// into a collapsed row because operators only look at them during config
|
||||
// rollout or post-incident.
|
||||
//
|
||||
// Counter → prom name mapping (OTel dot → underscore + `_total` for counters):
|
||||
// airi.gen_ai.gateway.fallback.count → airi_gen_ai_gateway_fallback_count_total
|
||||
// airi.gen_ai.gateway.upstream.errors → airi_gen_ai_gateway_upstream_errors_total
|
||||
// airi.gen_ai.gateway.key.exhausted → airi_gen_ai_gateway_key_exhausted_total
|
||||
// airi.gen_ai.gateway.same_status_exhaustion → airi_gen_ai_gateway_same_status_exhaustion_total
|
||||
// airi.gen_ai.gateway.config.reload → airi_gen_ai_gateway_config_reload_total
|
||||
// airi.gen_ai.gateway.decrypt.failures → airi_gen_ai_gateway_decrypt_failures_total
|
||||
// airi.gen_ai.gateway.subscriber_state → airi_gen_ai_gateway_subscriber_state_total
|
||||
// airi.gen_ai.gateway.config.write → airi_gen_ai_gateway_config_write_total
|
||||
// airi.gen_ai.gateway.config.invalid_hmac → airi_gen_ai_gateway_config_invalid_hmac_total
|
||||
elements['panel-60'] = statPanel(
|
||||
60,
|
||||
'Key Exhausted (5m)',
|
||||
'Number of (model, upstream) pairs that ran out of usable keys within one user request over the last 5 minutes. Any non-zero value means at least one user request walked an entire upstream\'s key list without a 2xx. Sustained > 0 = a provider account is dead or every stored ciphertext is failing to decrypt — page on-call.',
|
||||
[query(`sum(increase(airi_gen_ai_gateway_key_exhausted_total{${SERVICE_FILTER}}[5m]))`, 'events')],
|
||||
{ unit: 'short', steps: [{ color: 'green', value: 0 }, { color: 'red', value: 1 }], noValue: '0' },
|
||||
)
|
||||
|
||||
elements['panel-61'] = statPanel(
|
||||
61,
|
||||
'Decrypt Failures (5m)',
|
||||
'Envelope-crypto decrypt failures in the key rotator. A non-zero value here is security-relevant: either the master key was rotated without re-wrapping ciphertexts, or someone forged a config blob. Triages straight to the seed script + master-key audit trail.',
|
||||
[query(`sum(increase(airi_gen_ai_gateway_decrypt_failures_total{${SERVICE_FILTER}}[5m]))`, 'events')],
|
||||
{ unit: 'short', steps: [{ color: 'green', value: 0 }, { color: 'red', value: 1 }], noValue: '0' },
|
||||
)
|
||||
|
||||
elements['panel-62'] = gaugePanel(
|
||||
62,
|
||||
'Fallback Ratio % (5m)',
|
||||
'Fallback attempts ÷ total LLM operations over the last 5m. Sustained > 30% means one provider is degraded and the router is silently masking it for users (but burning quota on the failing upstream). chat + tts share the operation counter so this is a cluster-wide health gauge.',
|
||||
[query(
|
||||
`100 * sum(rate(airi_gen_ai_gateway_fallback_count_total{${SERVICE_FILTER}}[5m])) / clamp_min(sum(rate(gen_ai_client_operation_count_total{${SERVICE_FILTER}}[5m])), 1)`,
|
||||
'fallback %',
|
||||
)],
|
||||
{ steps: [{ color: 'green', value: 0 }, { color: 'yellow', value: 10 }, { color: 'red', value: 30 }], max: 100, decimals: 1, noValue: '0' },
|
||||
)
|
||||
|
||||
// panel-63 (Invalid HMAC Writes) intentionally not built: the producer
|
||||
// (`config_invalid_hmac` counter) lives in the Plan U9 admin HTTP endpoint
|
||||
// that hasn't shipped yet. Adding the panel now would surface a permanent
|
||||
// zero that misleads readers into thinking "no attack" when it actually
|
||||
// means "no producer". Re-add together with the U9 endpoint PR.
|
||||
|
||||
elements['panel-64'] = timeseriesPanel(
|
||||
64,
|
||||
'Fallback Count by Provider + Reason',
|
||||
'Per-provider fallback events broken down by failure reason (HTTP status or `timeout`). A wide spread of reasons under one provider = transient upstream; a single reason dominating = systematic issue (e.g. 429 quota cap, 401 expired key).',
|
||||
[query(
|
||||
`sum by (provider, reason) (rate(airi_gen_ai_gateway_fallback_count_total{${SERVICE_FILTER}}[$__rate_interval]))`,
|
||||
'{{provider}} · {{reason}}',
|
||||
)],
|
||||
{ unit: 'ops' },
|
||||
)
|
||||
|
||||
elements['panel-65'] = timeseriesPanel(
|
||||
65,
|
||||
'Upstream Errors by Status Code',
|
||||
'Per-upstream non-2xx response rate split by status code. Only counts attempts where the upstream actually answered (network timeouts and adapter aborts go to the fallback counter under `reason=timeout`). 401/403 = bad key; 429 = quota; 5xx = upstream outage.',
|
||||
[query(
|
||||
`sum by (provider, status_code) (rate(airi_gen_ai_gateway_upstream_errors_total{${SERVICE_FILTER}}[$__rate_interval]))`,
|
||||
'{{provider}} · {{status_code}}',
|
||||
)],
|
||||
{ unit: 'ops' },
|
||||
)
|
||||
|
||||
elements['panel-66'] = timeseriesPanel(
|
||||
66,
|
||||
'Same-Status Exhaustion by Provider/Status',
|
||||
'Full-chain exhaustions where every attempt returned the same status code (or all timed out). A strong signal that ordinary key fallback cannot recover — points at an account-level cap or a shared backend brownout on the provider side. Each spike is one user request.',
|
||||
[query(
|
||||
`sum by (provider, status_code) (rate(airi_gen_ai_gateway_same_status_exhaustion_total{${SERVICE_FILTER}}[$__rate_interval]))`,
|
||||
'{{provider}} · {{status_code}}',
|
||||
)],
|
||||
{ unit: 'ops' },
|
||||
)
|
||||
|
||||
elements['panel-67'] = timeseriesPanel(
|
||||
67,
|
||||
'Config Reload Events',
|
||||
'Cache invalidations of `LLM_ROUTER_CONFIG` split by source (`pubsub` = Redis fan-out from a peer\'s write; `boot` = first load on startup). Steady low rate is normal. A sudden burst aligned with a deploy = expected. An unexplained burst = someone is writing to the config without auditing.',
|
||||
[query(
|
||||
`sum by (source) (rate(airi_gen_ai_gateway_config_reload_total{${SERVICE_FILTER}}[$__rate_interval]))`,
|
||||
'{{source}}',
|
||||
)],
|
||||
{ unit: 'ops' },
|
||||
)
|
||||
|
||||
// panel-68 (Config Writes) intentionally not built: same reasoning as panel-63
|
||||
// — the `config_write` counter has no producer until the Plan U9 admin
|
||||
// endpoint ships. Re-add together with the U9 endpoint PR so the `by result`
|
||||
// label set (success / etag_mismatch / validation_failed) is meaningful.
|
||||
|
||||
elements['panel-69'] = statPanel(
|
||||
69,
|
||||
'Subscriber State Events (5m)',
|
||||
'Lifecycle events for the cross-instance Redis Pub/Sub subscriber (`connected` on initial subscribe, `error` on connection error or subscribe failure, `reconnecting` while ioredis is re-establishing the connection). One `connected` per replica per deploy is normal. Sustained `error` or `reconnecting` without a matching `connected` means an instance is desynced from `configkv:invalidate` — its in-memory router config will drift up to the configCacheTtlMs fallback window.',
|
||||
[query(
|
||||
`sum by (state, service_instance_id) (increase(airi_gen_ai_gateway_subscriber_state_total{${SERVICE_FILTER}}[5m]))`,
|
||||
'{{state}} ({{service_instance_id}})',
|
||||
)],
|
||||
{ unit: 'short', noValue: '0' },
|
||||
)
|
||||
|
||||
// Row 7: Infrastructure — process / DB health (collapsed by default)
|
||||
elements['panel-50'] = statPanel(
|
||||
50,
|
||||
@@ -867,6 +979,32 @@ const rows = [
|
||||
item('panel-31', 8, 0, 8, 7),
|
||||
item('panel-32', 16, 0, 8, 7),
|
||||
]),
|
||||
// Row 6.5: LLM router health — 3 stat/gauge × 8 wide × 5 high.
|
||||
// These are the "wake someone up" indicators (key exhausted, decrypt
|
||||
// failures, fallback storms). Always visible, no collapse, because they
|
||||
// answer "is the in-process router healthy right now". The Invalid HMAC
|
||||
// panel will join this row when the Plan U9 admin endpoint lands.
|
||||
row('LLM Router Health', [
|
||||
item('panel-60', 0, 0, 8, 5),
|
||||
item('panel-61', 8, 0, 8, 5),
|
||||
item('panel-62', 16, 0, 8, 5),
|
||||
]),
|
||||
// Row 6.6: LLM router trends — 4 timeseries × 6 wide × 8 high.
|
||||
// Same dimensions as Row 6.5 but answers "how is it changing / which
|
||||
// provider is contributing". Wider time context for triage.
|
||||
row('LLM Router Trends', [
|
||||
item('panel-64', 0, 0, 6, 8),
|
||||
item('panel-65', 6, 0, 6, 8),
|
||||
item('panel-66', 12, 0, 6, 8),
|
||||
item('panel-67', 18, 0, 6, 8),
|
||||
]),
|
||||
// Row 6.7: Gateway admin plane — 1 stat × 24 wide × 5 high, collapsed by
|
||||
// default. Tracks per-instance subscriber lifecycle for cross-instance
|
||||
// config consistency triage. Config Writes panel rejoins this row when the
|
||||
// Plan U9 admin endpoint lands.
|
||||
row('Gateway Admin Plane', [
|
||||
item('panel-69', 0, 0, 24, 5),
|
||||
], { collapse: true }),
|
||||
// Row 7: 1 stat + 3 by-instance timeseries × 6 wide × 6 high (collapsed by
|
||||
// default — only relevant when triaging. By-instance breakdowns catch
|
||||
// single-replica issues that cluster aggregates would average away.)
|
||||
|
||||
+9
-26
@@ -63,7 +63,7 @@ import { createConfigKVService } from './services/config-kv'
|
||||
import { createEmailService } from './services/email'
|
||||
import { createFluxService } from './services/flux'
|
||||
import { createFluxTransactionService } from './services/flux-transaction'
|
||||
import { createLlmRouterService } from './services/llm-router'
|
||||
import { createConfigSyncSubscriber, createLlmRouterService } from './services/llm-router'
|
||||
import { createPostHogClient } from './services/posthog'
|
||||
import { createProviderService } from './services/providers'
|
||||
import { createRequestLogService } from './services/request-log'
|
||||
@@ -164,31 +164,14 @@ export async function buildApp(deps: AppDeps) {
|
||||
return chatWsSetup(session.user.id)
|
||||
}))
|
||||
|
||||
// Subscribe to the cross-instance config invalidation channel so admin
|
||||
// writes to LLM_ROUTER_CONFIG propagate within ≤5s across the cluster
|
||||
// (R16 / KTD-4). Falls back to the in-memory cache TTL on missed messages.
|
||||
// Uses a dedicated ioredis subscriber connection (subscribe mode requires
|
||||
// a separate connection per ioredis docs).
|
||||
const configSub = deps.redis.duplicate()
|
||||
configSub.on('message', (channel, message) => {
|
||||
if (channel !== 'configkv:invalidate')
|
||||
return
|
||||
try {
|
||||
const payload = JSON.parse(message) as { key?: unknown }
|
||||
if (payload?.key !== 'LLM_ROUTER_CONFIG')
|
||||
return
|
||||
deps.llmRouter.invalidateConfig()
|
||||
deps.otel?.gateway?.configReload.add(1, {
|
||||
source: 'pubsub',
|
||||
service_instance_id: deps.env.OTEL_SERVICE_NAME,
|
||||
})
|
||||
}
|
||||
catch (err) {
|
||||
logger.withError(err).warn('Failed to parse configkv:invalidate payload')
|
||||
}
|
||||
})
|
||||
configSub.subscribe('configkv:invalidate').catch((err: unknown) => {
|
||||
logger.withError(err).warn('Failed to subscribe to configkv:invalidate channel')
|
||||
// Cross-instance config invalidation. The subscriber owns its own
|
||||
// connection + lifecycle metrics; see services/llm-router/config-sync-subscriber.ts.
|
||||
createConfigSyncSubscriber({
|
||||
redis: deps.redis,
|
||||
llmRouter: deps.llmRouter,
|
||||
gatewayMetrics: deps.otel?.gateway ?? null,
|
||||
instanceId: deps.env.OTEL_SERVICE_NAME,
|
||||
logger: useLogger('config-sync').useGlobalConfig(),
|
||||
})
|
||||
|
||||
const builtApp = app
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
import type { useLogger } from '@guiiai/logg'
|
||||
import type Redis from 'ioredis'
|
||||
|
||||
import type { GatewayMetrics } from '../../otel'
|
||||
import type { LlmRouterService } from './router'
|
||||
|
||||
/**
|
||||
* Dependencies needed to wire the cross-instance config invalidation
|
||||
* subscriber.
|
||||
*/
|
||||
export interface ConfigSyncSubscriberOptions {
|
||||
/**
|
||||
* Primary Redis client. The subscriber takes its own connection via
|
||||
* `.duplicate()` because ioredis forbids non-pubsub commands on a
|
||||
* connection in subscribe mode.
|
||||
*/
|
||||
redis: Redis
|
||||
/** Router service whose in-memory `LLM_ROUTER_CONFIG` cache we invalidate. */
|
||||
llmRouter: LlmRouterService
|
||||
/**
|
||||
* OTel gateway metric bundle. `null` when OTel is disabled — emit calls
|
||||
* become no-ops.
|
||||
*/
|
||||
gatewayMetrics: GatewayMetrics | null
|
||||
/** Value attached to the `service_instance_id` label on emitted metrics. */
|
||||
instanceId: string
|
||||
/** Logger handle. Caller supplies a scoped logger so namespacing is theirs. */
|
||||
logger: ReturnType<typeof useLogger>
|
||||
}
|
||||
|
||||
/**
|
||||
* Per-call shape returned to the caller. Kept narrow so the caller can hold
|
||||
* the subscriber handle for graceful shutdown or tests without leaking the
|
||||
* internal emit closure.
|
||||
*/
|
||||
export interface ConfigSyncSubscriber {
|
||||
/** Underlying ioredis subscriber connection. */
|
||||
subscriber: Redis
|
||||
}
|
||||
|
||||
/**
|
||||
* Wires the cross-instance `configkv:invalidate` subscriber to the router's
|
||||
* cache and OTel gateway metrics.
|
||||
*
|
||||
* Use when:
|
||||
* - Booting a server replica that has a live `LlmRouterService` and needs
|
||||
* to react to peer-instance config writes within the Pub/Sub propagation
|
||||
* window (R16 / KTD-4, ≤5s under healthy Redis).
|
||||
*
|
||||
* Expects:
|
||||
* - `redis` is the application's primary client. We `.duplicate()` it here
|
||||
* because ioredis forbids non-pubsub commands on a subscribed connection.
|
||||
* - `llmRouter` is already constructed. The caller owns its lifecycle.
|
||||
*
|
||||
* Returns:
|
||||
* - `subscriber` — the dedicated ioredis subscriber connection, so the
|
||||
* caller can `await subscriber.quit()` during graceful shutdown.
|
||||
*
|
||||
* Emits the following `airi.gen_ai.gateway.*` metrics:
|
||||
* - `config_reload` (source = `pubsub`) once per accepted invalidation msg
|
||||
* - `subscriber_state` with `state` = `connected` / `error` / `reconnecting`
|
||||
*
|
||||
* The router's in-memory cache reloads on either a pub/sub message OR the
|
||||
* `configCacheTtlMs` fallback (default 5s); a silently-disconnected
|
||||
* subscriber means the instance drifts inside that window. `subscriber_state`
|
||||
* is the only direct signal for that drift.
|
||||
*/
|
||||
export function createConfigSyncSubscriber(opts: ConfigSyncSubscriberOptions): ConfigSyncSubscriber {
|
||||
const subscriber = opts.redis.duplicate()
|
||||
|
||||
function recordSubscriberState(state: 'connected' | 'error' | 'reconnecting') {
|
||||
opts.gatewayMetrics?.subscriberState.add(1, {
|
||||
state,
|
||||
service_instance_id: opts.instanceId,
|
||||
})
|
||||
}
|
||||
|
||||
subscriber.on('message', (channel, message) => {
|
||||
if (channel !== 'configkv:invalidate')
|
||||
return
|
||||
try {
|
||||
const payload = JSON.parse(message) as { key?: unknown }
|
||||
// Only LLM_ROUTER_CONFIG drives a router invalidation right now;
|
||||
// future configKV entries with their own subscribers should route
|
||||
// through additional channels, not this branch.
|
||||
if (payload?.key !== 'LLM_ROUTER_CONFIG')
|
||||
return
|
||||
opts.llmRouter.invalidateConfig()
|
||||
opts.gatewayMetrics?.configReload.add(1, {
|
||||
source: 'pubsub',
|
||||
service_instance_id: opts.instanceId,
|
||||
})
|
||||
}
|
||||
catch (err) {
|
||||
opts.logger.withError(err).warn('Failed to parse configkv:invalidate payload')
|
||||
}
|
||||
})
|
||||
|
||||
subscriber.on('error', (err: Error) => {
|
||||
opts.logger.withError(err).warn('configkv:invalidate subscriber connection error')
|
||||
recordSubscriberState('error')
|
||||
})
|
||||
|
||||
// ioredis emits `reconnecting` before each reconnect attempt; the
|
||||
// subscription itself is restored automatically because `autoResubscribe`
|
||||
// defaults to true.
|
||||
subscriber.on('reconnecting', () => recordSubscriberState('reconnecting'))
|
||||
|
||||
subscriber.subscribe('configkv:invalidate')
|
||||
.then(() => recordSubscriberState('connected'))
|
||||
.catch((err: unknown) => {
|
||||
opts.logger.withError(err).warn('Failed to subscribe to configkv:invalidate channel')
|
||||
recordSubscriberState('error')
|
||||
})
|
||||
|
||||
return { subscriber }
|
||||
}
|
||||
@@ -1,6 +1,9 @@
|
||||
export { createConfigLoader } from './config-loader'
|
||||
export type { ConfigLoader, ConfigLoaderOptions, ModelConfigSlice } from './config-loader'
|
||||
|
||||
export { createConfigSyncSubscriber } from './config-sync-subscriber'
|
||||
export type { ConfigSyncSubscriber, ConfigSyncSubscriberOptions } from './config-sync-subscriber'
|
||||
|
||||
export { mapUpstreamError } from './error-mapping'
|
||||
export type { UpstreamErrorContext } from './error-mapping'
|
||||
|
||||
|
||||
Reference in New Issue
Block a user