feat(server/otel): enhance observability with new metrics for email, rate limiting, and billing
- Added new metrics for email service including send, failures, and duration tracking. - Introduced rate limit metrics to monitor blocked requests and improve abuse detection. - Enhanced billing metrics to track credited and unbilled flux, as well as TTS character processing. - Updated OpenAI and Stripe routes to utilize new metrics for better revenue tracking and rate limiting. - Implemented a smoke test for OpenTelemetry metrics registration to ensure visibility at startup.
This commit is contained in:
RainbowBird
@@ -13,11 +13,26 @@
|
||||
* the CJS module, so `require-in-the-middle` hooks still intercept it.
|
||||
*/
|
||||
|
||||
import { env } from 'node:process'
|
||||
|
||||
import { registerInstrumentations } from '@opentelemetry/instrumentation'
|
||||
import { HttpInstrumentation } from '@opentelemetry/instrumentation-http'
|
||||
import { IORedisInstrumentation } from '@opentelemetry/instrumentation-ioredis'
|
||||
import { PgInstrumentation } from '@opentelemetry/instrumentation-pg'
|
||||
|
||||
// NOTICE:
|
||||
// instrumentation-http >=0.215 defaults to OLD semconv (http.server.duration in
|
||||
// ms). Our Grafana dashboards / alerts only query the STABLE name
|
||||
// (http.server.request.duration in seconds), and grep across the repo confirms
|
||||
// no OLD-name consumer exists, so we go straight to STABLE-only — no `http/dup`
|
||||
// transition phase, no doubled cardinality.
|
||||
// Source: node_modules/.pnpm/@opentelemetry+instrumentation-http@0.215.0/.../build/src/http.js L25-72
|
||||
// MUST run before `new HttpInstrumentation(...)` below — its constructor reads
|
||||
// the env var once and caches the result.
|
||||
// Removal condition: ops sets OTEL_SEMCONV_STABILITY_OPT_IN explicitly in the
|
||||
// deployment platform (Railway env), then this preload default can be deleted.
|
||||
env.OTEL_SEMCONV_STABILITY_OPT_IN ??= 'http'
|
||||
|
||||
registerInstrumentations({
|
||||
instrumentations: [
|
||||
new HttpInstrumentation({
|
||||
|
||||
@@ -1764,6 +1764,226 @@
|
||||
"version": "13.0.0-23630096546"
|
||||
}
|
||||
}
|
||||
},
|
||||
"panel-34": {
|
||||
"kind": "Panel",
|
||||
"spec": {
|
||||
"data": { "kind": "QueryGroup", "spec": { "queries": [{ "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "100 * sum(rate(auth_failures_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval])) / clamp_min(sum(rate(auth_attempts_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval])), 1)", "legendFormat": "fail %" }, "version": "v0" }, "refId": "A" } }], "queryOptions": {}, "transformations": [] } },
|
||||
"description": "auth_failures / auth_attempts; high values mean credential stuffing or upstream OAuth breakage.",
|
||||
"id": 34,
|
||||
"links": [],
|
||||
"title": "Auth Failure Rate %",
|
||||
"vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "yellow", "value": 10 }, { "color": "red", "value": 30 }] }, "unit": "percent" }, "overrides": [] }, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true } }, "version": "13.0.0-23630096546" }
|
||||
}
|
||||
},
|
||||
"panel-50": {
|
||||
"kind": "Panel",
|
||||
"spec": {
|
||||
"data": { "kind": "QueryGroup", "spec": { "queries": [{ "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(ws_connections_active{service_name=~\"$service\", deployment_environment=~\"$env\"})", "legendFormat": "active" }, "version": "v0" }, "refId": "A" } }], "queryOptions": {}, "transformations": [] } },
|
||||
"description": "Sum of active WS connections across all instances.",
|
||||
"id": 50,
|
||||
"links": [],
|
||||
"title": "WS Connections Active",
|
||||
"vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "yellow", "value": 1000 }, { "color": "red", "value": 10000 }] }, "unit": "short" }, "overrides": [] }, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true } }, "version": "13.0.0-23630096546" }
|
||||
}
|
||||
},
|
||||
"panel-51": {
|
||||
"kind": "Panel",
|
||||
"spec": {
|
||||
"data": { "kind": "QueryGroup", "spec": { "queries": [{ "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(rate(ws_messages_sent_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))", "legendFormat": "sent/s" }, "version": "v0" }, "refId": "A" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(rate(ws_messages_received_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))", "legendFormat": "received/s" }, "version": "v0" }, "refId": "B" } }], "queryOptions": {}, "transformations": [] } },
|
||||
"description": "WebSocket throughput in both directions.",
|
||||
"id": 51,
|
||||
"links": [],
|
||||
"title": "WS Messages I/O",
|
||||
"vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "red", "value": 80 }] }, "unit": "ops" }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.0.0-23630096546" }
|
||||
}
|
||||
},
|
||||
"panel-52": {
|
||||
"kind": "Panel",
|
||||
"spec": {
|
||||
"data": { "kind": "QueryGroup", "spec": { "queries": [{ "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(rate(chat_messages_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))", "legendFormat": "chat msgs/s" }, "version": "v0" }, "refId": "A" } }], "queryOptions": {}, "transformations": [] } },
|
||||
"description": "Chat messages persisted to chat sync (chats.pushMessages).",
|
||||
"id": 52,
|
||||
"links": [],
|
||||
"title": "Chat Messages Rate",
|
||||
"vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "red", "value": 80 }] }, "unit": "ops" }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.0.0-23630096546" }
|
||||
}
|
||||
},
|
||||
"panel-53": {
|
||||
"kind": "Panel",
|
||||
"spec": {
|
||||
"data": { "kind": "QueryGroup", "spec": { "queries": [{ "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(rate(character_created_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))", "legendFormat": "created/s" }, "version": "v0" }, "refId": "A" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(rate(character_deleted_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))", "legendFormat": "deleted/s" }, "version": "v0" }, "refId": "B" } }], "queryOptions": {}, "transformations": [] } },
|
||||
"description": "",
|
||||
"id": 53,
|
||||
"links": [],
|
||||
"title": "Character Lifecycle",
|
||||
"vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "red", "value": 80 }] }, "unit": "ops" }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.0.0-23630096546" }
|
||||
}
|
||||
},
|
||||
"panel-54": {
|
||||
"kind": "Panel",
|
||||
"spec": {
|
||||
"data": { "kind": "QueryGroup", "spec": { "queries": [{ "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (action) (rate(character_engagement_total{service_name=~\"$service\", deployment_environment=~\"$env\", action!=\"\"}[$__rate_interval]))", "legendFormat": "{{action}}" }, "version": "v0" }, "refId": "A" } }], "queryOptions": {}, "transformations": [] } },
|
||||
"description": "Like / unlike / bookmark / unbookmark actions on characters.",
|
||||
"id": 54,
|
||||
"links": [],
|
||||
"title": "Character Engagement",
|
||||
"vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "red", "value": 80 }] }, "unit": "ops" }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.0.0-23630096546" }
|
||||
}
|
||||
},
|
||||
"panel-55": {
|
||||
"kind": "Panel",
|
||||
"spec": {
|
||||
"data": { "kind": "QueryGroup", "spec": { "queries": [{ "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (event_type) (rate(stripe_events_total{service_name=~\"$service\", deployment_environment=~\"$env\", event_type!=\"\"}[$__rate_interval]))", "legendFormat": "{{event_type}}" }, "version": "v0" }, "refId": "A" } }], "queryOptions": {}, "transformations": [] } },
|
||||
"description": "All Stripe webhook events grouped by event.type.",
|
||||
"id": 55,
|
||||
"links": [],
|
||||
"title": "Stripe Events by Type",
|
||||
"vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "red", "value": 80 }] }, "unit": "ops" }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.0.0-23630096546" }
|
||||
}
|
||||
},
|
||||
"panel-56": {
|
||||
"kind": "Panel",
|
||||
"spec": {
|
||||
"data": { "kind": "QueryGroup", "spec": { "queries": [{ "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (event_type) (rate(stripe_subscription_event_total{service_name=~\"$service\", deployment_environment=~\"$env\", event_type!=\"\"}[$__rate_interval]))", "legendFormat": "{{event_type}}" }, "version": "v0" }, "refId": "A" } }], "queryOptions": {}, "transformations": [] } },
|
||||
"description": "Subscription lifecycle events (created/updated/deleted).",
|
||||
"id": 56,
|
||||
"links": [],
|
||||
"title": "Subscription Events by Type",
|
||||
"vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "red", "value": 80 }] }, "unit": "ops" }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.0.0-23630096546" }
|
||||
}
|
||||
},
|
||||
"panel-57": {
|
||||
"kind": "Panel",
|
||||
"spec": {
|
||||
"data": { "kind": "QueryGroup", "spec": { "queries": [{ "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(rate(stripe_payment_failed_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))", "legendFormat": "failed/s" }, "version": "v0" }, "refId": "A" } }], "queryOptions": {}, "transformations": [] } },
|
||||
"description": "invoice.payment_failed events. Spikes mean dunning storms or card decline cascades.",
|
||||
"id": 57,
|
||||
"links": [],
|
||||
"title": "Payment Failures",
|
||||
"vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "yellow", "value": 0.01 }, { "color": "red", "value": 0.1 }] }, "unit": "ops" }, "overrides": [] }, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true } }, "version": "13.0.0-23630096546" }
|
||||
}
|
||||
},
|
||||
"panel-58": {
|
||||
"kind": "Panel",
|
||||
"spec": {
|
||||
"data": { "kind": "QueryGroup", "spec": { "queries": [{ "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "100 * sum(increase(stripe_checkout_completed_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__range])) / clamp_min(sum(increase(stripe_checkout_created_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__range])), 1)", "legendFormat": "completed %" }, "version": "v0" }, "refId": "A" } }], "queryOptions": {}, "transformations": [] } },
|
||||
"description": "Checkouts completed / created over the dashboard time range. Drops can flag price-page bugs or payment-method outages.",
|
||||
"id": 58,
|
||||
"links": [],
|
||||
"title": "Checkout Conversion",
|
||||
"vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": 0 }, { "color": "yellow", "value": 30 }, { "color": "green", "value": 60 }] }, "unit": "percent" }, "overrides": [] }, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true } }, "version": "13.0.0-23630096546" }
|
||||
}
|
||||
},
|
||||
"panel-60": {
|
||||
"kind": "Panel",
|
||||
"spec": {
|
||||
"data": { "kind": "QueryGroup", "spec": { "queries": [{ "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (source) (rate(airi_billing_flux_credited_total{service_name=~\"$service\", deployment_environment=~\"$env\", source!=\"\"}[$__rate_interval]))", "legendFormat": "{{source}}" }, "version": "v0" }, "refId": "A" } }], "queryOptions": {}, "transformations": [] } },
|
||||
"description": "Flux added to user balances, broken down by source: stripe.checkout / stripe.invoice / promo / admin grants.",
|
||||
"id": 60,
|
||||
"links": [],
|
||||
"title": "Flux Credited by Source",
|
||||
"vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }] }, "unit": "short" }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.0.0-23630096546" }
|
||||
}
|
||||
},
|
||||
"panel-61": {
|
||||
"kind": "Panel",
|
||||
"spec": {
|
||||
"data": { "kind": "QueryGroup", "spec": { "queries": [{ "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (currency) (rate(airi_stripe_revenue_minor_unit_total{service_name=~\"$service\", deployment_environment=~\"$env\", currency!=\"\"}[$__rate_interval])) / 100", "legendFormat": "{{currency}}" }, "version": "v0" }, "refId": "A" } }], "queryOptions": {}, "transformations": [] } },
|
||||
"description": "Real revenue per second, in major currency unit (cents -> dollars / equivalent). Always grouped by currency — cross-currency sums are meaningless.",
|
||||
"id": 61,
|
||||
"links": [],
|
||||
"title": "Stripe Revenue Rate (major unit/s)",
|
||||
"vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }] }, "unit": "short" }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.0.0-23630096546" }
|
||||
}
|
||||
},
|
||||
"panel-62": {
|
||||
"kind": "Panel",
|
||||
"spec": {
|
||||
"data": { "kind": "QueryGroup", "spec": { "queries": [{ "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (gen_ai_request_model) (rate(airi_billing_flux_unbilled_total{service_name=~\"$service\", deployment_environment=~\"$env\", gen_ai_request_model!=\"\"}[$__rate_interval]))", "legendFormat": "{{gen_ai_request_model}}" }, "version": "v0" }, "refId": "A" } }], "queryOptions": {}, "transformations": [] } },
|
||||
"description": "Flux that should have been debited but wasn't (debit failure after streaming response sent). Real revenue leak — anything above 0 sustained should page.",
|
||||
"id": 62,
|
||||
"links": [],
|
||||
"title": "Flux Unbilled (revenue leak)",
|
||||
"vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 30, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "red", "value": 0.001 }] }, "unit": "short" }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.0.0-23630096546" }
|
||||
}
|
||||
},
|
||||
"panel-63": {
|
||||
"kind": "Panel",
|
||||
"spec": {
|
||||
"data": { "kind": "QueryGroup", "spec": { "queries": [{ "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (gen_ai_request_model, stage) (rate(airi_gen_ai_stream_interrupted_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))", "legendFormat": "{{gen_ai_request_model}} ({{stage}})" }, "version": "v0" }, "refId": "A" } }], "queryOptions": {}, "transformations": [] } },
|
||||
"description": "Streaming responses that died mid-flight. `before_first_chunk` = upstream blew up before any token; `mid_stream` = partial delivery.",
|
||||
"id": 63,
|
||||
"links": [],
|
||||
"title": "Stream Interruptions",
|
||||
"vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "red", "value": 0.5 }] }, "unit": "ops" }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.0.0-23630096546" }
|
||||
}
|
||||
},
|
||||
"panel-64": {
|
||||
"kind": "Panel",
|
||||
"spec": {
|
||||
"data": { "kind": "QueryGroup", "spec": { "queries": [{ "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "histogram_quantile(0.95, sum by (le, gen_ai_request_model) (rate(gen_ai_client_first_token_duration_seconds_bucket{service_name=~\"$service\", deployment_environment=~\"$env\", gen_ai_request_model!=\"\"}[$__rate_interval])))", "legendFormat": "{{gen_ai_request_model}}" }, "version": "v0" }, "refId": "A" } }], "queryOptions": {}, "transformations": [] } },
|
||||
"description": "Time from request start to first streamed token (TTFB). Tracks streaming chat experience independently from total operation duration.",
|
||||
"id": 64,
|
||||
"links": [],
|
||||
"title": "First Token P95 Latency",
|
||||
"vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "yellow", "value": 1 }, { "color": "red", "value": 3 }] }, "unit": "s" }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.0.0-23630096546" }
|
||||
}
|
||||
},
|
||||
"panel-65": {
|
||||
"kind": "Panel",
|
||||
"spec": {
|
||||
"data": { "kind": "QueryGroup", "spec": { "queries": [{ "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (model) (rate(airi_billing_tts_chars_total{service_name=~\"$service\", deployment_environment=~\"$env\", model!=\"\"}[$__rate_interval]))", "legendFormat": "{{model}}" }, "version": "v0" }, "refId": "A" } }], "queryOptions": {}, "transformations": [] } },
|
||||
"description": "TTS input characters processed per second — billing base before whole-Flux rounding. Spikes here without proportional flux_consumed = debt accumulating in Redis.",
|
||||
"id": 65,
|
||||
"links": [],
|
||||
"title": "TTS Chars Rate",
|
||||
"vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }] }, "unit": "short" }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.0.0-23630096546" }
|
||||
}
|
||||
},
|
||||
"panel-66": {
|
||||
"kind": "Panel",
|
||||
"spec": {
|
||||
"data": { "kind": "QueryGroup", "spec": { "queries": [{ "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (meter, reason) (rate(airi_billing_tts_preflight_rejections_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))", "legendFormat": "{{meter}} ({{reason}})" }, "version": "v0" }, "refId": "A" } }], "queryOptions": {}, "transformations": [] } },
|
||||
"description": "Pre-flight rejections from flux-meter assertCanAfford. Volume here = users hitting paywall before upstream call.",
|
||||
"id": 66,
|
||||
"links": [],
|
||||
"title": "TTS Preflight Rejections",
|
||||
"vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }] }, "unit": "ops" }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.0.0-23630096546" }
|
||||
}
|
||||
},
|
||||
"panel-67": {
|
||||
"kind": "Panel",
|
||||
"spec": {
|
||||
"data": { "kind": "QueryGroup", "spec": { "queries": [{ "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (template) (rate(airi_email_send_total{service_name=~\"$service\", deployment_environment=~\"$env\", template!=\"\"}[$__rate_interval]))", "legendFormat": "{{template}}" }, "version": "v0" }, "refId": "A" } }], "queryOptions": {}, "transformations": [] } },
|
||||
"description": "Resend accepted emails by template (verification / password_reset / magic_link / change_email / delete_account).",
|
||||
"id": 67,
|
||||
"links": [],
|
||||
"title": "Email Send Rate",
|
||||
"vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }] }, "unit": "ops" }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.0.0-23630096546" }
|
||||
}
|
||||
},
|
||||
"panel-68": {
|
||||
"kind": "Panel",
|
||||
"spec": {
|
||||
"data": { "kind": "QueryGroup", "spec": { "queries": [{ "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "100 * sum(rate(airi_email_failures_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval])) / clamp_min(sum(rate(airi_email_send_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval])) + sum(rate(airi_email_failures_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval])), 1)", "legendFormat": "fail %" }, "version": "v0" }, "refId": "A" } }], "queryOptions": {}, "transformations": [] } },
|
||||
"description": "Email failures / total attempts. >5% means Resend / DNS / suppression list issues blocking auth flows.",
|
||||
"id": 68,
|
||||
"links": [],
|
||||
"title": "Email Failure Rate %",
|
||||
"vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "yellow", "value": 1 }, { "color": "red", "value": 5 }] }, "unit": "percent" }, "overrides": [] }, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true } }, "version": "13.0.0-23630096546" }
|
||||
}
|
||||
},
|
||||
"panel-69": {
|
||||
"kind": "Panel",
|
||||
"spec": {
|
||||
"data": { "kind": "QueryGroup", "spec": { "queries": [{ "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (route, key_type) (rate(airi_rate_limit_blocked_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))", "legendFormat": "{{route}} ({{key_type}})" }, "version": "v0" }, "refId": "A" } }], "queryOptions": {}, "transformations": [] } },
|
||||
"description": "Requests denied by rate limiter, broken down by route + key type (user vs ip). Sustained ip-keyed traffic = unauthenticated abuse / scraping.",
|
||||
"id": 69,
|
||||
"links": [],
|
||||
"title": "Rate Limit Blocked",
|
||||
"vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "red", "value": 1 }] }, "unit": "ops" }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.0.0-23630096546" }
|
||||
}
|
||||
}
|
||||
},
|
||||
"layout": {
|
||||
@@ -1799,15 +2019,35 @@
|
||||
"kind": "GridLayout",
|
||||
"spec": {
|
||||
"items": [
|
||||
{ "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-31" }, "height": 8, "width": 8, "x": 0, "y": 0 } },
|
||||
{ "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-32" }, "height": 8, "width": 8, "x": 8, "y": 0 } },
|
||||
{ "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-33" }, "height": 8, "width": 8, "x": 16, "y": 0 } }
|
||||
{ "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-31" }, "height": 8, "width": 6, "x": 0, "y": 0 } },
|
||||
{ "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-34" }, "height": 8, "width": 6, "x": 6, "y": 0 } },
|
||||
{ "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-32" }, "height": 8, "width": 6, "x": 12, "y": 0 } },
|
||||
{ "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-33" }, "height": 8, "width": 6, "x": 18, "y": 0 } }
|
||||
]
|
||||
}
|
||||
},
|
||||
"title": "Auth & Users"
|
||||
}
|
||||
},
|
||||
{
|
||||
"kind": "RowsLayoutRow",
|
||||
"spec": {
|
||||
"collapse": false,
|
||||
"layout": {
|
||||
"kind": "GridLayout",
|
||||
"spec": {
|
||||
"items": [
|
||||
{ "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-50" }, "height": 8, "width": 6, "x": 0, "y": 0 } },
|
||||
{ "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-51" }, "height": 8, "width": 9, "x": 6, "y": 0 } },
|
||||
{ "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-52" }, "height": 8, "width": 9, "x": 15, "y": 0 } },
|
||||
{ "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-53" }, "height": 8, "width": 12, "x": 0, "y": 8 } },
|
||||
{ "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-54" }, "height": 8, "width": 12, "x": 12, "y": 8 } }
|
||||
]
|
||||
}
|
||||
},
|
||||
"title": "Engagement"
|
||||
}
|
||||
},
|
||||
{
|
||||
"kind": "RowsLayoutRow",
|
||||
"spec": {
|
||||
@@ -1819,14 +2059,34 @@
|
||||
{ "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-40" }, "height": 8, "width": 8, "x": 0, "y": 0 } },
|
||||
{ "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-41" }, "height": 8, "width": 8, "x": 8, "y": 0 } },
|
||||
{ "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-44" }, "height": 8, "width": 8, "x": 16, "y": 0 } },
|
||||
{ "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-42" }, "height": 8, "width": 12, "x": 0, "y": 8 } },
|
||||
{ "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-43" }, "height": 8, "width": 12, "x": 12, "y": 8 } }
|
||||
{ "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-42" }, "height": 8, "width": 8, "x": 0, "y": 8 } },
|
||||
{ "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-43" }, "height": 8, "width": 8, "x": 8, "y": 8 } },
|
||||
{ "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-60" }, "height": 8, "width": 8, "x": 16, "y": 8 } }
|
||||
]
|
||||
}
|
||||
},
|
||||
"title": "Business Metrics"
|
||||
}
|
||||
},
|
||||
{
|
||||
"kind": "RowsLayoutRow",
|
||||
"spec": {
|
||||
"collapse": false,
|
||||
"layout": {
|
||||
"kind": "GridLayout",
|
||||
"spec": {
|
||||
"items": [
|
||||
{ "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-57" }, "height": 8, "width": 6, "x": 0, "y": 0 } },
|
||||
{ "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-58" }, "height": 8, "width": 6, "x": 6, "y": 0 } },
|
||||
{ "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-55" }, "height": 8, "width": 12, "x": 12, "y": 0 } },
|
||||
{ "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-56" }, "height": 8, "width": 12, "x": 0, "y": 8 } },
|
||||
{ "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-61" }, "height": 8, "width": 12, "x": 12, "y": 8 } }
|
||||
]
|
||||
}
|
||||
},
|
||||
"title": "Stripe Detail"
|
||||
}
|
||||
},
|
||||
{
|
||||
"kind": "RowsLayoutRow",
|
||||
"spec": {
|
||||
@@ -1859,15 +2119,37 @@
|
||||
{ "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-20" }, "height": 8, "width": 12, "x": 0, "y": 8 } },
|
||||
{ "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-21" }, "height": 8, "width": 12, "x": 12, "y": 8 } },
|
||||
{ "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-22" }, "height": 8, "width": 12, "x": 0, "y": 16 } },
|
||||
{ "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-23" }, "height": 8, "width": 12, "x": 12, "y": 16 } },
|
||||
{ "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-24" }, "height": 8, "width": 12, "x": 0, "y": 24 } },
|
||||
{ "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-25" }, "height": 8, "width": 12, "x": 12, "y": 24 } }
|
||||
{ "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-64" }, "height": 8, "width": 12, "x": 12, "y": 16 } },
|
||||
{ "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-23" }, "height": 8, "width": 12, "x": 0, "y": 24 } },
|
||||
{ "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-62" }, "height": 8, "width": 12, "x": 12, "y": 24 } },
|
||||
{ "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-63" }, "height": 8, "width": 12, "x": 0, "y": 32 } },
|
||||
{ "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-65" }, "height": 8, "width": 12, "x": 12, "y": 32 } },
|
||||
{ "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-24" }, "height": 8, "width": 12, "x": 0, "y": 40 } },
|
||||
{ "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-25" }, "height": 8, "width": 12, "x": 12, "y": 40 } }
|
||||
]
|
||||
}
|
||||
},
|
||||
"title": "LLM Gateway"
|
||||
}
|
||||
},
|
||||
{
|
||||
"kind": "RowsLayoutRow",
|
||||
"spec": {
|
||||
"collapse": false,
|
||||
"layout": {
|
||||
"kind": "GridLayout",
|
||||
"spec": {
|
||||
"items": [
|
||||
{ "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-68" }, "height": 8, "width": 6, "x": 0, "y": 0 } },
|
||||
{ "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-67" }, "height": 8, "width": 18, "x": 6, "y": 0 } },
|
||||
{ "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-69" }, "height": 8, "width": 12, "x": 0, "y": 8 } },
|
||||
{ "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-66" }, "height": 8, "width": 12, "x": 12, "y": 8 } }
|
||||
]
|
||||
}
|
||||
},
|
||||
"title": "Reliability"
|
||||
}
|
||||
},
|
||||
{
|
||||
"kind": "RowsLayoutRow",
|
||||
"spec": {
|
||||
|
||||
@@ -191,6 +191,7 @@ export async function buildApp(deps: AppDeps) {
|
||||
db: deps.db,
|
||||
env: deps.env,
|
||||
configKV: deps.configKV,
|
||||
rateLimitMetrics: deps.otel?.rateLimit,
|
||||
}))
|
||||
|
||||
/**
|
||||
@@ -211,7 +212,7 @@ export async function buildApp(deps: AppDeps) {
|
||||
/**
|
||||
* V1 routes for official provider.
|
||||
*/
|
||||
.route('/api/v1/openai', createV1CompletionsRoutes(deps.fluxService, deps.billingService, deps.configKV, deps.requestLogService, deps.ttsMeter, deps.redis, deps.env, deps.otel?.genAi))
|
||||
.route('/api/v1/openai', createV1CompletionsRoutes(deps.fluxService, deps.billingService, deps.configKV, deps.requestLogService, deps.ttsMeter, deps.redis, deps.env, deps.otel?.genAi, deps.otel?.revenue, deps.otel?.rateLimit))
|
||||
|
||||
/**
|
||||
* Flux routes.
|
||||
@@ -221,7 +222,7 @@ export async function buildApp(deps: AppDeps) {
|
||||
/**
|
||||
* Stripe routes.
|
||||
*/
|
||||
.route('/api/v1/stripe', createStripeRoutes(deps.fluxService, deps.stripeService, deps.billingService, deps.configKV, deps.env, deps.redis, deps.otel?.revenue))
|
||||
.route('/api/v1/stripe', createStripeRoutes(deps.fluxService, deps.stripeService, deps.billingService, deps.configKV, deps.env, deps.redis, deps.otel?.revenue, deps.otel?.rateLimit))
|
||||
|
||||
/**
|
||||
* Admin routes — guarded by `ADMIN_EMAILS` allowlist + verified email.
|
||||
@@ -329,12 +330,12 @@ export async function createApp() {
|
||||
})
|
||||
|
||||
const emailService = injeca.provide('services:email', {
|
||||
dependsOn: { env: parsedEnv },
|
||||
dependsOn: { env: parsedEnv, otel },
|
||||
build: ({ dependsOn }) => createEmailService({
|
||||
apiKey: dependsOn.env.RESEND_API_KEY,
|
||||
fromEmail: dependsOn.env.RESEND_FROM_EMAIL,
|
||||
fromName: dependsOn.env.RESEND_FROM_NAME,
|
||||
}),
|
||||
}, undefined, dependsOn.otel?.email),
|
||||
})
|
||||
|
||||
const characterService = injeca.provide('services:characters', {
|
||||
@@ -433,7 +434,7 @@ export async function createApp() {
|
||||
})
|
||||
|
||||
const ttsMeter = injeca.provide('services:ttsMeter', {
|
||||
dependsOn: { redis, billingService, configKV },
|
||||
dependsOn: { redis, billingService, configKV, otel },
|
||||
build: ({ dependsOn }) => createFluxMeter(dependsOn.redis, dependsOn.billingService, {
|
||||
name: 'tts',
|
||||
// Lazy config read: missing FLUX_PER_1K_CHARS_TTS surfaces as a
|
||||
@@ -447,7 +448,7 @@ export async function createApp() {
|
||||
debtTtlSeconds: ttl,
|
||||
}
|
||||
},
|
||||
}),
|
||||
}, dependsOn.otel?.revenue),
|
||||
})
|
||||
|
||||
await injeca.start()
|
||||
|
||||
@@ -19,6 +19,16 @@ import { BatchSpanProcessor, ParentBasedSampler, TraceIdRatioBasedSampler } from
|
||||
import { ATTR_SERVICE_NAME, ATTR_SERVICE_VERSION } from '@opentelemetry/semantic-conventions'
|
||||
|
||||
import {
|
||||
METRIC_AIRI_EMAIL_DURATION,
|
||||
METRIC_AIRI_EMAIL_FAILURES,
|
||||
METRIC_AIRI_EMAIL_SEND,
|
||||
METRIC_AIRI_FLUX_CREDITED,
|
||||
METRIC_AIRI_FLUX_UNBILLED,
|
||||
METRIC_AIRI_GEN_AI_STREAM_INTERRUPTED,
|
||||
METRIC_AIRI_RATE_LIMIT_BLOCKED,
|
||||
METRIC_AIRI_STRIPE_REVENUE,
|
||||
METRIC_AIRI_TTS_CHARS,
|
||||
METRIC_AIRI_TTS_PREFLIGHT_REJECTIONS,
|
||||
METRIC_AUTH_ATTEMPTS,
|
||||
METRIC_AUTH_FAILURES,
|
||||
METRIC_CHARACTER_CREATED,
|
||||
@@ -27,6 +37,7 @@ import {
|
||||
METRIC_CHAT_MESSAGES,
|
||||
METRIC_FLUX_CONSUMED,
|
||||
METRIC_FLUX_INSUFFICIENT_BALANCE,
|
||||
METRIC_GEN_AI_CLIENT_FIRST_TOKEN_DURATION,
|
||||
METRIC_GEN_AI_CLIENT_OPERATION_COUNT,
|
||||
METRIC_GEN_AI_CLIENT_OPERATION_DURATION,
|
||||
METRIC_GEN_AI_CLIENT_TOKEN_USAGE_INPUT,
|
||||
@@ -77,7 +88,12 @@ export interface RevenueMetrics {
|
||||
stripePaymentFailed: Counter
|
||||
stripeSubscriptionEvent: Counter
|
||||
stripeEvents: Counter
|
||||
stripeRevenue: Counter
|
||||
fluxInsufficientBalance: Counter
|
||||
fluxCredited: Counter
|
||||
fluxUnbilled: Counter
|
||||
ttsChars: Counter
|
||||
ttsPreflightRejections: Counter
|
||||
}
|
||||
|
||||
export interface GenAiMetrics {
|
||||
@@ -86,6 +102,18 @@ export interface GenAiMetrics {
|
||||
tokenUsageInput: Counter
|
||||
tokenUsageOutput: Counter
|
||||
fluxConsumed: Counter
|
||||
firstTokenDuration: Histogram
|
||||
streamInterrupted: Counter
|
||||
}
|
||||
|
||||
export interface EmailMetrics {
|
||||
send: Counter
|
||||
failures: Counter
|
||||
duration: Histogram
|
||||
}
|
||||
|
||||
export interface RateLimitMetrics {
|
||||
blocked: Counter
|
||||
}
|
||||
|
||||
// NOTICE: Database metrics (db.client.operation.duration, redis.client.command.duration) were
|
||||
@@ -100,6 +128,8 @@ export interface OtelInstance {
|
||||
engagement: EngagementMetrics
|
||||
revenue: RevenueMetrics
|
||||
genAi: GenAiMetrics
|
||||
email: EmailMetrics
|
||||
rateLimit: RateLimitMetrics
|
||||
shutdown: () => Promise<void>
|
||||
}
|
||||
|
||||
@@ -256,9 +286,25 @@ export function initOtel(env: Env): OtelInstance | undefined {
|
||||
stripeEvents: meter.createCounter(METRIC_STRIPE_EVENTS, {
|
||||
description: 'Number of Stripe webhook events processed',
|
||||
}),
|
||||
stripeRevenue: meter.createCounter(METRIC_AIRI_STRIPE_REVENUE, {
|
||||
description: 'Stripe revenue in smallest currency unit (e.g. cents)',
|
||||
unit: 'minor_unit',
|
||||
}),
|
||||
fluxInsufficientBalance: meter.createCounter(METRIC_FLUX_INSUFFICIENT_BALANCE, {
|
||||
description: 'Number of insufficient flux balance errors',
|
||||
}),
|
||||
fluxCredited: meter.createCounter(METRIC_AIRI_FLUX_CREDITED, {
|
||||
description: 'Total flux credited to user balances, by source',
|
||||
}),
|
||||
fluxUnbilled: meter.createCounter(METRIC_AIRI_FLUX_UNBILLED, {
|
||||
description: 'Flux that should have been debited but was not (revenue leak)',
|
||||
}),
|
||||
ttsChars: meter.createCounter(METRIC_AIRI_TTS_CHARS, {
|
||||
description: 'TTS input characters processed (billing base unit)',
|
||||
}),
|
||||
ttsPreflightRejections: meter.createCounter(METRIC_AIRI_TTS_PREFLIGHT_REJECTIONS, {
|
||||
description: 'Pre-flight rejections from flux-meter assertCanAfford',
|
||||
}),
|
||||
}
|
||||
|
||||
// GenAI metrics (semconv: gen_ai.client.*)
|
||||
@@ -279,8 +325,76 @@ export function initOtel(env: Env): OtelInstance | undefined {
|
||||
fluxConsumed: meter.createCounter(METRIC_FLUX_CONSUMED, {
|
||||
description: 'Total flux consumed',
|
||||
}),
|
||||
firstTokenDuration: meter.createHistogram(METRIC_GEN_AI_CLIENT_FIRST_TOKEN_DURATION, {
|
||||
description: 'Time from request start to first streamed token (TTFB for streaming)',
|
||||
unit: 's',
|
||||
}),
|
||||
streamInterrupted: meter.createCounter(METRIC_AIRI_GEN_AI_STREAM_INTERRUPTED, {
|
||||
description: 'Streaming responses interrupted before completion',
|
||||
}),
|
||||
}
|
||||
|
||||
const email: EmailMetrics = {
|
||||
send: meter.createCounter(METRIC_AIRI_EMAIL_SEND, {
|
||||
description: 'Transactional emails accepted by Resend',
|
||||
}),
|
||||
failures: meter.createCounter(METRIC_AIRI_EMAIL_FAILURES, {
|
||||
description: 'Transactional email send failures',
|
||||
}),
|
||||
duration: meter.createHistogram(METRIC_AIRI_EMAIL_DURATION, {
|
||||
description: 'Email provider call duration',
|
||||
unit: 's',
|
||||
}),
|
||||
}
|
||||
|
||||
const rateLimit: RateLimitMetrics = {
|
||||
blocked: meter.createCounter(METRIC_AIRI_RATE_LIMIT_BLOCKED, {
|
||||
description: 'Requests blocked by rate limiter',
|
||||
}),
|
||||
}
|
||||
|
||||
// NOTICE:
|
||||
// OTel SDK only emits a Counter time series after .add() runs the first time.
|
||||
// Without this priming step, low-traffic counters (auth_failures_total,
|
||||
// stripe_*_total, payment_failed, ...) never appear in Prometheus / Grafana
|
||||
// until an event happens — making panels look broken on fresh deploys and
|
||||
// making absence-based alerts impossible to author. add(0) registers the
|
||||
// series with a baseline of 0 without distorting any rates.
|
||||
// Removal condition: OTel SDK changes default to register Counters at create
|
||||
// time (https://github.com/open-telemetry/opentelemetry-specification/issues/2298).
|
||||
function primeCounter(counter: Counter): void {
|
||||
counter.add(0)
|
||||
}
|
||||
primeCounter(auth.attempts)
|
||||
primeCounter(auth.failures)
|
||||
primeCounter(auth.userRegistered)
|
||||
primeCounter(auth.userLogin)
|
||||
primeCounter(engagement.chatMessages)
|
||||
primeCounter(engagement.characterCreated)
|
||||
primeCounter(engagement.characterDeleted)
|
||||
primeCounter(engagement.characterEngagement)
|
||||
primeCounter(engagement.wsMessagesSent)
|
||||
primeCounter(engagement.wsMessagesReceived)
|
||||
primeCounter(revenue.stripeCheckoutCreated)
|
||||
primeCounter(revenue.stripeCheckoutCompleted)
|
||||
primeCounter(revenue.stripePaymentFailed)
|
||||
primeCounter(revenue.stripeSubscriptionEvent)
|
||||
primeCounter(revenue.stripeEvents)
|
||||
primeCounter(revenue.stripeRevenue)
|
||||
primeCounter(revenue.fluxInsufficientBalance)
|
||||
primeCounter(revenue.fluxCredited)
|
||||
primeCounter(revenue.fluxUnbilled)
|
||||
primeCounter(revenue.ttsChars)
|
||||
primeCounter(revenue.ttsPreflightRejections)
|
||||
primeCounter(genAi.operationCount)
|
||||
primeCounter(genAi.tokenUsageInput)
|
||||
primeCounter(genAi.tokenUsageOutput)
|
||||
primeCounter(genAi.fluxConsumed)
|
||||
primeCounter(genAi.streamInterrupted)
|
||||
primeCounter(email.send)
|
||||
primeCounter(email.failures)
|
||||
primeCounter(rateLimit.blocked)
|
||||
|
||||
// Graceful shutdown
|
||||
const shutdown = async () => {
|
||||
try {
|
||||
@@ -299,6 +413,8 @@ export function initOtel(env: Env): OtelInstance | undefined {
|
||||
engagement,
|
||||
revenue,
|
||||
genAi,
|
||||
email,
|
||||
rateLimit,
|
||||
shutdown,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import type { Context } from 'hono'
|
||||
|
||||
import type { RateLimitMetrics } from '../libs/otel'
|
||||
import type { HonoEnv } from '../types/hono'
|
||||
|
||||
import { getConnInfo } from '@hono/node-server/conninfo'
|
||||
@@ -12,6 +13,20 @@ interface RateLimitOptions {
|
||||
windowSec: number
|
||||
/** Key generator: extracts a unique identifier from the request */
|
||||
keyGenerator?: (c: Context<HonoEnv>) => string
|
||||
/**
|
||||
* Optional metrics handle. When provided, blocked requests increment
|
||||
* `airi_rate_limit_blocked_total{route, key_type, limit}`.
|
||||
* `key_type` reflects whether the limiter keyed off authenticated user id
|
||||
* or remote IP — important for distinguishing logged-in abuse from
|
||||
* anonymous scraping.
|
||||
*/
|
||||
metrics?: RateLimitMetrics | null
|
||||
/**
|
||||
* Stable label for the route this limiter guards (e.g. `auth.api`,
|
||||
* `openai.completions`, `stripe.checkout`). Avoids high-cardinality URL
|
||||
* paths in metric labels.
|
||||
*/
|
||||
routeLabel?: string
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -19,6 +34,18 @@ interface RateLimitOptions {
|
||||
* Uses in-memory store by default (single-instance).
|
||||
*/
|
||||
export function rateLimiter(opts: RateLimitOptions) {
|
||||
const keyGen = opts.keyGenerator
|
||||
?? ((c) => {
|
||||
const userId = c.get('user')?.id
|
||||
if (userId)
|
||||
return userId
|
||||
|
||||
// NOTICE: prefer hono conninfo (uses underlying socket address) over
|
||||
// x-forwarded-for which can be spoofed. Falls back to header then 'anonymous'.
|
||||
const info = getConnInfo(c)
|
||||
return info.remote?.address ?? c.req.header('x-forwarded-for') ?? 'anonymous'
|
||||
})
|
||||
|
||||
return createRateLimiter<HonoEnv>({
|
||||
windowMs: opts.windowSec * 1000,
|
||||
limit: opts.max,
|
||||
@@ -26,16 +53,17 @@ export function rateLimiter(opts: RateLimitOptions) {
|
||||
// `RateLimit-*` header set. `draft-7`/`draft-8` switch to newer combined
|
||||
// header formats that are easier to break in existing clients and proxies.
|
||||
standardHeaders: 'draft-6',
|
||||
keyGenerator: opts.keyGenerator
|
||||
?? ((c) => {
|
||||
const userId = c.get('user')?.id
|
||||
if (userId)
|
||||
return userId
|
||||
|
||||
// NOTICE: prefer hono conninfo (uses underlying socket address) over
|
||||
// x-forwarded-for which can be spoofed. Falls back to header then 'anonymous'.
|
||||
const info = getConnInfo(c)
|
||||
return info.remote?.address ?? c.req.header('x-forwarded-for') ?? 'anonymous'
|
||||
}),
|
||||
keyGenerator: keyGen,
|
||||
handler: (c) => {
|
||||
// Record before producing the 429 response so the time series captures
|
||||
// every block, even when the response shape later changes.
|
||||
const keyType = c.get('user')?.id ? 'user' : 'ip'
|
||||
opts.metrics?.blocked.add(1, {
|
||||
route: opts.routeLabel ?? 'unknown',
|
||||
key_type: keyType,
|
||||
limit: String(opts.max),
|
||||
})
|
||||
return c.json({ error: 'TOO_MANY_REQUESTS', message: 'Too many requests' }, 429)
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import type { AuthInstance } from '../../libs/auth'
|
||||
import type { Database } from '../../libs/db'
|
||||
import type { Env } from '../../libs/env'
|
||||
import type { RateLimitMetrics } from '../../libs/otel'
|
||||
import type { ConfigKVService } from '../../services/config-kv'
|
||||
import type { HonoEnv } from '../../types/hono'
|
||||
|
||||
@@ -31,6 +32,7 @@ export interface AuthRoutesDeps {
|
||||
db: Database
|
||||
env: Env
|
||||
configKV: ConfigKVService
|
||||
rateLimitMetrics?: RateLimitMetrics | null
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -129,6 +131,8 @@ export async function createAuthRoutes(deps: AuthRoutesDeps) {
|
||||
max: await deps.configKV.getOrThrow('AUTH_RATE_LIMIT_MAX'),
|
||||
windowSec: await deps.configKV.getOrThrow('AUTH_RATE_LIMIT_WINDOW_SEC'),
|
||||
keyGenerator: c => c.req.header('x-forwarded-for') ?? c.req.header('x-real-ip') ?? 'unknown',
|
||||
metrics: deps.rateLimitMetrics,
|
||||
routeLabel: 'auth.api',
|
||||
}))
|
||||
.use('/api/auth/oauth2/authorize', async (c, next) => {
|
||||
await ensureDynamicFirstPartyRedirectUri(deps.db, c.req.raw)
|
||||
|
||||
@@ -2,7 +2,7 @@ import type { Context } from 'hono'
|
||||
import type Redis from 'ioredis'
|
||||
|
||||
import type { Env } from '../../../libs/env'
|
||||
import type { GenAiMetrics } from '../../../libs/otel'
|
||||
import type { GenAiMetrics, RateLimitMetrics, RevenueMetrics } from '../../../libs/otel'
|
||||
import type { UsageInfo } from '../../../services/billing/billing'
|
||||
import type { BillingService } from '../../../services/billing/billing-service'
|
||||
import type { FluxMeter } from '../../../services/billing/flux-meter'
|
||||
@@ -81,7 +81,18 @@ function getLlmMetricAttributes(opts: { model: string, type: string, status: num
|
||||
}
|
||||
}
|
||||
|
||||
export function createV1CompletionsRoutes(fluxService: FluxService, billingService: BillingService, configKV: ConfigKVService, requestLogService: RequestLogService, ttsMeter: FluxMeter, redis: Redis, env: Env, genAi?: GenAiMetrics | null) {
|
||||
export function createV1CompletionsRoutes(
|
||||
fluxService: FluxService,
|
||||
billingService: BillingService,
|
||||
configKV: ConfigKVService,
|
||||
requestLogService: RequestLogService,
|
||||
ttsMeter: FluxMeter,
|
||||
redis: Redis,
|
||||
env: Env,
|
||||
genAi?: GenAiMetrics | null,
|
||||
revenue?: RevenueMetrics | null,
|
||||
rateLimitMetrics?: RateLimitMetrics | null,
|
||||
) {
|
||||
const logger = useLogger('v1-completions').useGlobalConfig()
|
||||
// TODO: Extract this compat route into smaller facades/modules.
|
||||
// It currently mixes auth, rate limiting, proxying, billing, telemetry, and event publishing in one transport layer entrypoint.
|
||||
@@ -173,6 +184,11 @@ export function createV1CompletionsRoutes(fluxService: FluxService, billingServi
|
||||
let tailBuffer = ''
|
||||
let streamCompleted = false
|
||||
let streamInterrupted = false
|
||||
// First-chunk timestamp for gen_ai.client.first_token.duration. Latched
|
||||
// on the first byte from upstream — captures perceived "time to first
|
||||
// token" for streaming clients. NaN until the first chunk lands so
|
||||
// `Number.isFinite` gates the histogram record.
|
||||
let firstChunkAt = Number.NaN
|
||||
|
||||
// Process stream in background
|
||||
;(async () => {
|
||||
@@ -183,6 +199,13 @@ export function createV1CompletionsRoutes(fluxService: FluxService, billingServi
|
||||
streamCompleted = true
|
||||
break
|
||||
}
|
||||
if (!Number.isFinite(firstChunkAt)) {
|
||||
firstChunkAt = Date.now()
|
||||
genAi?.firstTokenDuration.record((firstChunkAt - startedAt) / 1000, {
|
||||
[GEN_AI_ATTR_REQUEST_MODEL]: requestModel,
|
||||
[GEN_AI_ATTR_OPERATION_NAME]: 'chat',
|
||||
})
|
||||
}
|
||||
await writer.write(value)
|
||||
const text = decoder.decode(value, { stream: true })
|
||||
tailBuffer = (tailBuffer + text).slice(-2048)
|
||||
@@ -192,6 +215,12 @@ export function createV1CompletionsRoutes(fluxService: FluxService, billingServi
|
||||
streamInterrupted = true
|
||||
span.setStatus({ code: SpanStatusCode.ERROR, message: 'Gateway stream interrupted' })
|
||||
span.setAttribute(AIRI_ATTR_GEN_AI_STREAM_INTERRUPTED, true)
|
||||
// Counter so alerts/dashboards can fire on interrupted streams; the
|
||||
// span attribute alone only shows up in trace search, not metrics.
|
||||
genAi?.streamInterrupted.add(1, {
|
||||
[GEN_AI_ATTR_REQUEST_MODEL]: requestModel,
|
||||
stage: Number.isFinite(firstChunkAt) ? 'mid_stream' : 'before_first_chunk',
|
||||
})
|
||||
|
||||
try {
|
||||
await writer.abort(err)
|
||||
@@ -255,7 +284,16 @@ export function createV1CompletionsRoutes(fluxService: FluxService, billingServi
|
||||
})
|
||||
actualCharged = fluxConsumed
|
||||
}
|
||||
catch (err) { logger.withError(err).withFields({ userId: user.id, fluxConsumed, requestId }).error('Failed to debit flux after streaming — unpaid usage') }
|
||||
catch (err) {
|
||||
// Flux that should have been collected but wasn't. Real revenue leak —
|
||||
// counter so on-call can alert and reconcile.
|
||||
revenue?.fluxUnbilled.add(fluxConsumed, {
|
||||
[GEN_AI_ATTR_REQUEST_MODEL]: requestModel,
|
||||
reason: 'debit_failed',
|
||||
stage: 'streaming',
|
||||
})
|
||||
logger.withError(err).withFields({ userId: user.id, fluxConsumed, requestId }).error('Failed to debit flux after streaming — unpaid usage')
|
||||
}
|
||||
|
||||
recordRequestLog({
|
||||
userId: user.id,
|
||||
@@ -449,7 +487,7 @@ export function createV1CompletionsRoutes(fluxService: FluxService, billingServi
|
||||
const ttsGuard = configGuard(configKV, ['FLUX_PER_1K_CHARS_TTS'], 'TTS service is not available yet')
|
||||
|
||||
// 60 requests per minute per user for LLM completions
|
||||
const completionsRateLimit = rateLimiter({ max: 60, windowSec: 60 })
|
||||
const completionsRateLimit = rateLimiter({ max: 60, windowSec: 60, metrics: rateLimitMetrics, routeLabel: 'openai.completions' })
|
||||
|
||||
return new Hono<HonoEnv>()
|
||||
.use('*', authGuard)
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import type Redis from 'ioredis'
|
||||
|
||||
import type { Env } from '../../libs/env'
|
||||
import type { RevenueMetrics } from '../../libs/otel'
|
||||
import type { RateLimitMetrics, RevenueMetrics } from '../../libs/otel'
|
||||
import type { BillingService } from '../../services/billing/billing-service'
|
||||
import type { ConfigKVService } from '../../services/config-kv'
|
||||
import type { FluxService } from '../../services/flux'
|
||||
@@ -49,6 +49,7 @@ export function createStripeRoutes(
|
||||
env: Env,
|
||||
redis: Redis,
|
||||
metrics?: RevenueMetrics | null,
|
||||
rateLimitMetrics?: RateLimitMetrics | null,
|
||||
) {
|
||||
const stripe = env.STRIPE_SECRET_KEY ? new Stripe(env.STRIPE_SECRET_KEY) : null
|
||||
|
||||
@@ -116,7 +117,7 @@ export function createStripeRoutes(
|
||||
}
|
||||
}))
|
||||
})
|
||||
.post('/checkout', authGuard, rateLimiter({ max: 10, windowSec: 60 }), async (c) => {
|
||||
.post('/checkout', authGuard, rateLimiter({ max: 10, windowSec: 60, metrics: rateLimitMetrics, routeLabel: 'stripe.checkout' }), async (c) => {
|
||||
const fluxProductId = await configKV.getOptional('STRIPE_FLUX_PRODUCT_ID')
|
||||
if (!stripe || !fluxProductId)
|
||||
throw createServiceUnavailableError('Stripe is not configured', 'STRIPE_NOT_CONFIGURED')
|
||||
@@ -286,6 +287,14 @@ export function createStripeRoutes(
|
||||
case 'checkout.session.completed': {
|
||||
await handleCheckoutSessionCompleted(event.id, event.data.object, fluxService, stripeService, billingService)
|
||||
metrics?.stripeCheckoutCompleted.add(1)
|
||||
// Revenue capture in smallest currency unit (e.g. cents).
|
||||
// Cross-currency aggregation is meaningless, so always group by `currency` in queries.
|
||||
if (event.data.object.amount_total != null && event.data.object.currency) {
|
||||
metrics?.stripeRevenue.add(event.data.object.amount_total, {
|
||||
currency: event.data.object.currency,
|
||||
source: 'checkout',
|
||||
})
|
||||
}
|
||||
break
|
||||
}
|
||||
case 'customer.created':
|
||||
@@ -308,6 +317,12 @@ export function createStripeRoutes(
|
||||
if (event.type === 'invoice.payment_failed') {
|
||||
metrics?.stripePaymentFailed.add(1)
|
||||
}
|
||||
if (event.type === 'invoice.paid' && event.data.object.amount_paid && event.data.object.currency) {
|
||||
metrics?.stripeRevenue.add(event.data.object.amount_paid, {
|
||||
currency: event.data.object.currency,
|
||||
source: 'invoice',
|
||||
})
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
/**
|
||||
* OTel metric-registration smoke test.
|
||||
*
|
||||
* Verifies every Counter created by `initOtel` is primed (.add(0)) so the
|
||||
* Prometheus exporter sees the time series at boot, not just after the first
|
||||
* real event. Without priming, low-traffic counters
|
||||
* (auth_failures_total, stripe_*_total, payment_failed, ...) never appear in
|
||||
* Grafana until an event happens — making panels look broken on fresh deploys
|
||||
* and absence-based alerts impossible to author.
|
||||
*
|
||||
* Histograms (gen_ai.client.first_token.duration, airi.email.duration, ...)
|
||||
* are intentionally NOT in the output — they only register on first .record().
|
||||
*
|
||||
* Usage:
|
||||
* pnpm -F @proj-airi/server exec node --import tsx ./src/scripts/otel-smoke.mjs
|
||||
*/
|
||||
import { env, exit } from 'node:process'
|
||||
|
||||
import { metrics } from '@opentelemetry/api'
|
||||
import { AggregationTemporality, InMemoryMetricExporter, MeterProvider, PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics'
|
||||
|
||||
const exporter = new InMemoryMetricExporter(AggregationTemporality.CUMULATIVE)
|
||||
const reader = new PeriodicExportingMetricReader({ exporter, exportIntervalMillis: 100 })
|
||||
const provider = new MeterProvider({ readers: [reader] })
|
||||
metrics.setGlobalMeterProvider(provider)
|
||||
|
||||
// Smoke-test env: stand in for required vars so parseEnv passes; OTLP endpoint
|
||||
// is set to a localhost URL that won't actually receive anything because we
|
||||
// flush via the in-memory reader.
|
||||
env.DATABASE_URL ??= 'postgres://test'
|
||||
env.REDIS_URL ??= 'redis://test'
|
||||
env.BETTER_AUTH_SECRET ??= 'test'
|
||||
env.AUTH_GOOGLE_CLIENT_ID ??= 'test'
|
||||
env.AUTH_GOOGLE_CLIENT_SECRET ??= 'test'
|
||||
env.AUTH_GITHUB_CLIENT_ID ??= 'test'
|
||||
env.AUTH_GITHUB_CLIENT_SECRET ??= 'test'
|
||||
env.GATEWAY_BASE_URL ??= 'http://test'
|
||||
env.DEFAULT_CHAT_MODEL ??= 'test'
|
||||
env.DEFAULT_TTS_MODEL ??= 'test'
|
||||
env.OTEL_EXPORTER_OTLP_ENDPOINT ??= 'http://localhost:4318'
|
||||
|
||||
const { initOtel } = await import('../libs/otel.ts')
|
||||
const { parseEnv } = await import('../libs/env.ts')
|
||||
|
||||
const parsed = parseEnv(env)
|
||||
const inst = initOtel(parsed)
|
||||
if (!inst) {
|
||||
console.error('initOtel returned undefined')
|
||||
exit(1)
|
||||
}
|
||||
|
||||
await new Promise(r => setTimeout(r, 300))
|
||||
await reader.forceFlush()
|
||||
const exported = exporter.getMetrics()
|
||||
const names = []
|
||||
for (const rm of exported) {
|
||||
for (const sm of rm.scopeMetrics) {
|
||||
for (const m of sm.metrics) names.push(m.descriptor.name)
|
||||
}
|
||||
}
|
||||
console.info('REGISTERED:')
|
||||
for (const n of [...new Set(names)].sort()) console.info(` ${n}`)
|
||||
await inst.shutdown()
|
||||
exit(0)
|
||||
@@ -260,6 +260,7 @@ export function createBillingService(
|
||||
}
|
||||
|
||||
await updateRedisCache(input.userId, txResult.balanceAfter)
|
||||
metrics?.fluxCredited.add(input.amount, { source: input.source, type: ledgerType })
|
||||
|
||||
logger.withFields({ userId: input.userId, amount: input.amount, balance: txResult.balanceAfter }).log('Credited flux')
|
||||
return txResult
|
||||
@@ -336,6 +337,7 @@ export function createBillingService(
|
||||
|
||||
if (txResult.applied && txResult.balanceAfter != null) {
|
||||
await updateRedisCache(input.userId, txResult.balanceAfter)
|
||||
metrics?.fluxCredited.add(input.fluxAmount, { source: 'stripe.checkout', type: 'credit' })
|
||||
}
|
||||
|
||||
return txResult
|
||||
@@ -410,6 +412,7 @@ export function createBillingService(
|
||||
|
||||
if (txResult.applied && txResult.balanceAfter != null) {
|
||||
await updateRedisCache(input.userId, txResult.balanceAfter)
|
||||
metrics?.fluxCredited.add(input.fluxAmount, { source: 'stripe.invoice', type: 'credit' })
|
||||
}
|
||||
|
||||
return txResult
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import type Redis from 'ioredis'
|
||||
|
||||
import type { RevenueMetrics } from '../../libs/otel'
|
||||
import type { BillingService } from './billing-service'
|
||||
|
||||
import { useLogger } from '@guiiai/logg'
|
||||
@@ -80,6 +81,7 @@ export function createFluxMeter(
|
||||
redis: Redis,
|
||||
billingService: BillingService,
|
||||
config: FluxMeterConfig,
|
||||
metrics?: RevenueMetrics | null,
|
||||
) {
|
||||
async function getRuntime(): Promise<FluxMeterRuntime> {
|
||||
const runtime = await config.resolveRuntime()
|
||||
@@ -119,8 +121,10 @@ export function createFluxMeter(
|
||||
// At minimum require the user can cover a single Flux crossing; avoids
|
||||
// letting zero-balance users accumulate indefinitely on the boundary.
|
||||
const required = Math.max(projectedFlux, currentBalance <= 0 ? 1 : 0)
|
||||
if (currentBalance < required)
|
||||
if (currentBalance < required) {
|
||||
metrics?.ttsPreflightRejections.add(1, { meter: config.name, reason: 'insufficient_balance' })
|
||||
throw createPaymentRequiredError('Insufficient flux')
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -132,6 +136,9 @@ export function createFluxMeter(
|
||||
if (!Number.isFinite(input.units) || input.units <= 0)
|
||||
return { fluxDebited: 0, debtAfter: await readDebt(input.userId), balanceAfter: input.currentBalance }
|
||||
|
||||
const modelLabel = typeof input.metadata?.model === 'string' ? input.metadata.model : 'unknown'
|
||||
metrics?.ttsChars.add(input.units, { meter: config.name, model: modelLabel })
|
||||
|
||||
const runtime = await getRuntime()
|
||||
const key = userFluxMeterDebtRedisKey(input.userId, config.name)
|
||||
const [fluxDebited, debtAfter] = await runScript(key, input.units, runtime)
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
import type { Logger } from '@guiiai/logg'
|
||||
|
||||
import type { EmailMetrics } from '../libs/otel'
|
||||
|
||||
import { useLogger } from '@guiiai/logg'
|
||||
import { errorMessageFrom } from '@moeru/std'
|
||||
import { Resend } from 'resend'
|
||||
@@ -92,7 +94,7 @@ function formatFrom(config: EmailConfig): string {
|
||||
* instead of silently dropping mail — Better Auth surfaces it back to the
|
||||
* caller so frontend can show a clear "email service not configured" error.
|
||||
*/
|
||||
export function createEmailService(config: EmailConfig, logger: Logger = useLogger('email')): EmailService {
|
||||
export function createEmailService(config: EmailConfig, logger: Logger = useLogger('email'), metrics?: EmailMetrics | null): EmailService {
|
||||
// NOTICE:
|
||||
// Construct Resend lazily so the server can boot in environments where the
|
||||
// RESEND_API_KEY is intentionally empty (e.g. local dev that never exercises
|
||||
@@ -118,7 +120,8 @@ export function createEmailService(config: EmailConfig, logger: Logger = useLogg
|
||||
|
||||
const from = formatFrom(config)
|
||||
|
||||
async function send(payload: EmailPayload): Promise<void> {
|
||||
async function send(payload: EmailPayload, template: string = 'unknown'): Promise<void> {
|
||||
const startedAt = Date.now()
|
||||
try {
|
||||
const { error } = await getClient().emails.send({
|
||||
from,
|
||||
@@ -130,8 +133,12 @@ export function createEmailService(config: EmailConfig, logger: Logger = useLogg
|
||||
|
||||
if (error) {
|
||||
logger.withFields({ to: payload.to, subject: payload.subject, errorName: error.name }).error(error.message)
|
||||
metrics?.failures.add(1, { template, error_name: error.name })
|
||||
metrics?.duration.record((Date.now() - startedAt) / 1000, { template, outcome: 'error' })
|
||||
throw new ApiError(502, 'email/send_failed', error.message, { providerError: error.name })
|
||||
}
|
||||
metrics?.send.add(1, { template })
|
||||
metrics?.duration.record((Date.now() - startedAt) / 1000, { template, outcome: 'ok' })
|
||||
}
|
||||
catch (error) {
|
||||
if (error instanceof ApiError)
|
||||
@@ -139,6 +146,8 @@ export function createEmailService(config: EmailConfig, logger: Logger = useLogg
|
||||
|
||||
const message = errorMessageFrom(error) ?? 'Unknown email send error'
|
||||
logger.withFields({ to: payload.to, subject: payload.subject }).error(message)
|
||||
metrics?.failures.add(1, { template, error_name: 'unhandled' })
|
||||
metrics?.duration.record((Date.now() - startedAt) / 1000, { template, outcome: 'error' })
|
||||
throw new ApiError(502, 'email/send_failed', message)
|
||||
}
|
||||
}
|
||||
@@ -151,7 +160,7 @@ export function createEmailService(config: EmailConfig, logger: Logger = useLogg
|
||||
subject: 'Verify your email for Project AIRI',
|
||||
html: renderVerificationHtml(url),
|
||||
text: renderVerificationText(url),
|
||||
})
|
||||
}, 'verification')
|
||||
},
|
||||
async sendPasswordReset({ to, url }) {
|
||||
await send({
|
||||
@@ -159,7 +168,7 @@ export function createEmailService(config: EmailConfig, logger: Logger = useLogg
|
||||
subject: 'Reset your Project AIRI password',
|
||||
html: renderPasswordResetHtml(url),
|
||||
text: renderPasswordResetText(url),
|
||||
})
|
||||
}, 'password_reset')
|
||||
},
|
||||
async sendMagicLink({ to, url }) {
|
||||
await send({
|
||||
@@ -167,7 +176,7 @@ export function createEmailService(config: EmailConfig, logger: Logger = useLogg
|
||||
subject: 'Your Project AIRI sign-in link',
|
||||
html: renderMagicLinkHtml(url),
|
||||
text: renderMagicLinkText(url),
|
||||
})
|
||||
}, 'magic_link')
|
||||
},
|
||||
async sendChangeEmailConfirmation({ to, newEmail, url }) {
|
||||
await send({
|
||||
@@ -175,7 +184,7 @@ export function createEmailService(config: EmailConfig, logger: Logger = useLogg
|
||||
subject: 'Confirm your new email address for Project AIRI',
|
||||
html: renderChangeEmailHtml(url, newEmail),
|
||||
text: renderChangeEmailText(url, newEmail),
|
||||
})
|
||||
}, 'change_email')
|
||||
},
|
||||
async sendDeleteAccountVerification({ to, url }) {
|
||||
await send({
|
||||
@@ -183,7 +192,7 @@ export function createEmailService(config: EmailConfig, logger: Logger = useLogg
|
||||
subject: 'Confirm account deletion for Project AIRI',
|
||||
html: renderDeleteAccountHtml(url),
|
||||
text: renderDeleteAccountText(url),
|
||||
})
|
||||
}, 'delete_account')
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -62,6 +62,27 @@ export const METRIC_GEN_AI_CLIENT_TOKEN_USAGE_INPUT = 'gen_ai.client.token.usage
|
||||
export const METRIC_GEN_AI_CLIENT_TOKEN_USAGE_OUTPUT = 'gen_ai.client.token.usage.output'
|
||||
export const METRIC_FLUX_CONSUMED = 'airi.billing.flux.consumed'
|
||||
|
||||
// AIRI billing — credit/debit visibility beyond raw consumption
|
||||
export const METRIC_AIRI_FLUX_CREDITED = 'airi.billing.flux.credited'
|
||||
export const METRIC_AIRI_FLUX_UNBILLED = 'airi.billing.flux.unbilled'
|
||||
export const METRIC_AIRI_TTS_CHARS = 'airi.billing.tts.chars'
|
||||
export const METRIC_AIRI_TTS_PREFLIGHT_REJECTIONS = 'airi.billing.tts.preflight_rejections'
|
||||
|
||||
// AIRI revenue — actual money in (smallest currency unit, e.g. cents)
|
||||
export const METRIC_AIRI_STRIPE_REVENUE = 'airi.stripe.revenue'
|
||||
|
||||
// AIRI email — transactional delivery health
|
||||
export const METRIC_AIRI_EMAIL_SEND = 'airi.email.send'
|
||||
export const METRIC_AIRI_EMAIL_FAILURES = 'airi.email.failures'
|
||||
export const METRIC_AIRI_EMAIL_DURATION = 'airi.email.duration'
|
||||
|
||||
// AIRI rate limiting — abuse / attack visibility
|
||||
export const METRIC_AIRI_RATE_LIMIT_BLOCKED = 'airi.rate_limit.blocked'
|
||||
|
||||
// AIRI GenAI — stream quality
|
||||
export const METRIC_AIRI_GEN_AI_STREAM_INTERRUPTED = 'airi.gen_ai.stream.interrupted'
|
||||
export const METRIC_GEN_AI_CLIENT_FIRST_TOKEN_DURATION = 'gen_ai.client.first_token.duration'
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
Reference in New Issue
Block a user