From 917450fdfbc6c8b7e9d4324a37eb7e5505ba91e6 Mon Sep 17 00:00:00 2001 From: RainbowBird Date: Fri, 8 May 2026 23:35:38 +0800 Subject: [PATCH] feat(server/otel): enhance observability with new metrics for email, rate limiting, and billing - Added new metrics for email service including send, failures, and duration tracking. - Introduced rate limit metrics to monitor blocked requests and improve abuse detection. - Enhanced billing metrics to track credited and unbilled flux, as well as TTS character processing. - Updated OpenAI and Stripe routes to utilize new metrics for better revenue tracking and rate limiting. - Implemented a smoke test for OpenTelemetry metrics registration to ensure visibility at startup. --- apps/server/instrumentation.mjs | 15 + .../airi-server-overview-cloud.json | 298 +++++++++++++++++- apps/server/src/app.ts | 13 +- apps/server/src/libs/otel.ts | 116 +++++++ apps/server/src/middlewares/rate-limit.ts | 50 ++- apps/server/src/routes/auth/index.ts | 4 + apps/server/src/routes/openai/v1/index.ts | 46 ++- apps/server/src/routes/stripe/index.ts | 19 +- apps/server/src/scripts/otel-smoke.mjs | 64 ++++ .../src/services/billing/billing-service.ts | 3 + .../server/src/services/billing/flux-meter.ts | 9 +- apps/server/src/services/email.ts | 23 +- apps/server/src/utils/observability.ts | 21 ++ 13 files changed, 642 insertions(+), 39 deletions(-) create mode 100644 apps/server/src/scripts/otel-smoke.mjs diff --git a/apps/server/instrumentation.mjs b/apps/server/instrumentation.mjs index 2cdc6e6de..8b005743b 100644 --- a/apps/server/instrumentation.mjs +++ b/apps/server/instrumentation.mjs @@ -13,11 +13,26 @@ * the CJS module, so `require-in-the-middle` hooks still intercept it. */ +import { env } from 'node:process' + import { registerInstrumentations } from '@opentelemetry/instrumentation' import { HttpInstrumentation } from '@opentelemetry/instrumentation-http' import { IORedisInstrumentation } from '@opentelemetry/instrumentation-ioredis' import { PgInstrumentation } from '@opentelemetry/instrumentation-pg' +// NOTICE: +// instrumentation-http >=0.215 defaults to OLD semconv (http.server.duration in +// ms). Our Grafana dashboards / alerts only query the STABLE name +// (http.server.request.duration in seconds), and grep across the repo confirms +// no OLD-name consumer exists, so we go straight to STABLE-only — no `http/dup` +// transition phase, no doubled cardinality. +// Source: node_modules/.pnpm/@opentelemetry+instrumentation-http@0.215.0/.../build/src/http.js L25-72 +// MUST run before `new HttpInstrumentation(...)` below — its constructor reads +// the env var once and caches the result. +// Removal condition: ops sets OTEL_SEMCONV_STABILITY_OPT_IN explicitly in the +// deployment platform (Railway env), then this preload default can be deleted. +env.OTEL_SEMCONV_STABILITY_OPT_IN ??= 'http' + registerInstrumentations({ instrumentations: [ new HttpInstrumentation({ diff --git a/apps/server/otel/grafana/dashboards/airi-server-overview-cloud.json b/apps/server/otel/grafana/dashboards/airi-server-overview-cloud.json index 715b81f65..21a9f5e46 100644 --- a/apps/server/otel/grafana/dashboards/airi-server-overview-cloud.json +++ b/apps/server/otel/grafana/dashboards/airi-server-overview-cloud.json @@ -1764,6 +1764,226 @@ "version": "13.0.0-23630096546" } } + }, + "panel-34": { + "kind": "Panel", + "spec": { + "data": { "kind": "QueryGroup", "spec": { "queries": [{ "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "100 * sum(rate(auth_failures_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval])) / clamp_min(sum(rate(auth_attempts_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval])), 1)", "legendFormat": "fail %" }, "version": "v0" }, "refId": "A" } }], "queryOptions": {}, "transformations": [] } }, + "description": "auth_failures / auth_attempts; high values mean credential stuffing or upstream OAuth breakage.", + "id": 34, + "links": [], + "title": "Auth Failure Rate %", + "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "yellow", "value": 10 }, { "color": "red", "value": 30 }] }, "unit": "percent" }, "overrides": [] }, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true } }, "version": "13.0.0-23630096546" } + } + }, + "panel-50": { + "kind": "Panel", + "spec": { + "data": { "kind": "QueryGroup", "spec": { "queries": [{ "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(ws_connections_active{service_name=~\"$service\", deployment_environment=~\"$env\"})", "legendFormat": "active" }, "version": "v0" }, "refId": "A" } }], "queryOptions": {}, "transformations": [] } }, + "description": "Sum of active WS connections across all instances.", + "id": 50, + "links": [], + "title": "WS Connections Active", + "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "yellow", "value": 1000 }, { "color": "red", "value": 10000 }] }, "unit": "short" }, "overrides": [] }, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true } }, "version": "13.0.0-23630096546" } + } + }, + "panel-51": { + "kind": "Panel", + "spec": { + "data": { "kind": "QueryGroup", "spec": { "queries": [{ "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(rate(ws_messages_sent_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))", "legendFormat": "sent/s" }, "version": "v0" }, "refId": "A" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(rate(ws_messages_received_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))", "legendFormat": "received/s" }, "version": "v0" }, "refId": "B" } }], "queryOptions": {}, "transformations": [] } }, + "description": "WebSocket throughput in both directions.", + "id": 51, + "links": [], + "title": "WS Messages I/O", + "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "red", "value": 80 }] }, "unit": "ops" }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.0.0-23630096546" } + } + }, + "panel-52": { + "kind": "Panel", + "spec": { + "data": { "kind": "QueryGroup", "spec": { "queries": [{ "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(rate(chat_messages_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))", "legendFormat": "chat msgs/s" }, "version": "v0" }, "refId": "A" } }], "queryOptions": {}, "transformations": [] } }, + "description": "Chat messages persisted to chat sync (chats.pushMessages).", + "id": 52, + "links": [], + "title": "Chat Messages Rate", + "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "red", "value": 80 }] }, "unit": "ops" }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.0.0-23630096546" } + } + }, + "panel-53": { + "kind": "Panel", + "spec": { + "data": { "kind": "QueryGroup", "spec": { "queries": [{ "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(rate(character_created_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))", "legendFormat": "created/s" }, "version": "v0" }, "refId": "A" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(rate(character_deleted_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))", "legendFormat": "deleted/s" }, "version": "v0" }, "refId": "B" } }], "queryOptions": {}, "transformations": [] } }, + "description": "", + "id": 53, + "links": [], + "title": "Character Lifecycle", + "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "red", "value": 80 }] }, "unit": "ops" }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.0.0-23630096546" } + } + }, + "panel-54": { + "kind": "Panel", + "spec": { + "data": { "kind": "QueryGroup", "spec": { "queries": [{ "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (action) (rate(character_engagement_total{service_name=~\"$service\", deployment_environment=~\"$env\", action!=\"\"}[$__rate_interval]))", "legendFormat": "{{action}}" }, "version": "v0" }, "refId": "A" } }], "queryOptions": {}, "transformations": [] } }, + "description": "Like / unlike / bookmark / unbookmark actions on characters.", + "id": 54, + "links": [], + "title": "Character Engagement", + "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "red", "value": 80 }] }, "unit": "ops" }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.0.0-23630096546" } + } + }, + "panel-55": { + "kind": "Panel", + "spec": { + "data": { "kind": "QueryGroup", "spec": { "queries": [{ "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (event_type) (rate(stripe_events_total{service_name=~\"$service\", deployment_environment=~\"$env\", event_type!=\"\"}[$__rate_interval]))", "legendFormat": "{{event_type}}" }, "version": "v0" }, "refId": "A" } }], "queryOptions": {}, "transformations": [] } }, + "description": "All Stripe webhook events grouped by event.type.", + "id": 55, + "links": [], + "title": "Stripe Events by Type", + "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "red", "value": 80 }] }, "unit": "ops" }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.0.0-23630096546" } + } + }, + "panel-56": { + "kind": "Panel", + "spec": { + "data": { "kind": "QueryGroup", "spec": { "queries": [{ "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (event_type) (rate(stripe_subscription_event_total{service_name=~\"$service\", deployment_environment=~\"$env\", event_type!=\"\"}[$__rate_interval]))", "legendFormat": "{{event_type}}" }, "version": "v0" }, "refId": "A" } }], "queryOptions": {}, "transformations": [] } }, + "description": "Subscription lifecycle events (created/updated/deleted).", + "id": 56, + "links": [], + "title": "Subscription Events by Type", + "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "red", "value": 80 }] }, "unit": "ops" }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.0.0-23630096546" } + } + }, + "panel-57": { + "kind": "Panel", + "spec": { + "data": { "kind": "QueryGroup", "spec": { "queries": [{ "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(rate(stripe_payment_failed_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))", "legendFormat": "failed/s" }, "version": "v0" }, "refId": "A" } }], "queryOptions": {}, "transformations": [] } }, + "description": "invoice.payment_failed events. Spikes mean dunning storms or card decline cascades.", + "id": 57, + "links": [], + "title": "Payment Failures", + "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "yellow", "value": 0.01 }, { "color": "red", "value": 0.1 }] }, "unit": "ops" }, "overrides": [] }, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true } }, "version": "13.0.0-23630096546" } + } + }, + "panel-58": { + "kind": "Panel", + "spec": { + "data": { "kind": "QueryGroup", "spec": { "queries": [{ "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "100 * sum(increase(stripe_checkout_completed_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__range])) / clamp_min(sum(increase(stripe_checkout_created_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__range])), 1)", "legendFormat": "completed %" }, "version": "v0" }, "refId": "A" } }], "queryOptions": {}, "transformations": [] } }, + "description": "Checkouts completed / created over the dashboard time range. Drops can flag price-page bugs or payment-method outages.", + "id": 58, + "links": [], + "title": "Checkout Conversion", + "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": 0 }, { "color": "yellow", "value": 30 }, { "color": "green", "value": 60 }] }, "unit": "percent" }, "overrides": [] }, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true } }, "version": "13.0.0-23630096546" } + } + }, + "panel-60": { + "kind": "Panel", + "spec": { + "data": { "kind": "QueryGroup", "spec": { "queries": [{ "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (source) (rate(airi_billing_flux_credited_total{service_name=~\"$service\", deployment_environment=~\"$env\", source!=\"\"}[$__rate_interval]))", "legendFormat": "{{source}}" }, "version": "v0" }, "refId": "A" } }], "queryOptions": {}, "transformations": [] } }, + "description": "Flux added to user balances, broken down by source: stripe.checkout / stripe.invoice / promo / admin grants.", + "id": 60, + "links": [], + "title": "Flux Credited by Source", + "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }] }, "unit": "short" }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.0.0-23630096546" } + } + }, + "panel-61": { + "kind": "Panel", + "spec": { + "data": { "kind": "QueryGroup", "spec": { "queries": [{ "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (currency) (rate(airi_stripe_revenue_minor_unit_total{service_name=~\"$service\", deployment_environment=~\"$env\", currency!=\"\"}[$__rate_interval])) / 100", "legendFormat": "{{currency}}" }, "version": "v0" }, "refId": "A" } }], "queryOptions": {}, "transformations": [] } }, + "description": "Real revenue per second, in major currency unit (cents -> dollars / equivalent). Always grouped by currency — cross-currency sums are meaningless.", + "id": 61, + "links": [], + "title": "Stripe Revenue Rate (major unit/s)", + "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }] }, "unit": "short" }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.0.0-23630096546" } + } + }, + "panel-62": { + "kind": "Panel", + "spec": { + "data": { "kind": "QueryGroup", "spec": { "queries": [{ "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (gen_ai_request_model) (rate(airi_billing_flux_unbilled_total{service_name=~\"$service\", deployment_environment=~\"$env\", gen_ai_request_model!=\"\"}[$__rate_interval]))", "legendFormat": "{{gen_ai_request_model}}" }, "version": "v0" }, "refId": "A" } }], "queryOptions": {}, "transformations": [] } }, + "description": "Flux that should have been debited but wasn't (debit failure after streaming response sent). Real revenue leak — anything above 0 sustained should page.", + "id": 62, + "links": [], + "title": "Flux Unbilled (revenue leak)", + "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 30, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "red", "value": 0.001 }] }, "unit": "short" }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.0.0-23630096546" } + } + }, + "panel-63": { + "kind": "Panel", + "spec": { + "data": { "kind": "QueryGroup", "spec": { "queries": [{ "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (gen_ai_request_model, stage) (rate(airi_gen_ai_stream_interrupted_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))", "legendFormat": "{{gen_ai_request_model}} ({{stage}})" }, "version": "v0" }, "refId": "A" } }], "queryOptions": {}, "transformations": [] } }, + "description": "Streaming responses that died mid-flight. `before_first_chunk` = upstream blew up before any token; `mid_stream` = partial delivery.", + "id": 63, + "links": [], + "title": "Stream Interruptions", + "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "red", "value": 0.5 }] }, "unit": "ops" }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.0.0-23630096546" } + } + }, + "panel-64": { + "kind": "Panel", + "spec": { + "data": { "kind": "QueryGroup", "spec": { "queries": [{ "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "histogram_quantile(0.95, sum by (le, gen_ai_request_model) (rate(gen_ai_client_first_token_duration_seconds_bucket{service_name=~\"$service\", deployment_environment=~\"$env\", gen_ai_request_model!=\"\"}[$__rate_interval])))", "legendFormat": "{{gen_ai_request_model}}" }, "version": "v0" }, "refId": "A" } }], "queryOptions": {}, "transformations": [] } }, + "description": "Time from request start to first streamed token (TTFB). Tracks streaming chat experience independently from total operation duration.", + "id": 64, + "links": [], + "title": "First Token P95 Latency", + "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "yellow", "value": 1 }, { "color": "red", "value": 3 }] }, "unit": "s" }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.0.0-23630096546" } + } + }, + "panel-65": { + "kind": "Panel", + "spec": { + "data": { "kind": "QueryGroup", "spec": { "queries": [{ "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (model) (rate(airi_billing_tts_chars_total{service_name=~\"$service\", deployment_environment=~\"$env\", model!=\"\"}[$__rate_interval]))", "legendFormat": "{{model}}" }, "version": "v0" }, "refId": "A" } }], "queryOptions": {}, "transformations": [] } }, + "description": "TTS input characters processed per second — billing base before whole-Flux rounding. Spikes here without proportional flux_consumed = debt accumulating in Redis.", + "id": 65, + "links": [], + "title": "TTS Chars Rate", + "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }] }, "unit": "short" }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.0.0-23630096546" } + } + }, + "panel-66": { + "kind": "Panel", + "spec": { + "data": { "kind": "QueryGroup", "spec": { "queries": [{ "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (meter, reason) (rate(airi_billing_tts_preflight_rejections_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))", "legendFormat": "{{meter}} ({{reason}})" }, "version": "v0" }, "refId": "A" } }], "queryOptions": {}, "transformations": [] } }, + "description": "Pre-flight rejections from flux-meter assertCanAfford. Volume here = users hitting paywall before upstream call.", + "id": 66, + "links": [], + "title": "TTS Preflight Rejections", + "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }] }, "unit": "ops" }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.0.0-23630096546" } + } + }, + "panel-67": { + "kind": "Panel", + "spec": { + "data": { "kind": "QueryGroup", "spec": { "queries": [{ "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (template) (rate(airi_email_send_total{service_name=~\"$service\", deployment_environment=~\"$env\", template!=\"\"}[$__rate_interval]))", "legendFormat": "{{template}}" }, "version": "v0" }, "refId": "A" } }], "queryOptions": {}, "transformations": [] } }, + "description": "Resend accepted emails by template (verification / password_reset / magic_link / change_email / delete_account).", + "id": 67, + "links": [], + "title": "Email Send Rate", + "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }] }, "unit": "ops" }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.0.0-23630096546" } + } + }, + "panel-68": { + "kind": "Panel", + "spec": { + "data": { "kind": "QueryGroup", "spec": { "queries": [{ "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "100 * sum(rate(airi_email_failures_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval])) / clamp_min(sum(rate(airi_email_send_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval])) + sum(rate(airi_email_failures_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval])), 1)", "legendFormat": "fail %" }, "version": "v0" }, "refId": "A" } }], "queryOptions": {}, "transformations": [] } }, + "description": "Email failures / total attempts. >5% means Resend / DNS / suppression list issues blocking auth flows.", + "id": 68, + "links": [], + "title": "Email Failure Rate %", + "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "yellow", "value": 1 }, { "color": "red", "value": 5 }] }, "unit": "percent" }, "overrides": [] }, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true } }, "version": "13.0.0-23630096546" } + } + }, + "panel-69": { + "kind": "Panel", + "spec": { + "data": { "kind": "QueryGroup", "spec": { "queries": [{ "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (route, key_type) (rate(airi_rate_limit_blocked_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))", "legendFormat": "{{route}} ({{key_type}})" }, "version": "v0" }, "refId": "A" } }], "queryOptions": {}, "transformations": [] } }, + "description": "Requests denied by rate limiter, broken down by route + key type (user vs ip). Sustained ip-keyed traffic = unauthenticated abuse / scraping.", + "id": 69, + "links": [], + "title": "Rate Limit Blocked", + "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "red", "value": 1 }] }, "unit": "ops" }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.0.0-23630096546" } + } } }, "layout": { @@ -1799,15 +2019,35 @@ "kind": "GridLayout", "spec": { "items": [ - { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-31" }, "height": 8, "width": 8, "x": 0, "y": 0 } }, - { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-32" }, "height": 8, "width": 8, "x": 8, "y": 0 } }, - { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-33" }, "height": 8, "width": 8, "x": 16, "y": 0 } } + { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-31" }, "height": 8, "width": 6, "x": 0, "y": 0 } }, + { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-34" }, "height": 8, "width": 6, "x": 6, "y": 0 } }, + { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-32" }, "height": 8, "width": 6, "x": 12, "y": 0 } }, + { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-33" }, "height": 8, "width": 6, "x": 18, "y": 0 } } ] } }, "title": "Auth & Users" } }, + { + "kind": "RowsLayoutRow", + "spec": { + "collapse": false, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-50" }, "height": 8, "width": 6, "x": 0, "y": 0 } }, + { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-51" }, "height": 8, "width": 9, "x": 6, "y": 0 } }, + { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-52" }, "height": 8, "width": 9, "x": 15, "y": 0 } }, + { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-53" }, "height": 8, "width": 12, "x": 0, "y": 8 } }, + { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-54" }, "height": 8, "width": 12, "x": 12, "y": 8 } } + ] + } + }, + "title": "Engagement" + } + }, { "kind": "RowsLayoutRow", "spec": { @@ -1819,14 +2059,34 @@ { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-40" }, "height": 8, "width": 8, "x": 0, "y": 0 } }, { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-41" }, "height": 8, "width": 8, "x": 8, "y": 0 } }, { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-44" }, "height": 8, "width": 8, "x": 16, "y": 0 } }, - { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-42" }, "height": 8, "width": 12, "x": 0, "y": 8 } }, - { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-43" }, "height": 8, "width": 12, "x": 12, "y": 8 } } + { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-42" }, "height": 8, "width": 8, "x": 0, "y": 8 } }, + { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-43" }, "height": 8, "width": 8, "x": 8, "y": 8 } }, + { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-60" }, "height": 8, "width": 8, "x": 16, "y": 8 } } ] } }, "title": "Business Metrics" } }, + { + "kind": "RowsLayoutRow", + "spec": { + "collapse": false, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-57" }, "height": 8, "width": 6, "x": 0, "y": 0 } }, + { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-58" }, "height": 8, "width": 6, "x": 6, "y": 0 } }, + { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-55" }, "height": 8, "width": 12, "x": 12, "y": 0 } }, + { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-56" }, "height": 8, "width": 12, "x": 0, "y": 8 } }, + { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-61" }, "height": 8, "width": 12, "x": 12, "y": 8 } } + ] + } + }, + "title": "Stripe Detail" + } + }, { "kind": "RowsLayoutRow", "spec": { @@ -1859,15 +2119,37 @@ { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-20" }, "height": 8, "width": 12, "x": 0, "y": 8 } }, { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-21" }, "height": 8, "width": 12, "x": 12, "y": 8 } }, { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-22" }, "height": 8, "width": 12, "x": 0, "y": 16 } }, - { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-23" }, "height": 8, "width": 12, "x": 12, "y": 16 } }, - { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-24" }, "height": 8, "width": 12, "x": 0, "y": 24 } }, - { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-25" }, "height": 8, "width": 12, "x": 12, "y": 24 } } + { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-64" }, "height": 8, "width": 12, "x": 12, "y": 16 } }, + { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-23" }, "height": 8, "width": 12, "x": 0, "y": 24 } }, + { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-62" }, "height": 8, "width": 12, "x": 12, "y": 24 } }, + { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-63" }, "height": 8, "width": 12, "x": 0, "y": 32 } }, + { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-65" }, "height": 8, "width": 12, "x": 12, "y": 32 } }, + { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-24" }, "height": 8, "width": 12, "x": 0, "y": 40 } }, + { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-25" }, "height": 8, "width": 12, "x": 12, "y": 40 } } ] } }, "title": "LLM Gateway" } }, + { + "kind": "RowsLayoutRow", + "spec": { + "collapse": false, + "layout": { + "kind": "GridLayout", + "spec": { + "items": [ + { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-68" }, "height": 8, "width": 6, "x": 0, "y": 0 } }, + { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-67" }, "height": 8, "width": 18, "x": 6, "y": 0 } }, + { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-69" }, "height": 8, "width": 12, "x": 0, "y": 8 } }, + { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-66" }, "height": 8, "width": 12, "x": 12, "y": 8 } } + ] + } + }, + "title": "Reliability" + } + }, { "kind": "RowsLayoutRow", "spec": { diff --git a/apps/server/src/app.ts b/apps/server/src/app.ts index a5b4ba162..bfb5ba828 100644 --- a/apps/server/src/app.ts +++ b/apps/server/src/app.ts @@ -191,6 +191,7 @@ export async function buildApp(deps: AppDeps) { db: deps.db, env: deps.env, configKV: deps.configKV, + rateLimitMetrics: deps.otel?.rateLimit, })) /** @@ -211,7 +212,7 @@ export async function buildApp(deps: AppDeps) { /** * V1 routes for official provider. */ - .route('/api/v1/openai', createV1CompletionsRoutes(deps.fluxService, deps.billingService, deps.configKV, deps.requestLogService, deps.ttsMeter, deps.redis, deps.env, deps.otel?.genAi)) + .route('/api/v1/openai', createV1CompletionsRoutes(deps.fluxService, deps.billingService, deps.configKV, deps.requestLogService, deps.ttsMeter, deps.redis, deps.env, deps.otel?.genAi, deps.otel?.revenue, deps.otel?.rateLimit)) /** * Flux routes. @@ -221,7 +222,7 @@ export async function buildApp(deps: AppDeps) { /** * Stripe routes. */ - .route('/api/v1/stripe', createStripeRoutes(deps.fluxService, deps.stripeService, deps.billingService, deps.configKV, deps.env, deps.redis, deps.otel?.revenue)) + .route('/api/v1/stripe', createStripeRoutes(deps.fluxService, deps.stripeService, deps.billingService, deps.configKV, deps.env, deps.redis, deps.otel?.revenue, deps.otel?.rateLimit)) /** * Admin routes — guarded by `ADMIN_EMAILS` allowlist + verified email. @@ -329,12 +330,12 @@ export async function createApp() { }) const emailService = injeca.provide('services:email', { - dependsOn: { env: parsedEnv }, + dependsOn: { env: parsedEnv, otel }, build: ({ dependsOn }) => createEmailService({ apiKey: dependsOn.env.RESEND_API_KEY, fromEmail: dependsOn.env.RESEND_FROM_EMAIL, fromName: dependsOn.env.RESEND_FROM_NAME, - }), + }, undefined, dependsOn.otel?.email), }) const characterService = injeca.provide('services:characters', { @@ -433,7 +434,7 @@ export async function createApp() { }) const ttsMeter = injeca.provide('services:ttsMeter', { - dependsOn: { redis, billingService, configKV }, + dependsOn: { redis, billingService, configKV, otel }, build: ({ dependsOn }) => createFluxMeter(dependsOn.redis, dependsOn.billingService, { name: 'tts', // Lazy config read: missing FLUX_PER_1K_CHARS_TTS surfaces as a @@ -447,7 +448,7 @@ export async function createApp() { debtTtlSeconds: ttl, } }, - }), + }, dependsOn.otel?.revenue), }) await injeca.start() diff --git a/apps/server/src/libs/otel.ts b/apps/server/src/libs/otel.ts index f845f28d1..d386ca56e 100644 --- a/apps/server/src/libs/otel.ts +++ b/apps/server/src/libs/otel.ts @@ -19,6 +19,16 @@ import { BatchSpanProcessor, ParentBasedSampler, TraceIdRatioBasedSampler } from import { ATTR_SERVICE_NAME, ATTR_SERVICE_VERSION } from '@opentelemetry/semantic-conventions' import { + METRIC_AIRI_EMAIL_DURATION, + METRIC_AIRI_EMAIL_FAILURES, + METRIC_AIRI_EMAIL_SEND, + METRIC_AIRI_FLUX_CREDITED, + METRIC_AIRI_FLUX_UNBILLED, + METRIC_AIRI_GEN_AI_STREAM_INTERRUPTED, + METRIC_AIRI_RATE_LIMIT_BLOCKED, + METRIC_AIRI_STRIPE_REVENUE, + METRIC_AIRI_TTS_CHARS, + METRIC_AIRI_TTS_PREFLIGHT_REJECTIONS, METRIC_AUTH_ATTEMPTS, METRIC_AUTH_FAILURES, METRIC_CHARACTER_CREATED, @@ -27,6 +37,7 @@ import { METRIC_CHAT_MESSAGES, METRIC_FLUX_CONSUMED, METRIC_FLUX_INSUFFICIENT_BALANCE, + METRIC_GEN_AI_CLIENT_FIRST_TOKEN_DURATION, METRIC_GEN_AI_CLIENT_OPERATION_COUNT, METRIC_GEN_AI_CLIENT_OPERATION_DURATION, METRIC_GEN_AI_CLIENT_TOKEN_USAGE_INPUT, @@ -77,7 +88,12 @@ export interface RevenueMetrics { stripePaymentFailed: Counter stripeSubscriptionEvent: Counter stripeEvents: Counter + stripeRevenue: Counter fluxInsufficientBalance: Counter + fluxCredited: Counter + fluxUnbilled: Counter + ttsChars: Counter + ttsPreflightRejections: Counter } export interface GenAiMetrics { @@ -86,6 +102,18 @@ export interface GenAiMetrics { tokenUsageInput: Counter tokenUsageOutput: Counter fluxConsumed: Counter + firstTokenDuration: Histogram + streamInterrupted: Counter +} + +export interface EmailMetrics { + send: Counter + failures: Counter + duration: Histogram +} + +export interface RateLimitMetrics { + blocked: Counter } // NOTICE: Database metrics (db.client.operation.duration, redis.client.command.duration) were @@ -100,6 +128,8 @@ export interface OtelInstance { engagement: EngagementMetrics revenue: RevenueMetrics genAi: GenAiMetrics + email: EmailMetrics + rateLimit: RateLimitMetrics shutdown: () => Promise } @@ -256,9 +286,25 @@ export function initOtel(env: Env): OtelInstance | undefined { stripeEvents: meter.createCounter(METRIC_STRIPE_EVENTS, { description: 'Number of Stripe webhook events processed', }), + stripeRevenue: meter.createCounter(METRIC_AIRI_STRIPE_REVENUE, { + description: 'Stripe revenue in smallest currency unit (e.g. cents)', + unit: 'minor_unit', + }), fluxInsufficientBalance: meter.createCounter(METRIC_FLUX_INSUFFICIENT_BALANCE, { description: 'Number of insufficient flux balance errors', }), + fluxCredited: meter.createCounter(METRIC_AIRI_FLUX_CREDITED, { + description: 'Total flux credited to user balances, by source', + }), + fluxUnbilled: meter.createCounter(METRIC_AIRI_FLUX_UNBILLED, { + description: 'Flux that should have been debited but was not (revenue leak)', + }), + ttsChars: meter.createCounter(METRIC_AIRI_TTS_CHARS, { + description: 'TTS input characters processed (billing base unit)', + }), + ttsPreflightRejections: meter.createCounter(METRIC_AIRI_TTS_PREFLIGHT_REJECTIONS, { + description: 'Pre-flight rejections from flux-meter assertCanAfford', + }), } // GenAI metrics (semconv: gen_ai.client.*) @@ -279,8 +325,76 @@ export function initOtel(env: Env): OtelInstance | undefined { fluxConsumed: meter.createCounter(METRIC_FLUX_CONSUMED, { description: 'Total flux consumed', }), + firstTokenDuration: meter.createHistogram(METRIC_GEN_AI_CLIENT_FIRST_TOKEN_DURATION, { + description: 'Time from request start to first streamed token (TTFB for streaming)', + unit: 's', + }), + streamInterrupted: meter.createCounter(METRIC_AIRI_GEN_AI_STREAM_INTERRUPTED, { + description: 'Streaming responses interrupted before completion', + }), } + const email: EmailMetrics = { + send: meter.createCounter(METRIC_AIRI_EMAIL_SEND, { + description: 'Transactional emails accepted by Resend', + }), + failures: meter.createCounter(METRIC_AIRI_EMAIL_FAILURES, { + description: 'Transactional email send failures', + }), + duration: meter.createHistogram(METRIC_AIRI_EMAIL_DURATION, { + description: 'Email provider call duration', + unit: 's', + }), + } + + const rateLimit: RateLimitMetrics = { + blocked: meter.createCounter(METRIC_AIRI_RATE_LIMIT_BLOCKED, { + description: 'Requests blocked by rate limiter', + }), + } + + // NOTICE: + // OTel SDK only emits a Counter time series after .add() runs the first time. + // Without this priming step, low-traffic counters (auth_failures_total, + // stripe_*_total, payment_failed, ...) never appear in Prometheus / Grafana + // until an event happens — making panels look broken on fresh deploys and + // making absence-based alerts impossible to author. add(0) registers the + // series with a baseline of 0 without distorting any rates. + // Removal condition: OTel SDK changes default to register Counters at create + // time (https://github.com/open-telemetry/opentelemetry-specification/issues/2298). + function primeCounter(counter: Counter): void { + counter.add(0) + } + primeCounter(auth.attempts) + primeCounter(auth.failures) + primeCounter(auth.userRegistered) + primeCounter(auth.userLogin) + primeCounter(engagement.chatMessages) + primeCounter(engagement.characterCreated) + primeCounter(engagement.characterDeleted) + primeCounter(engagement.characterEngagement) + primeCounter(engagement.wsMessagesSent) + primeCounter(engagement.wsMessagesReceived) + primeCounter(revenue.stripeCheckoutCreated) + primeCounter(revenue.stripeCheckoutCompleted) + primeCounter(revenue.stripePaymentFailed) + primeCounter(revenue.stripeSubscriptionEvent) + primeCounter(revenue.stripeEvents) + primeCounter(revenue.stripeRevenue) + primeCounter(revenue.fluxInsufficientBalance) + primeCounter(revenue.fluxCredited) + primeCounter(revenue.fluxUnbilled) + primeCounter(revenue.ttsChars) + primeCounter(revenue.ttsPreflightRejections) + primeCounter(genAi.operationCount) + primeCounter(genAi.tokenUsageInput) + primeCounter(genAi.tokenUsageOutput) + primeCounter(genAi.fluxConsumed) + primeCounter(genAi.streamInterrupted) + primeCounter(email.send) + primeCounter(email.failures) + primeCounter(rateLimit.blocked) + // Graceful shutdown const shutdown = async () => { try { @@ -299,6 +413,8 @@ export function initOtel(env: Env): OtelInstance | undefined { engagement, revenue, genAi, + email, + rateLimit, shutdown, } } diff --git a/apps/server/src/middlewares/rate-limit.ts b/apps/server/src/middlewares/rate-limit.ts index 4e38c1d1d..1aaa692a2 100644 --- a/apps/server/src/middlewares/rate-limit.ts +++ b/apps/server/src/middlewares/rate-limit.ts @@ -1,5 +1,6 @@ import type { Context } from 'hono' +import type { RateLimitMetrics } from '../libs/otel' import type { HonoEnv } from '../types/hono' import { getConnInfo } from '@hono/node-server/conninfo' @@ -12,6 +13,20 @@ interface RateLimitOptions { windowSec: number /** Key generator: extracts a unique identifier from the request */ keyGenerator?: (c: Context) => string + /** + * Optional metrics handle. When provided, blocked requests increment + * `airi_rate_limit_blocked_total{route, key_type, limit}`. + * `key_type` reflects whether the limiter keyed off authenticated user id + * or remote IP — important for distinguishing logged-in abuse from + * anonymous scraping. + */ + metrics?: RateLimitMetrics | null + /** + * Stable label for the route this limiter guards (e.g. `auth.api`, + * `openai.completions`, `stripe.checkout`). Avoids high-cardinality URL + * paths in metric labels. + */ + routeLabel?: string } /** @@ -19,6 +34,18 @@ interface RateLimitOptions { * Uses in-memory store by default (single-instance). */ export function rateLimiter(opts: RateLimitOptions) { + const keyGen = opts.keyGenerator + ?? ((c) => { + const userId = c.get('user')?.id + if (userId) + return userId + + // NOTICE: prefer hono conninfo (uses underlying socket address) over + // x-forwarded-for which can be spoofed. Falls back to header then 'anonymous'. + const info = getConnInfo(c) + return info.remote?.address ?? c.req.header('x-forwarded-for') ?? 'anonymous' + }) + return createRateLimiter({ windowMs: opts.windowSec * 1000, limit: opts.max, @@ -26,16 +53,17 @@ export function rateLimiter(opts: RateLimitOptions) { // `RateLimit-*` header set. `draft-7`/`draft-8` switch to newer combined // header formats that are easier to break in existing clients and proxies. standardHeaders: 'draft-6', - keyGenerator: opts.keyGenerator - ?? ((c) => { - const userId = c.get('user')?.id - if (userId) - return userId - - // NOTICE: prefer hono conninfo (uses underlying socket address) over - // x-forwarded-for which can be spoofed. Falls back to header then 'anonymous'. - const info = getConnInfo(c) - return info.remote?.address ?? c.req.header('x-forwarded-for') ?? 'anonymous' - }), + keyGenerator: keyGen, + handler: (c) => { + // Record before producing the 429 response so the time series captures + // every block, even when the response shape later changes. + const keyType = c.get('user')?.id ? 'user' : 'ip' + opts.metrics?.blocked.add(1, { + route: opts.routeLabel ?? 'unknown', + key_type: keyType, + limit: String(opts.max), + }) + return c.json({ error: 'TOO_MANY_REQUESTS', message: 'Too many requests' }, 429) + }, }) } diff --git a/apps/server/src/routes/auth/index.ts b/apps/server/src/routes/auth/index.ts index 66f30147a..454f8c14d 100644 --- a/apps/server/src/routes/auth/index.ts +++ b/apps/server/src/routes/auth/index.ts @@ -1,6 +1,7 @@ import type { AuthInstance } from '../../libs/auth' import type { Database } from '../../libs/db' import type { Env } from '../../libs/env' +import type { RateLimitMetrics } from '../../libs/otel' import type { ConfigKVService } from '../../services/config-kv' import type { HonoEnv } from '../../types/hono' @@ -31,6 +32,7 @@ export interface AuthRoutesDeps { db: Database env: Env configKV: ConfigKVService + rateLimitMetrics?: RateLimitMetrics | null } /** @@ -129,6 +131,8 @@ export async function createAuthRoutes(deps: AuthRoutesDeps) { max: await deps.configKV.getOrThrow('AUTH_RATE_LIMIT_MAX'), windowSec: await deps.configKV.getOrThrow('AUTH_RATE_LIMIT_WINDOW_SEC'), keyGenerator: c => c.req.header('x-forwarded-for') ?? c.req.header('x-real-ip') ?? 'unknown', + metrics: deps.rateLimitMetrics, + routeLabel: 'auth.api', })) .use('/api/auth/oauth2/authorize', async (c, next) => { await ensureDynamicFirstPartyRedirectUri(deps.db, c.req.raw) diff --git a/apps/server/src/routes/openai/v1/index.ts b/apps/server/src/routes/openai/v1/index.ts index 400a83da3..c6df04c5a 100644 --- a/apps/server/src/routes/openai/v1/index.ts +++ b/apps/server/src/routes/openai/v1/index.ts @@ -2,7 +2,7 @@ import type { Context } from 'hono' import type Redis from 'ioredis' import type { Env } from '../../../libs/env' -import type { GenAiMetrics } from '../../../libs/otel' +import type { GenAiMetrics, RateLimitMetrics, RevenueMetrics } from '../../../libs/otel' import type { UsageInfo } from '../../../services/billing/billing' import type { BillingService } from '../../../services/billing/billing-service' import type { FluxMeter } from '../../../services/billing/flux-meter' @@ -81,7 +81,18 @@ function getLlmMetricAttributes(opts: { model: string, type: string, status: num } } -export function createV1CompletionsRoutes(fluxService: FluxService, billingService: BillingService, configKV: ConfigKVService, requestLogService: RequestLogService, ttsMeter: FluxMeter, redis: Redis, env: Env, genAi?: GenAiMetrics | null) { +export function createV1CompletionsRoutes( + fluxService: FluxService, + billingService: BillingService, + configKV: ConfigKVService, + requestLogService: RequestLogService, + ttsMeter: FluxMeter, + redis: Redis, + env: Env, + genAi?: GenAiMetrics | null, + revenue?: RevenueMetrics | null, + rateLimitMetrics?: RateLimitMetrics | null, +) { const logger = useLogger('v1-completions').useGlobalConfig() // TODO: Extract this compat route into smaller facades/modules. // It currently mixes auth, rate limiting, proxying, billing, telemetry, and event publishing in one transport layer entrypoint. @@ -173,6 +184,11 @@ export function createV1CompletionsRoutes(fluxService: FluxService, billingServi let tailBuffer = '' let streamCompleted = false let streamInterrupted = false + // First-chunk timestamp for gen_ai.client.first_token.duration. Latched + // on the first byte from upstream — captures perceived "time to first + // token" for streaming clients. NaN until the first chunk lands so + // `Number.isFinite` gates the histogram record. + let firstChunkAt = Number.NaN // Process stream in background ;(async () => { @@ -183,6 +199,13 @@ export function createV1CompletionsRoutes(fluxService: FluxService, billingServi streamCompleted = true break } + if (!Number.isFinite(firstChunkAt)) { + firstChunkAt = Date.now() + genAi?.firstTokenDuration.record((firstChunkAt - startedAt) / 1000, { + [GEN_AI_ATTR_REQUEST_MODEL]: requestModel, + [GEN_AI_ATTR_OPERATION_NAME]: 'chat', + }) + } await writer.write(value) const text = decoder.decode(value, { stream: true }) tailBuffer = (tailBuffer + text).slice(-2048) @@ -192,6 +215,12 @@ export function createV1CompletionsRoutes(fluxService: FluxService, billingServi streamInterrupted = true span.setStatus({ code: SpanStatusCode.ERROR, message: 'Gateway stream interrupted' }) span.setAttribute(AIRI_ATTR_GEN_AI_STREAM_INTERRUPTED, true) + // Counter so alerts/dashboards can fire on interrupted streams; the + // span attribute alone only shows up in trace search, not metrics. + genAi?.streamInterrupted.add(1, { + [GEN_AI_ATTR_REQUEST_MODEL]: requestModel, + stage: Number.isFinite(firstChunkAt) ? 'mid_stream' : 'before_first_chunk', + }) try { await writer.abort(err) @@ -255,7 +284,16 @@ export function createV1CompletionsRoutes(fluxService: FluxService, billingServi }) actualCharged = fluxConsumed } - catch (err) { logger.withError(err).withFields({ userId: user.id, fluxConsumed, requestId }).error('Failed to debit flux after streaming — unpaid usage') } + catch (err) { + // Flux that should have been collected but wasn't. Real revenue leak — + // counter so on-call can alert and reconcile. + revenue?.fluxUnbilled.add(fluxConsumed, { + [GEN_AI_ATTR_REQUEST_MODEL]: requestModel, + reason: 'debit_failed', + stage: 'streaming', + }) + logger.withError(err).withFields({ userId: user.id, fluxConsumed, requestId }).error('Failed to debit flux after streaming — unpaid usage') + } recordRequestLog({ userId: user.id, @@ -449,7 +487,7 @@ export function createV1CompletionsRoutes(fluxService: FluxService, billingServi const ttsGuard = configGuard(configKV, ['FLUX_PER_1K_CHARS_TTS'], 'TTS service is not available yet') // 60 requests per minute per user for LLM completions - const completionsRateLimit = rateLimiter({ max: 60, windowSec: 60 }) + const completionsRateLimit = rateLimiter({ max: 60, windowSec: 60, metrics: rateLimitMetrics, routeLabel: 'openai.completions' }) return new Hono() .use('*', authGuard) diff --git a/apps/server/src/routes/stripe/index.ts b/apps/server/src/routes/stripe/index.ts index 899285bef..aa0c8c7ce 100644 --- a/apps/server/src/routes/stripe/index.ts +++ b/apps/server/src/routes/stripe/index.ts @@ -1,7 +1,7 @@ import type Redis from 'ioredis' import type { Env } from '../../libs/env' -import type { RevenueMetrics } from '../../libs/otel' +import type { RateLimitMetrics, RevenueMetrics } from '../../libs/otel' import type { BillingService } from '../../services/billing/billing-service' import type { ConfigKVService } from '../../services/config-kv' import type { FluxService } from '../../services/flux' @@ -49,6 +49,7 @@ export function createStripeRoutes( env: Env, redis: Redis, metrics?: RevenueMetrics | null, + rateLimitMetrics?: RateLimitMetrics | null, ) { const stripe = env.STRIPE_SECRET_KEY ? new Stripe(env.STRIPE_SECRET_KEY) : null @@ -116,7 +117,7 @@ export function createStripeRoutes( } })) }) - .post('/checkout', authGuard, rateLimiter({ max: 10, windowSec: 60 }), async (c) => { + .post('/checkout', authGuard, rateLimiter({ max: 10, windowSec: 60, metrics: rateLimitMetrics, routeLabel: 'stripe.checkout' }), async (c) => { const fluxProductId = await configKV.getOptional('STRIPE_FLUX_PRODUCT_ID') if (!stripe || !fluxProductId) throw createServiceUnavailableError('Stripe is not configured', 'STRIPE_NOT_CONFIGURED') @@ -286,6 +287,14 @@ export function createStripeRoutes( case 'checkout.session.completed': { await handleCheckoutSessionCompleted(event.id, event.data.object, fluxService, stripeService, billingService) metrics?.stripeCheckoutCompleted.add(1) + // Revenue capture in smallest currency unit (e.g. cents). + // Cross-currency aggregation is meaningless, so always group by `currency` in queries. + if (event.data.object.amount_total != null && event.data.object.currency) { + metrics?.stripeRevenue.add(event.data.object.amount_total, { + currency: event.data.object.currency, + source: 'checkout', + }) + } break } case 'customer.created': @@ -308,6 +317,12 @@ export function createStripeRoutes( if (event.type === 'invoice.payment_failed') { metrics?.stripePaymentFailed.add(1) } + if (event.type === 'invoice.paid' && event.data.object.amount_paid && event.data.object.currency) { + metrics?.stripeRevenue.add(event.data.object.amount_paid, { + currency: event.data.object.currency, + source: 'invoice', + }) + } break } } diff --git a/apps/server/src/scripts/otel-smoke.mjs b/apps/server/src/scripts/otel-smoke.mjs new file mode 100644 index 000000000..458c76ecc --- /dev/null +++ b/apps/server/src/scripts/otel-smoke.mjs @@ -0,0 +1,64 @@ +/** + * OTel metric-registration smoke test. + * + * Verifies every Counter created by `initOtel` is primed (.add(0)) so the + * Prometheus exporter sees the time series at boot, not just after the first + * real event. Without priming, low-traffic counters + * (auth_failures_total, stripe_*_total, payment_failed, ...) never appear in + * Grafana until an event happens — making panels look broken on fresh deploys + * and absence-based alerts impossible to author. + * + * Histograms (gen_ai.client.first_token.duration, airi.email.duration, ...) + * are intentionally NOT in the output — they only register on first .record(). + * + * Usage: + * pnpm -F @proj-airi/server exec node --import tsx ./src/scripts/otel-smoke.mjs + */ +import { env, exit } from 'node:process' + +import { metrics } from '@opentelemetry/api' +import { AggregationTemporality, InMemoryMetricExporter, MeterProvider, PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics' + +const exporter = new InMemoryMetricExporter(AggregationTemporality.CUMULATIVE) +const reader = new PeriodicExportingMetricReader({ exporter, exportIntervalMillis: 100 }) +const provider = new MeterProvider({ readers: [reader] }) +metrics.setGlobalMeterProvider(provider) + +// Smoke-test env: stand in for required vars so parseEnv passes; OTLP endpoint +// is set to a localhost URL that won't actually receive anything because we +// flush via the in-memory reader. +env.DATABASE_URL ??= 'postgres://test' +env.REDIS_URL ??= 'redis://test' +env.BETTER_AUTH_SECRET ??= 'test' +env.AUTH_GOOGLE_CLIENT_ID ??= 'test' +env.AUTH_GOOGLE_CLIENT_SECRET ??= 'test' +env.AUTH_GITHUB_CLIENT_ID ??= 'test' +env.AUTH_GITHUB_CLIENT_SECRET ??= 'test' +env.GATEWAY_BASE_URL ??= 'http://test' +env.DEFAULT_CHAT_MODEL ??= 'test' +env.DEFAULT_TTS_MODEL ??= 'test' +env.OTEL_EXPORTER_OTLP_ENDPOINT ??= 'http://localhost:4318' + +const { initOtel } = await import('../libs/otel.ts') +const { parseEnv } = await import('../libs/env.ts') + +const parsed = parseEnv(env) +const inst = initOtel(parsed) +if (!inst) { + console.error('initOtel returned undefined') + exit(1) +} + +await new Promise(r => setTimeout(r, 300)) +await reader.forceFlush() +const exported = exporter.getMetrics() +const names = [] +for (const rm of exported) { + for (const sm of rm.scopeMetrics) { + for (const m of sm.metrics) names.push(m.descriptor.name) + } +} +console.info('REGISTERED:') +for (const n of [...new Set(names)].sort()) console.info(` ${n}`) +await inst.shutdown() +exit(0) diff --git a/apps/server/src/services/billing/billing-service.ts b/apps/server/src/services/billing/billing-service.ts index fb10bea45..e40505311 100644 --- a/apps/server/src/services/billing/billing-service.ts +++ b/apps/server/src/services/billing/billing-service.ts @@ -260,6 +260,7 @@ export function createBillingService( } await updateRedisCache(input.userId, txResult.balanceAfter) + metrics?.fluxCredited.add(input.amount, { source: input.source, type: ledgerType }) logger.withFields({ userId: input.userId, amount: input.amount, balance: txResult.balanceAfter }).log('Credited flux') return txResult @@ -336,6 +337,7 @@ export function createBillingService( if (txResult.applied && txResult.balanceAfter != null) { await updateRedisCache(input.userId, txResult.balanceAfter) + metrics?.fluxCredited.add(input.fluxAmount, { source: 'stripe.checkout', type: 'credit' }) } return txResult @@ -410,6 +412,7 @@ export function createBillingService( if (txResult.applied && txResult.balanceAfter != null) { await updateRedisCache(input.userId, txResult.balanceAfter) + metrics?.fluxCredited.add(input.fluxAmount, { source: 'stripe.invoice', type: 'credit' }) } return txResult diff --git a/apps/server/src/services/billing/flux-meter.ts b/apps/server/src/services/billing/flux-meter.ts index 5854bd1e3..7452c8096 100644 --- a/apps/server/src/services/billing/flux-meter.ts +++ b/apps/server/src/services/billing/flux-meter.ts @@ -1,5 +1,6 @@ import type Redis from 'ioredis' +import type { RevenueMetrics } from '../../libs/otel' import type { BillingService } from './billing-service' import { useLogger } from '@guiiai/logg' @@ -80,6 +81,7 @@ export function createFluxMeter( redis: Redis, billingService: BillingService, config: FluxMeterConfig, + metrics?: RevenueMetrics | null, ) { async function getRuntime(): Promise { const runtime = await config.resolveRuntime() @@ -119,8 +121,10 @@ export function createFluxMeter( // At minimum require the user can cover a single Flux crossing; avoids // letting zero-balance users accumulate indefinitely on the boundary. const required = Math.max(projectedFlux, currentBalance <= 0 ? 1 : 0) - if (currentBalance < required) + if (currentBalance < required) { + metrics?.ttsPreflightRejections.add(1, { meter: config.name, reason: 'insufficient_balance' }) throw createPaymentRequiredError('Insufficient flux') + } } /** @@ -132,6 +136,9 @@ export function createFluxMeter( if (!Number.isFinite(input.units) || input.units <= 0) return { fluxDebited: 0, debtAfter: await readDebt(input.userId), balanceAfter: input.currentBalance } + const modelLabel = typeof input.metadata?.model === 'string' ? input.metadata.model : 'unknown' + metrics?.ttsChars.add(input.units, { meter: config.name, model: modelLabel }) + const runtime = await getRuntime() const key = userFluxMeterDebtRedisKey(input.userId, config.name) const [fluxDebited, debtAfter] = await runScript(key, input.units, runtime) diff --git a/apps/server/src/services/email.ts b/apps/server/src/services/email.ts index 2fd6988e2..d46e9e164 100644 --- a/apps/server/src/services/email.ts +++ b/apps/server/src/services/email.ts @@ -1,5 +1,7 @@ import type { Logger } from '@guiiai/logg' +import type { EmailMetrics } from '../libs/otel' + import { useLogger } from '@guiiai/logg' import { errorMessageFrom } from '@moeru/std' import { Resend } from 'resend' @@ -92,7 +94,7 @@ function formatFrom(config: EmailConfig): string { * instead of silently dropping mail — Better Auth surfaces it back to the * caller so frontend can show a clear "email service not configured" error. */ -export function createEmailService(config: EmailConfig, logger: Logger = useLogger('email')): EmailService { +export function createEmailService(config: EmailConfig, logger: Logger = useLogger('email'), metrics?: EmailMetrics | null): EmailService { // NOTICE: // Construct Resend lazily so the server can boot in environments where the // RESEND_API_KEY is intentionally empty (e.g. local dev that never exercises @@ -118,7 +120,8 @@ export function createEmailService(config: EmailConfig, logger: Logger = useLogg const from = formatFrom(config) - async function send(payload: EmailPayload): Promise { + async function send(payload: EmailPayload, template: string = 'unknown'): Promise { + const startedAt = Date.now() try { const { error } = await getClient().emails.send({ from, @@ -130,8 +133,12 @@ export function createEmailService(config: EmailConfig, logger: Logger = useLogg if (error) { logger.withFields({ to: payload.to, subject: payload.subject, errorName: error.name }).error(error.message) + metrics?.failures.add(1, { template, error_name: error.name }) + metrics?.duration.record((Date.now() - startedAt) / 1000, { template, outcome: 'error' }) throw new ApiError(502, 'email/send_failed', error.message, { providerError: error.name }) } + metrics?.send.add(1, { template }) + metrics?.duration.record((Date.now() - startedAt) / 1000, { template, outcome: 'ok' }) } catch (error) { if (error instanceof ApiError) @@ -139,6 +146,8 @@ export function createEmailService(config: EmailConfig, logger: Logger = useLogg const message = errorMessageFrom(error) ?? 'Unknown email send error' logger.withFields({ to: payload.to, subject: payload.subject }).error(message) + metrics?.failures.add(1, { template, error_name: 'unhandled' }) + metrics?.duration.record((Date.now() - startedAt) / 1000, { template, outcome: 'error' }) throw new ApiError(502, 'email/send_failed', message) } } @@ -151,7 +160,7 @@ export function createEmailService(config: EmailConfig, logger: Logger = useLogg subject: 'Verify your email for Project AIRI', html: renderVerificationHtml(url), text: renderVerificationText(url), - }) + }, 'verification') }, async sendPasswordReset({ to, url }) { await send({ @@ -159,7 +168,7 @@ export function createEmailService(config: EmailConfig, logger: Logger = useLogg subject: 'Reset your Project AIRI password', html: renderPasswordResetHtml(url), text: renderPasswordResetText(url), - }) + }, 'password_reset') }, async sendMagicLink({ to, url }) { await send({ @@ -167,7 +176,7 @@ export function createEmailService(config: EmailConfig, logger: Logger = useLogg subject: 'Your Project AIRI sign-in link', html: renderMagicLinkHtml(url), text: renderMagicLinkText(url), - }) + }, 'magic_link') }, async sendChangeEmailConfirmation({ to, newEmail, url }) { await send({ @@ -175,7 +184,7 @@ export function createEmailService(config: EmailConfig, logger: Logger = useLogg subject: 'Confirm your new email address for Project AIRI', html: renderChangeEmailHtml(url, newEmail), text: renderChangeEmailText(url, newEmail), - }) + }, 'change_email') }, async sendDeleteAccountVerification({ to, url }) { await send({ @@ -183,7 +192,7 @@ export function createEmailService(config: EmailConfig, logger: Logger = useLogg subject: 'Confirm account deletion for Project AIRI', html: renderDeleteAccountHtml(url), text: renderDeleteAccountText(url), - }) + }, 'delete_account') }, } } diff --git a/apps/server/src/utils/observability.ts b/apps/server/src/utils/observability.ts index 109de0201..03c42bc1a 100644 --- a/apps/server/src/utils/observability.ts +++ b/apps/server/src/utils/observability.ts @@ -62,6 +62,27 @@ export const METRIC_GEN_AI_CLIENT_TOKEN_USAGE_INPUT = 'gen_ai.client.token.usage export const METRIC_GEN_AI_CLIENT_TOKEN_USAGE_OUTPUT = 'gen_ai.client.token.usage.output' export const METRIC_FLUX_CONSUMED = 'airi.billing.flux.consumed' +// AIRI billing — credit/debit visibility beyond raw consumption +export const METRIC_AIRI_FLUX_CREDITED = 'airi.billing.flux.credited' +export const METRIC_AIRI_FLUX_UNBILLED = 'airi.billing.flux.unbilled' +export const METRIC_AIRI_TTS_CHARS = 'airi.billing.tts.chars' +export const METRIC_AIRI_TTS_PREFLIGHT_REJECTIONS = 'airi.billing.tts.preflight_rejections' + +// AIRI revenue — actual money in (smallest currency unit, e.g. cents) +export const METRIC_AIRI_STRIPE_REVENUE = 'airi.stripe.revenue' + +// AIRI email — transactional delivery health +export const METRIC_AIRI_EMAIL_SEND = 'airi.email.send' +export const METRIC_AIRI_EMAIL_FAILURES = 'airi.email.failures' +export const METRIC_AIRI_EMAIL_DURATION = 'airi.email.duration' + +// AIRI rate limiting — abuse / attack visibility +export const METRIC_AIRI_RATE_LIMIT_BLOCKED = 'airi.rate_limit.blocked' + +// AIRI GenAI — stream quality +export const METRIC_AIRI_GEN_AI_STREAM_INTERRUPTED = 'airi.gen_ai.stream.interrupted' +export const METRIC_GEN_AI_CLIENT_FIRST_TOKEN_DURATION = 'gen_ai.client.first_token.duration' + // --------------------------------------------------------------------------- // Helpers // ---------------------------------------------------------------------------