From fe59f91c841eb9e6f6792e6b2a6503ac5caf1749 Mon Sep 17 00:00:00 2001 From: RainbowBird Date: Sat, 9 May 2026 00:23:02 +0800 Subject: [PATCH] refactor(server/flux): replace UpDownCounter with ObservableGauge for WebSocket connections - Updated the EngagementMetrics interface to use ObservableGauge for tracking active WebSocket connections. - Added detailed comments explaining the rationale for this change, highlighting the benefits of using a pull-based gauge over a delta-based counter. - Implemented the ObservableGauge in the createChatWsHandlers function, ensuring it accurately reflects the live count of active connections. - Removed the previous UpDownCounter logic to prevent issues with connection drift during process crashes or network interruptions. --- apps/server/docs/ai-context/README.md | 5 +- .../ai-context/observability-conventions.md | 72 +- .../docs/ai-context/observability-metrics.md | 157 + apps/server/instrumentation.mjs | 19 +- .../airi-server-overview-cloud.json | 4915 ++++++++++++++--- apps/server/src/libs/otel.ts | 32 +- apps/server/src/routes/chat-ws/index.ts | 13 +- 7 files changed, 4396 insertions(+), 817 deletions(-) create mode 100644 apps/server/docs/ai-context/observability-metrics.md diff --git a/apps/server/docs/ai-context/README.md b/apps/server/docs/ai-context/README.md index 1eaf38771..3f61ae61e 100644 --- a/apps/server/docs/ai-context/README.md +++ b/apps/server/docs/ai-context/README.md @@ -28,7 +28,9 @@ - `flux-meter.md` - Sub-Flux 计量服务（TTS/STT 等）的债务账本机制与复用指南 - `observability-conventions.md` - - traces / metrics 命名规则，标准 OTel 字段与 `airi.*` 自定义字段边界 + - traces / metrics 命名规则，标准 OTel 字段与 `airi.*` 自定义字段边界，SemconvStability 迁移、Counter priming、Dashboard 变量陷阱 +- `observability-metrics.md` + - 全量 metric 目录（按域分组：HTTP / Auth / Engagement / Revenue / GenAI / Email / Rate limit / Runtime），含名字、类型、Labels、落点 - `auth-and-oidc.md` - 认证与 OIDC Provider 架构、登录流程、trusted clients、踩坑记录 - `email-auth-resend.md` @@ -64,6 +66,7 @@ - 改扣费、充值、Stripe：先看 `billing-architecture.md` - 改 Flux 充值价格 / 多币种 / Stripe Product/Price：先看 `stripe-pricing.md` - 改 trace / metric attributes、OTel 命名：先看 `observability-conventions.md` +- 加新 metric / 找当前 metric 全量列表：先看 `observability-metrics.md` - 改认证、OIDC、登录流程：先看 `auth-and-oidc.md` - 改邮件 service / Better Auth 邮件 callback：先看 `email-auth-resend.md` - 改账号注销 / 业务 service 的 `deleteAllForUser`：先看 `account-deletion.md` diff --git a/apps/server/docs/ai-context/observability-conventions.md b/apps/server/docs/ai-context/observability-conventions.md index 42d56f35c..34d81ddf3 100644 --- a/apps/server/docs/ai-context/observability-conventions.md +++ b/apps/server/docs/ai-context/observability-conventions.md @@ -56,7 +56,7 @@ Redis 相关优先复用 instrumentation 自动产生的标准属性，不要重 - 仅 AIRI 内部存在的流式控制字段 - 临时调试但仍需要进入可观测系统的业务字段 -当前示例： +当前 attribute 示例： - `AIRI_ATTR_BILLING_FLUX_CONSUMED` - `AIRI_ATTR_GEN_AI_STREAM` @@ -66,6 +66,14 @@ Redis 相关优先复用 instrumentation 自动产生的标准属性，不要重 - `AIRI_ATTR_GEN_AI_INPUT_TEXT` - `AIRI_ATTR_GEN_AI_OUTPUT_TEXT` +当前 `airi.*` metric 命名空间（Prom 系列名见 [`observability-metrics.md`](./observability-metrics.md)）： + +- 计费：`airi.billing.flux.consumed` / `.credited` / `.unbilled` / `.tts.chars` / `.tts.preflight_rejections` +- 收入：`airi.stripe.revenue` +- 邮件：`airi.email.send` / `.failures` / `.duration` +- 限流：`airi.rate_limit.blocked` +- GenAI：`airi.gen_ai.stream.interrupted` + ## Metric Name 策略 当前 `apps/server` 仍保留以下 metric name： @@ -157,3 +165,65 @@ span name 目前允许保留业务可读格式，例如： - [apps/server/src/libs/otel.ts](/apps/server/src/libs/otel.ts) - [services/telegram-bot/src/llm/actions.ts](/services/telegram-bot/src/llm/actions.ts) - [services/telegram-bot/src/bots/telegram/agent/actions/read-message.ts](/services/telegram-bot/src/bots/telegram/agent/actions/read-message.ts) + +## SemconvStability 迁移说明 + +`@opentelemetry/instrumentation-http` 0.215+ 默认 OLD semconv（`http.server.duration` in ms），不是 STABLE 名。AIRI 在 [apps/server/instrumentation.mjs](/apps/server/instrumentation.mjs) 顶部强制 `OTEL_SEMCONV_STABILITY_OPT_IN=http`（仅 STABLE）。 + +| Semconv 模式 | 发哪些 series | 我们用 | +|---|---|---| +| OLD（默认）| `http.server.duration` (ms)、`http.client.duration` (ms)、attr 用 `http.method` / `http.status_code` | ❌ | +| STABLE（`=http`）| `http.server.request.duration` (s)、`http.client.request.duration` (s)、attr 用 `http.request.method` / `http.response.status_code` | ✅ | +| 双发（`=http/dup`）| 上面两套都发 | 仅在有外部 OLD-name 消费者待迁移时启用 | + +**为什么直接 STABLE-only**： + +- grep 整仓库零 OLD-name 引用 +- Dashboard 与服务代码 checked in 在一起，无外部 dashboard +- 迁移没有自然终点，OLD 系列不显式清理就一直占 storage +- 双发会让每条 HTTP 请求 cardinality 翻倍 + +**何时切回 `dup`**：将来如果有别的 service 主动 scrape 本 server 的 OLD-name 系列，临时切几周完成迁移即可。 + +## Counter priming 注意事项 + +OTel SDK 的 Counter / UpDownCounter 在第一次 `.add()` 之前**完全不出现在 Prometheus 抓取里**。Histogram 同理（要等第一次 `.record()`）。 + +后果：低流量 metric 在 dashboard 上看起来像「埋点丢了」，告警里 `absent()` 也无法工作。 + +[apps/server/src/libs/otel.ts](/apps/server/src/libs/otel.ts) 的 `primeCounter` 在 SDK 启动后给每个 Counter 调一次 `.add(0)`，把 series 注册出来；`0` 不影响 rate / sum 计算。 + +加新 Counter 时**记得加进 prime 列表**，否则未触发的指标在 Grafana 里就是空的。 + +验证脚本：[apps/server/src/scripts/otel-smoke.mjs](/apps/server/src/scripts/otel-smoke.mjs) + +```sh +pnpm -F @proj-airi/server exec node --import tsx ./src/scripts/otel-smoke.mjs +``` + +打印 SDK 启动后立即可见的所有 instrument 名字。 + +## Dashboard 变量陷阱 + +**变量定义里不要引用业务 metric**。早期 [airi-server-overview-cloud.json](/apps/server/otel/grafana/dashboards/airi-server-overview-cloud.json) 的 `$env` / `$service` 都从 `http_server_request_duration_seconds_count` 取 label values —— 升级 instrumentation-http 后这个系列没了，导致： + +1. 两个变量解析为空字符串 +2. 所有 panel 的 `{service_name=~"$service", deployment_environment=~"$env"}` 匹配零 series +3. 整个 dashboard 全 No Data，**包括那些 metric 还活着的 panel** + +修法：变量改用 `target_info`。这是 OTel SDK 启动就发的 resource-only series，永远存在，且天然自带 `service_name` / `deployment_environment` / `service_version` 这套 resource attributes。 + +```promql +# Good +label_values(target_info, deployment_environment) +label_values(target_info{deployment_environment=~"$env"}, service_name) + +# Bad — 任何业务 metric 改名/迁移就全盘崩 +label_values(http_server_request_duration_seconds_count, deployment_environment) +``` + +后续新增 dashboard 默认沿用 `target_info` 这条惯例。 + +## 完整 metric 目录 + +按域分组的全量 metric 清单（名字、类型、单位、labels、落点）见 [`observability-metrics.md`](./observability-metrics.md)。每加一个新 metric 时同步更新该文档。 diff --git a/apps/server/docs/ai-context/observability-metrics.md b/apps/server/docs/ai-context/observability-metrics.md new file mode 100644 index 000000000..9f20f0281 --- /dev/null +++ b/apps/server/docs/ai-context/observability-metrics.md @@ -0,0 +1,157 @@ +# Metrics Catalog + +服务端当前所有 metric 的完整目录。按业务领域分组。 + +> 命名规则、`airi.*` 边界、attribute 选择请看 [`observability-conventions.md`](./observability-conventions.md)。本文档只做"哪些 metric 存在、怎么查"。 + +## 名字到 Prometheus 系列的换算 + +OTel SDK 在导出到 Prometheus 时做两件事： + +1. `.` → `_`：`airi.billing.flux.consumed` → `airi_billing_flux_consumed` +2. Counter 加 `_total` 后缀：`auth.attempts` → `auth_attempts_total` +3. Histogram 拆三件套：`http.server.request.duration` → + - `http_server_request_duration_seconds_bucket`（含 `le` label） + - `http_server_request_duration_seconds_count` + - `http_server_request_duration_seconds_sum` +4. UpDownCounter 不加 `_total`：`ws.connections.active` → `ws_connections_active` +5. 带单位的 instrument 在 SDK 导出时把单位插进名字：`airi.stripe.revenue`（unit `minor_unit`）→ `airi_stripe_revenue_minor_unit_total` + +> 查询面板若拼名字时不确定后缀，先用 `{__name__=~"airi_billing_flux.*"}` 之类正则探一下。 + +## HTTP（来自 instrumentation-http） + +| Metric | 类型 | Unit | 来源 | 关键 attributes | +|---|---|---|---|---| +| `http.server.request.duration` | Histogram | s | `instrumentation-http`（STABLE semconv） | `http.request.method`、`http.route`、`http.response.status_code` | +| `http.server.active_requests` | UpDownCounter | — | [middlewares/otel.ts](../../src/middlewares/otel.ts) `otelMiddleware` | `http.request.method`、`http.route` | + +> **STABLE-only**：[instrumentation.mjs:25](../../instrumentation.mjs) 把 `OTEL_SEMCONV_STABILITY_OPT_IN=http` 提前注入。OLD 系列（`http.server.duration` in ms）不再发射。详见 [`observability-conventions.md` 的 SemconvStability 章节](./observability-conventions.md#semconvstability-迁移说明)。 + +## Auth & Users + +全部由 [libs/auth.ts](../../src/libs/auth.ts) Better Auth hooks 触发。 + +| Metric | 类型 | 落点（hook） | Labels | +|---|---|---|---| +| `auth.attempts` | Counter | `before` hook，path 含 `/sign-in` 或 `/sign-up` | `auth.method`（path 末段） | +| `auth.failures` | Counter | `after` hook，`ctx.context.returned` 含 `error` | `auth.method` | +| `user.registered` | Counter | `databaseHooks.user.create.after` | — | +| `user.login` | Counter | `databaseHooks.session.create.after` | — | +| `user.active_sessions` | UpDownCounter | session create / delete | — | + +## Engagement + +| Metric | 类型 | 落点 | Labels | +|---|---|---|---| +| `chat.messages` | Counter | [services/chats.ts](../../src/services/chats.ts) `pushMessages` | — | +| `character.created` | Counter | [services/characters.ts](../../src/services/characters.ts) | — | +| `character.deleted` | Counter | 同上 | — | +| `character.engagement` | Counter | 同上（like/bookmark） | `action`（`like` / `unlike` / `bookmark` / `unbookmark`） | +| `ws.connections.active` | UpDownCounter | [routes/chat-ws/index.ts](../../src/routes/chat-ws/index.ts) | — | +| `ws.messages.sent` | Counter | 同上 | — | +| `ws.messages.received` | Counter | [services/chats.ts](../../src/services/chats.ts) | — | + +## Revenue & Billing + +### Stripe lifecycle + +| Metric | 类型 | 落点 | Labels | +|---|---|---|---| +| `stripe.checkout.created` | Counter | [routes/stripe/index.ts](../../src/routes/stripe/index.ts) `/checkout` POST | — | +| `stripe.checkout.completed` | Counter | webhook `checkout.session.completed` | — | +| `stripe.payment.failed` | Counter | webhook `invoice.payment_failed` | — | +| `stripe.subscription.event` | Counter | webhook `customer.subscription.*` | `event_type`（`created`/`updated`/`deleted`） | +| `stripe.events` | Counter | 任何 webhook | `event_type`（完整 event.type，e.g. `invoice.paid`） | +| `airi.stripe.revenue` | Counter（`minor_unit`） | webhook `checkout.session.completed` + `invoice.paid` | `currency`、`source`（`checkout`/`invoice`） | + +> **金额单位**：`airi.stripe.revenue` 用最小币种单位（cents 等），跨币种 sum 没有意义，**永远 `sum by (currency)`**。要换主单位（dollars 等）做 `/ 100` 即可，前提是该币种没有不同 minor unit 比例。 + +### Flux ledger + +| Metric | 类型 | 落点 | Labels | +|---|---|---|---| +| `airi.billing.flux.consumed` | Counter | [routes/openai/v1/index.ts](../../src/routes/openai/v1/index.ts) `recordMetrics`（chat / tts） | `gen_ai.request.model`、`gen_ai.operation.name`/`airi.gen_ai.operation.kind`、`http.response.status_code` | +| `airi.billing.flux.credited` | Counter | [services/billing/billing-service.ts](../../src/services/billing/billing-service.ts) 三条入账路径 | `source`（`stripe.checkout`/`stripe.invoice`/`promo`/`admin_grant`/...）、`type`（`credit`/`promo`） | +| `airi.billing.flux.unbilled` | Counter | `routes/openai/v1/index.ts` 流式 debit 失败 catch | `gen_ai.request.model`、`reason`（`debit_failed`）、`stage`（`streaming`） | +| `flux.insufficient_balance` | Counter | [services/billing/billing-service.ts](../../src/services/billing/billing-service.ts) `debitFlux` | — | +| `airi.billing.tts.chars` | Counter | [services/billing/flux-meter.ts](../../src/services/billing/flux-meter.ts) `accumulate` | `meter`（`tts`）、`model` | +| `airi.billing.tts.preflight_rejections` | Counter | `flux-meter.ts` `assertCanAfford` | `meter`、`reason`（`insufficient_balance`） | + +> **`airi.billing.flux.unbilled` 是 P0 告警金线**：任何持续 > 0 都意味着真实收入泄漏，应当 page。语义上等于"流式响应已经发给用户但 DB debit 失败的 Flux 量"。 + +## GenAI + +| Metric | 类型 | Unit | 落点 | Labels | +|---|---|---|---|---| +| `gen_ai.client.operation.duration` | Histogram | s | `routes/openai/v1/index.ts` `recordMetrics` | `gen_ai.request.model`、`gen_ai.operation.name`/`airi.gen_ai.operation.kind`、`http.response.status_code` | +| `gen_ai.client.operation.count` | Counter | — | 同上 | 同上 | +| `gen_ai.client.token.usage.input` | Counter | — | 同上 | 同上 | +| `gen_ai.client.token.usage.output` | Counter | — | 同上 | 同上 | +| `gen_ai.client.first_token.duration` | Histogram | s | 流式 reader 第一个非空 chunk 抵达时 | `gen_ai.request.model`、`gen_ai.operation.name` | +| `airi.gen_ai.stream.interrupted` | Counter | — | 流式 reader catch | `gen_ai.request.model`、`stage`（`before_first_chunk`/`mid_stream`） | + +## Email（Resend） + +来源 [services/email.ts](../../src/services/email.ts) 的 `send()` 内部 try/catch。 + +| Metric | 类型 | Labels | +|---|---|---| +| `airi.email.send` | Counter | `template`（`verification`/`password_reset`/`magic_link`/`change_email`/`delete_account`/`unknown`） | +| `airi.email.failures` | Counter | `template`、`error_name`（Resend `error.name` 或 `unhandled`） | +| `airi.email.duration` | Histogram（s） | `template`、`outcome`（`ok`/`error`） | + +## Rate limiting + +来源 [middlewares/rate-limit.ts](../../src/middlewares/rate-limit.ts) 的 `handler`。 + +| Metric | 类型 | Labels | +|---|---|---| +| `airi.rate_limit.blocked` | Counter | `route`（callsite 提供，e.g. `auth.api` / `openai.completions` / `stripe.checkout`）、`key_type`（`user`/`ip`）、`limit`（窗口内最大次数） | + +> **注意**：`route` 是 callsite 显式提供的稳定 label，不是 raw URL path —— URL path 是高 cardinality，会爆炸。新加 rate limiter 时记得传 `routeLabel`。 + +## Node.js Runtime + +来自 `@opentelemetry/instrumentation-runtime-node`，下面这些是 dashboard 上用到的子集（不全列）： + +- `v8js.memory.heap.{used,limit,space.physical_size,space.available_size}` Gauge / bytes +- `nodejs.eventloop.delay.{p50,p99,mean,...}` Gauge / s +- `nodejs.eventloop.utilization` Gauge / ratio +- `v8js.gc.duration` Histogram / s + +## 已落地的 dashboard 行映射 + +[airi-server-overview-cloud.json](../../otel/grafana/dashboards/airi-server-overview-cloud.json)，从上到下： + +| Row | 关键 metric | +|---|---| +| HTTP Overview | `http.server.request.duration`（rate / P95 / by route / 5xx 率） | +| Auth & Users | `auth.attempts` / `auth.failures` / `user.{login,registered,active_sessions}` + 失败率 | +| Engagement | `ws.connections.active` / `ws.messages.{sent,received}` / `chat.messages` / `character.{created,deleted,engagement}` | +| Business Metrics | `airi.billing.flux.consumed` / `flux.insufficient_balance` / `gen_ai.client.token.usage.*` / `stripe.checkout.completed` / `airi.billing.flux.credited` | +| Stripe Detail | `stripe.{events,subscription.event,payment.failed}` / checkout funnel / `airi.stripe.revenue` | +| Node.js Runtime | runtime instrumentation 那一批 | +| LLM Gateway | `gen_ai.client.{operation.count,operation.duration,token.usage.*,first_token.duration}` / `airi.billing.flux.consumed` / `airi.billing.flux.unbilled` / `airi.gen_ai.stream.interrupted` / `airi.billing.tts.chars` | +| Reliability | `airi.email.{send,failures}` 失败率 / `airi.rate_limit.blocked` / `airi.billing.tts.preflight_rejections` | +| Application Logs | Loki，不是 Prometheus | + +## 验证 metric 是否已注册 + +[`src/scripts/otel-smoke.mjs`](../../src/scripts/otel-smoke.mjs) 跑一遍： + +```sh +pnpm -F @proj-airi/server exec node --import tsx ./src/scripts/otel-smoke.mjs +``` + +会打印 SDK 启动时立即 export 的所有 instrument 名字。**Counter 通过 `.add(0)` priming**（[libs/otel.ts](../../src/libs/otel.ts) `primeCounter`）后会出现在这里 —— Histogram 不会，要等真实 `.record()` 才出现。 + +## 加新 metric 时的 checklist + +1. 决定命名空间：能映射到 OTel semconv 就用标准名，否则放 `airi.*`（不要造新顶级前缀） +2. 在 [utils/observability.ts](../../src/utils/observability.ts) 加常量 +3. 在 [libs/otel.ts](../../src/libs/otel.ts) 的对应 metric group 接口（`HttpMetrics`/`AuthMetrics`/...）加字段，并在 `initOtel` 里 `meter.create*` 创建 +4. **如果是 Counter，在 `primeCounter` 调用列表里加一行** —— 否则低流量时 panel 看起来"没数据" +5. 在 callsite 通过 DI 拿到 metrics 对象后调 `.add()` / `.record()` +6. 跑 `pnpm -F @proj-airi/server exec node --import tsx ./src/scripts/otel-smoke.mjs` 确认注册 +7. 更新本文档对应章节 diff --git a/apps/server/instrumentation.mjs b/apps/server/instrumentation.mjs index 8b005743b..5d6b49a32 100644 --- a/apps/server/instrumentation.mjs +++ b/apps/server/instrumentation.mjs @@ -29,9 +29,24 @@ import { PgInstrumentation } from '@opentelemetry/instrumentation-pg' // Source: node_modules/.pnpm/@opentelemetry+instrumentation-http@0.215.0/.../build/src/http.js L25-72 // MUST run before `new HttpInstrumentation(...)` below — its constructor reads // the env var once and caches the result. +// +// Use a truthy check (not `??=`): `process.env.X` is `''` when the platform +// (e.g. Railway) registers the var without a value, and `??=` does NOT override +// empty strings — that would silently fall back to OLD semconv with no signal +// in logs. Truthy check covers both `undefined` and `''`. // Removal condition: ops sets OTEL_SEMCONV_STABILITY_OPT_IN explicitly in the -// deployment platform (Railway env), then this preload default can be deleted. -env.OTEL_SEMCONV_STABILITY_OPT_IN ??= 'http' +// deployment platform with a non-empty value, then this preload default can be +// deleted. +if (!env.OTEL_SEMCONV_STABILITY_OPT_IN) { + env.OTEL_SEMCONV_STABILITY_OPT_IN = 'http' +} + +// Surface the resolved value in stdout BEFORE any instrumentation constructor +// runs. Lets ops grep Railway logs for `[otel-preload]` to confirm the preload +// actually executed and what semconv mode is active. Without this, a misloaded +// preload (wrong `--import` path, missing flag, build cache) is invisible +// until you query Prometheus and notice STABLE-name series are missing. +console.info(`[otel-preload] OTEL_SEMCONV_STABILITY_OPT_IN=${env.OTEL_SEMCONV_STABILITY_OPT_IN}`) registerInstrumentations({ instrumentations: [ diff --git a/apps/server/otel/grafana/dashboards/airi-server-overview-cloud.json b/apps/server/otel/grafana/dashboards/airi-server-overview-cloud.json index 21a9f5e46..a8154ee61 100644 --- a/apps/server/otel/grafana/dashboards/airi-server-overview-cloud.json +++ b/apps/server/otel/grafana/dashboards/airi-server-overview-cloud.json @@ -9,7 +9,9 @@ "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "query": { - "datasource": { "name": "-- Grafana --" }, + "datasource": { + "name": "-- Grafana --" + }, "group": "grafana", "kind": "DataQuery", "spec": {}, @@ -21,558 +23,6 @@ "cursorSync": "Crosshair", "editable": true, "elements": { - "panel-1": { - "kind": "Panel", - "spec": { - "data": { - "kind": "QueryGroup", - "spec": { - "queries": [ - { - "kind": "PanelQuery", - "spec": { - "hidden": false, - "query": { - "datasource": { "name": "grafanacloud-projairi-prom" }, - "group": "prometheus", - "kind": "DataQuery", - "spec": { - "expr": "sum(rate(http_server_request_duration_seconds_count{service_name=~\"$service\", deployment_environment=~\"$env\", http_request_method!=\"OPTIONS\"}[$__rate_interval]))", - "legendFormat": "req/s" - }, - "version": "v0" - }, - "refId": "A" - } - } - ], - "queryOptions": {}, - "transformations": [] - } - }, - "description": "", - "id": 1, - "links": [], - "title": "Request Rate", - "vizConfig": { - "group": "stat", - "kind": "VizConfig", - "spec": { - "fieldConfig": { - "defaults": { - "color": { "mode": "palette-classic" }, - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "green", "value": 0 }, - { "color": "yellow", "value": 100 }, - { "color": "red", "value": 500 } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - } - }, - "version": "13.0.0-23630096546" - } - } - }, - "panel-2": { - "kind": "Panel", - "spec": { - "data": { - "kind": "QueryGroup", - "spec": { - "queries": [ - { - "kind": "PanelQuery", - "spec": { - "hidden": false, - "query": { - "datasource": { "name": "grafanacloud-projairi-prom" }, - "group": "prometheus", - "kind": "DataQuery", - "spec": { - "expr": "sum(rate(http_server_request_duration_seconds_count{service_name=~\"$service\", deployment_environment=~\"$env\", http_request_method!=\"OPTIONS\", http_response_status_code=~\"5..\"}[$__rate_interval])) / sum(rate(http_server_request_duration_seconds_count{service_name=~\"$service\", deployment_environment=~\"$env\", http_request_method!=\"OPTIONS\"}[$__rate_interval]))", - "legendFormat": "error rate" - }, - "version": "v0" - }, - "refId": "A" - } - } - ], - "queryOptions": {}, - "transformations": [] - } - }, - "description": "", - "id": 2, - "links": [], - "title": "Error Rate (5xx)", - "vizConfig": { - "group": "stat", - "kind": "VizConfig", - "spec": { - "fieldConfig": { - "defaults": { - "color": { "mode": "palette-classic" }, - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "green", "value": 0 }, - { "color": "yellow", "value": 0.01 }, - { "color": "red", "value": 0.05 } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - } - }, - "version": "13.0.0-23630096546" - } - } - }, - "panel-3": { - "kind": "Panel", - "spec": { - "data": { - "kind": "QueryGroup", - "spec": { - "queries": [ - { - "kind": "PanelQuery", - "spec": { - "hidden": false, - "query": { - "datasource": { "name": "grafanacloud-projairi-prom" }, - "group": "prometheus", - "kind": "DataQuery", - "spec": { - "expr": "histogram_quantile(0.95, sum(rate(http_server_request_duration_seconds_bucket{service_name=~\"$service\", deployment_environment=~\"$env\", http_request_method!=\"OPTIONS\", http_route!~\"/api/v1/openai/.*\"}[$__rate_interval])) by (le))", - "legendFormat": "p95" - }, - "version": "v0" - }, - "refId": "A" - } - } - ], - "queryOptions": {}, - "transformations": [] - } - }, - "description": "", - "id": 3, - "links": [], - "title": "P95 Latency (excl. LLM)", - "vizConfig": { - "group": "stat", - "kind": "VizConfig", - "spec": { - "fieldConfig": { - "defaults": { - "color": { "mode": "palette-classic" }, - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "green", "value": 0 }, - { "color": "yellow", "value": 0.2 }, - { "color": "red", "value": 1 } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "percentChangeColorMode": "standard", - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - } - }, - "version": "13.0.0-23630096546" - } - } - }, - "panel-4": { - "kind": "Panel", - "spec": { - "data": { - "kind": "QueryGroup", - "spec": { - "queries": [ - { - "kind": "PanelQuery", - "spec": { - "hidden": false, - "query": { - "datasource": { "name": "grafanacloud-projairi-prom" }, - "group": "prometheus", - "kind": "DataQuery", - "spec": { - "expr": "sum by (http_route, http_request_method) (rate(http_server_request_duration_seconds_count{service_name=~\"$service\", deployment_environment=~\"$env\", http_request_method!=\"OPTIONS\", http_response_status_code!=\"404\"}[$__rate_interval]))", - "legendFormat": "{{http_request_method}} {{http_route}}" - }, - "version": "v0" - }, - "refId": "A" - } - }, - { - "kind": "PanelQuery", - "spec": { - "hidden": false, - "query": { - "datasource": { "name": "grafanacloud-projairi-prom" }, - "group": "prometheus", - "kind": "DataQuery", - "spec": { - "expr": "sum(rate(http_server_request_duration_seconds_count{service_name=~\"$service\", deployment_environment=~\"$env\", http_request_method!=\"OPTIONS\", http_response_status_code=\"404\"}[$__rate_interval]))", - "legendFormat": "404 (all paths)" - }, - "version": "v0" - }, - "refId": "B" - } - } - ], - "queryOptions": {}, - "transformations": [] - } - }, - "description": "Excludes 404 responses to avoid high-cardinality route explosion", - "id": 4, - "links": [], - "title": "Request Rate by Route + Method", - "vizConfig": { - "group": "timeseries", - "kind": "VizConfig", - "spec": { - "fieldConfig": { - "defaults": { - "color": { "mode": "palette-classic" }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { "legend": false, "tooltip": false, "viz": false }, - "insertNulls": false, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { "type": "linear" }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { "group": "A", "mode": "none" }, - "thresholdsStyle": { "mode": "off" } - }, - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "red", "value": 80 }] }, - "unit": "reqps" - }, - "overrides": [] - }, - "options": { - "annotations": { "clustering": -1, "multiLane": false }, - "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, - "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } - } - }, - "version": "13.0.0-23630096546" - } - } - }, - "panel-5": { - "kind": "Panel", - "spec": { - "data": { - "kind": "QueryGroup", - "spec": { - "queries": [ - { - "kind": "PanelQuery", - "spec": { - "hidden": false, - "query": { - "datasource": { "name": "grafanacloud-projairi-prom" }, - "group": "prometheus", - "kind": "DataQuery", - "spec": { - "expr": "histogram_quantile(0.95, sum by (le, http_route) (rate(http_server_request_duration_seconds_bucket{service_name=~\"$service\", deployment_environment=~\"$env\", http_request_method!=\"OPTIONS\", http_response_status_code!=\"404\"}[$__rate_interval])))", - "legendFormat": "{{http_route}}" - }, - "version": "v0" - }, - "refId": "A" - } - }, - { - "kind": "PanelQuery", - "spec": { - "hidden": false, - "query": { - "datasource": { "name": "grafanacloud-projairi-prom" }, - "group": "prometheus", - "kind": "DataQuery", - "spec": { - "expr": "histogram_quantile(0.95, sum by (le) (rate(http_server_request_duration_seconds_bucket{service_name=~\"$service\", deployment_environment=~\"$env\", http_request_method!=\"OPTIONS\", http_response_status_code=\"404\"}[$__rate_interval])))", - "legendFormat": "404 (all paths)" - }, - "version": "v0" - }, - "refId": "B" - } - } - ], - "queryOptions": {}, - "transformations": [] - } - }, - "description": "Excludes 404 responses to avoid high-cardinality route explosion", - "id": 5, - "links": [], - "title": "Latency by Route (P95)", - "vizConfig": { - "group": "timeseries", - "kind": "VizConfig", - "spec": { - "fieldConfig": { - "defaults": { - "color": { "mode": "palette-classic" }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { "legend": false, "tooltip": false, "viz": false }, - "insertNulls": false, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { "type": "linear" }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { "group": "A", "mode": "none" }, - "thresholdsStyle": { "mode": "off" } - }, - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "red", "value": 80 }] }, - "unit": "s" - }, - "overrides": [] - }, - "options": { - "annotations": { "clustering": -1, "multiLane": false }, - "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, - "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } - } - }, - "version": "13.0.0-23630096546" - } - } - }, - "panel-6": { - "kind": "Panel", - "spec": { - "data": { - "kind": "QueryGroup", - "spec": { - "queries": [ - { - "kind": "PanelQuery", - "spec": { - "hidden": false, - "query": { - "datasource": { "name": "grafanacloud-projairi-prom" }, - "group": "prometheus", - "kind": "DataQuery", - "spec": { - "expr": "sum by (http_response_status_code) (rate(http_server_request_duration_seconds_count{service_name=~\"$service\", deployment_environment=~\"$env\", http_request_method!=\"OPTIONS\"}[$__rate_interval]))", - "legendFormat": "{{http_response_status_code}}" - }, - "version": "v0" - }, - "refId": "A" - } - } - ], - "queryOptions": {}, - "transformations": [] - } - }, - "description": "", - "id": 6, - "links": [], - "title": "Response Status Codes", - "vizConfig": { - "group": "timeseries", - "kind": "VizConfig", - "spec": { - "fieldConfig": { - "defaults": { - "color": { "mode": "palette-classic" }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "bars", - "fillOpacity": 80, - "gradientMode": "none", - "hideFrom": { "legend": false, "tooltip": false, "viz": false }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { "type": "linear" }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { "group": "A", "mode": "normal" }, - "thresholdsStyle": { "mode": "off" } - }, - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "red", "value": 80 }] }, - "unit": "reqps" - }, - "overrides": [] - }, - "options": { - "annotations": { "clustering": -1, "multiLane": false }, - "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, - "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } - } - }, - "version": "13.0.0-23630096546" - } - } - }, - "panel-30": { - "kind": "Panel", - "spec": { - "data": { - "kind": "QueryGroup", - "spec": { - "queries": [ - { - "kind": "PanelQuery", - "spec": { - "hidden": false, - "query": { - "datasource": { "name": "grafanacloud-projairi-prom" }, - "group": "prometheus", - "kind": "DataQuery", - "spec": { - "expr": "sum(rate(http_server_request_duration_seconds_count{service_name=~\"$service\", deployment_environment=~\"$env\", http_request_method!=\"OPTIONS\", http_response_status_code=\"404\"}[$__rate_interval]))", - "legendFormat": "404 total" - }, - "version": "v0" - }, - "refId": "A" - } - } - ], - "queryOptions": {}, - "transformations": [] - } - }, - "description": "All 404 responses aggregated into a single series to avoid high-cardinality route explosion", - "id": 30, - "links": [], - "title": "404 Request Rate (aggregated)", - "vizConfig": { - "group": "timeseries", - "kind": "VizConfig", - "spec": { - "fieldConfig": { - "defaults": { - "color": { "mode": "palette-classic" }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { "legend": false, "tooltip": false, "viz": false }, - "insertNulls": false, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { "type": "linear" }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { "group": "A", "mode": "none" }, - "thresholdsStyle": { "mode": "off" } - }, - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "red", "value": 80 }] }, - "unit": "reqps" - }, - "overrides": [] - }, - "options": { - "annotations": { "clustering": -1, "multiLane": false }, - "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, - "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } - } - }, - "version": "13.0.0-23630096546" - } - } - }, "panel-31": { "kind": "Panel", "spec": { @@ -585,7 +35,9 @@ "spec": { "hidden": false, "query": { - "datasource": { "name": "grafanacloud-projairi-prom" }, + "datasource": { + "name": "grafanacloud-projairi-prom" + }, "group": "prometheus", "kind": "DataQuery", "spec": { @@ -612,7 +64,9 @@ "spec": { "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, @@ -624,27 +78,62 @@ "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", - "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, - "scaleDistribution": { "type": "linear" }, + "scaleDistribution": { + "type": "linear" + }, "showPoints": "auto", "showValues": false, "spanNulls": false, - "stacking": { "group": "A", "mode": "none" }, - "thresholdsStyle": { "mode": "off" } + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] }, - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "red", "value": 80 }] }, "unit": "short" }, "overrides": [] }, "options": { - "annotations": { "clustering": -1, "multiLane": false }, - "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, - "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } } }, "version": "13.0.0-23630096546" @@ -663,7 +152,9 @@ "spec": { "hidden": false, "query": { - "datasource": { "name": "grafanacloud-projairi-prom" }, + "datasource": { + "name": "grafanacloud-projairi-prom" + }, "group": "prometheus", "kind": "DataQuery", "spec": { @@ -680,7 +171,9 @@ "spec": { "hidden": false, "query": { - "datasource": { "name": "grafanacloud-projairi-prom" }, + "datasource": { + "name": "grafanacloud-projairi-prom" + }, "group": "prometheus", "kind": "DataQuery", "spec": { @@ -707,7 +200,9 @@ "spec": { "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, @@ -719,27 +214,62 @@ "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", - "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, - "scaleDistribution": { "type": "linear" }, + "scaleDistribution": { + "type": "linear" + }, "showPoints": "auto", "showValues": false, "spanNulls": false, - "stacking": { "group": "A", "mode": "none" }, - "thresholdsStyle": { "mode": "off" } + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] }, - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "red", "value": 80 }] }, "unit": "short" }, "overrides": [] }, "options": { - "annotations": { "clustering": -1, "multiLane": false }, - "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, - "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } } }, "version": "13.0.0-23630096546" @@ -758,7 +288,9 @@ "spec": { "hidden": false, "query": { - "datasource": { "name": "grafanacloud-projairi-prom" }, + "datasource": { + "name": "grafanacloud-projairi-prom" + }, "group": "prometheus", "kind": "DataQuery", "spec": { @@ -785,7 +317,9 @@ "spec": { "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, @@ -797,27 +331,62 @@ "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", - "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, - "scaleDistribution": { "type": "linear" }, + "scaleDistribution": { + "type": "linear" + }, "showPoints": "auto", "showValues": false, "spanNulls": false, - "stacking": { "group": "A", "mode": "none" }, - "thresholdsStyle": { "mode": "off" } + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] }, - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "red", "value": 80 }] }, "unit": "short" }, "overrides": [] }, "options": { - "annotations": { "clustering": -1, "multiLane": false }, - "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, - "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } } }, "version": "13.0.0-23630096546" @@ -836,7 +405,9 @@ "spec": { "hidden": false, "query": { - "datasource": { "name": "grafanacloud-projairi-prom" }, + "datasource": { + "name": "grafanacloud-projairi-prom" + }, "group": "prometheus", "kind": "DataQuery", "spec": { @@ -853,7 +424,9 @@ "spec": { "hidden": false, "query": { - "datasource": { "name": "grafanacloud-projairi-prom" }, + "datasource": { + "name": "grafanacloud-projairi-prom" + }, "group": "prometheus", "kind": "DataQuery", "spec": { @@ -880,7 +453,9 @@ "spec": { "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, @@ -892,27 +467,62 @@ "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", - "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, - "scaleDistribution": { "type": "linear" }, + "scaleDistribution": { + "type": "linear" + }, "showPoints": "auto", "showValues": false, "spanNulls": false, - "stacking": { "group": "A", "mode": "none" }, - "thresholdsStyle": { "mode": "off" } + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] }, - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "red", "value": 80 }] }, "unit": "bytes" }, "overrides": [] }, "options": { - "annotations": { "clustering": -1, "multiLane": false }, - "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, - "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } } }, "version": "13.0.0-23630096546" @@ -931,7 +541,9 @@ "spec": { "hidden": false, "query": { - "datasource": { "name": "grafanacloud-projairi-prom" }, + "datasource": { + "name": "grafanacloud-projairi-prom" + }, "group": "prometheus", "kind": "DataQuery", "spec": { @@ -948,7 +560,9 @@ "spec": { "hidden": false, "query": { - "datasource": { "name": "grafanacloud-projairi-prom" }, + "datasource": { + "name": "grafanacloud-projairi-prom" + }, "group": "prometheus", "kind": "DataQuery", "spec": { @@ -975,7 +589,9 @@ "spec": { "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, @@ -987,27 +603,62 @@ "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", - "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, - "scaleDistribution": { "type": "linear" }, + "scaleDistribution": { + "type": "linear" + }, "showPoints": "auto", "showValues": false, "spanNulls": false, - "stacking": { "group": "A", "mode": "none" }, - "thresholdsStyle": { "mode": "off" } + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] }, - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "red", "value": 80 }] }, "unit": "bytes" }, "overrides": [] }, "options": { - "annotations": { "clustering": -1, "multiLane": false }, - "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, - "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } } }, "version": "13.0.0-23630096546" @@ -1026,7 +677,9 @@ "spec": { "hidden": false, "query": { - "datasource": { "name": "grafanacloud-projairi-prom" }, + "datasource": { + "name": "grafanacloud-projairi-prom" + }, "group": "prometheus", "kind": "DataQuery", "spec": { @@ -1043,7 +696,9 @@ "spec": { "hidden": false, "query": { - "datasource": { "name": "grafanacloud-projairi-prom" }, + "datasource": { + "name": "grafanacloud-projairi-prom" + }, "group": "prometheus", "kind": "DataQuery", "spec": { @@ -1060,7 +715,9 @@ "spec": { "hidden": false, "query": { - "datasource": { "name": "grafanacloud-projairi-prom" }, + "datasource": { + "name": "grafanacloud-projairi-prom" + }, "group": "prometheus", "kind": "DataQuery", "spec": { @@ -1087,7 +744,9 @@ "spec": { "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, @@ -1099,27 +758,62 @@ "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", - "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, - "scaleDistribution": { "type": "linear" }, + "scaleDistribution": { + "type": "linear" + }, "showPoints": "auto", "showValues": false, "spanNulls": false, - "stacking": { "group": "A", "mode": "none" }, - "thresholdsStyle": { "mode": "off" } + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] }, - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "red", "value": 80 }] }, "unit": "s" }, "overrides": [] }, "options": { - "annotations": { "clustering": -1, "multiLane": false }, - "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, - "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } } }, "version": "13.0.0-23630096546" @@ -1138,7 +832,9 @@ "spec": { "hidden": false, "query": { - "datasource": { "name": "grafanacloud-projairi-prom" }, + "datasource": { + "name": "grafanacloud-projairi-prom" + }, "group": "prometheus", "kind": "DataQuery", "spec": { @@ -1155,7 +851,9 @@ "spec": { "hidden": false, "query": { - "datasource": { "name": "grafanacloud-projairi-prom" }, + "datasource": { + "name": "grafanacloud-projairi-prom" + }, "group": "prometheus", "kind": "DataQuery", "spec": { @@ -1182,7 +880,9 @@ "spec": { "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, @@ -1194,27 +894,62 @@ "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", - "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, - "scaleDistribution": { "type": "linear" }, + "scaleDistribution": { + "type": "linear" + }, "showPoints": "auto", "showValues": false, "spanNulls": false, - "stacking": { "group": "A", "mode": "none" }, - "thresholdsStyle": { "mode": "off" } + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] }, - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "red", "value": 80 }] }, "unit": "short" }, "overrides": [] }, "options": { - "annotations": { "clustering": -1, "multiLane": false }, - "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, - "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } } }, "version": "13.0.0-23630096546" @@ -1233,7 +968,9 @@ "spec": { "hidden": false, "query": { - "datasource": { "name": "grafanacloud-projairi-logs" }, + "datasource": { + "name": "grafanacloud-projairi-logs" + }, "group": "loki", "kind": "DataQuery", "spec": { @@ -1258,7 +995,10 @@ "group": "logs", "kind": "VizConfig", "spec": { - "fieldConfig": { "defaults": {}, "overrides": [] }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "options": { "dedupStrategy": "none", "enableInfiniteScrolling": false, @@ -1284,100 +1024,1002 @@ "panel-17": { "kind": "Panel", "spec": { - "data": { "kind": "QueryGroup", "spec": { "queries": [{ "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(rate(gen_ai_client_operation_count_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))", "legendFormat": "req/s" }, "version": "v0" }, "refId": "A" } }], "queryOptions": {}, "transformations": [] } }, + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "grafanacloud-projairi-prom" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum(rate(gen_ai_client_operation_count_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))", + "legendFormat": "req/s" + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, "description": "", "id": 17, "links": [], "title": "LLM Request Rate", - "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "yellow", "value": 50 }, { "color": "red", "value": 200 }] }, "unit": "reqps" }, "overrides": [] }, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true } }, "version": "13.0.0-23630096546" } + "vizConfig": { + "group": "stat", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "yellow", + "value": 50 + }, + { + "color": "red", + "value": 200 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + } + }, + "version": "13.0.0-23630096546" + } } }, "panel-18": { "kind": "Panel", "spec": { - "data": { "kind": "QueryGroup", "spec": { "queries": [{ "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "histogram_quantile(0.95, sum(rate(gen_ai_client_operation_duration_seconds_bucket{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval])) by (le))", "legendFormat": "p95" }, "version": "v0" }, "refId": "A" } }], "queryOptions": {}, "transformations": [] } }, + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "grafanacloud-projairi-prom" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "histogram_quantile(0.95, sum(rate(gen_ai_client_operation_duration_seconds_bucket{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval])) by (le))", + "legendFormat": "p95" + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, "description": "", "id": 18, "links": [], "title": "LLM P95 Latency", - "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "yellow", "value": 0.5 }, { "color": "red", "value": 2 }] }, "unit": "s" }, "overrides": [] }, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true } }, "version": "13.0.0-23630096546" } + "vizConfig": { + "group": "stat", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "yellow", + "value": 0.5 + }, + { + "color": "red", + "value": 2 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + } + }, + "version": "13.0.0-23630096546" + } } }, "panel-19": { "kind": "Panel", "spec": { - "data": { "kind": "QueryGroup", "spec": { "queries": [{ "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(rate(airi_billing_flux_consumed_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))", "legendFormat": "flux/s" }, "version": "v0" }, "refId": "A" } }], "queryOptions": {}, "transformations": [] } }, + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "grafanacloud-projairi-prom" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum(rate(airi_billing_flux_consumed_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))", + "legendFormat": "flux/s" + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, "description": "", "id": 19, "links": [], "title": "Total Flux Consumed", - "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "yellow", "value": 100 }, { "color": "red", "value": 1000 }] }, "unit": "short" }, "overrides": [] }, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true } }, "version": "13.0.0-23630096546" } + "vizConfig": { + "group": "stat", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "yellow", + "value": 100 + }, + { + "color": "red", + "value": 1000 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + } + }, + "version": "13.0.0-23630096546" + } } }, "panel-20": { "kind": "Panel", "spec": { - "data": { "kind": "QueryGroup", "spec": { "queries": [{ "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (gen_ai_request_model) (rate(gen_ai_client_operation_count_total{service_name=~\"$service\", deployment_environment=~\"$env\", gen_ai_request_model!=\"\"}[$__rate_interval]))", "legendFormat": "{{gen_ai_request_model}}" }, "version": "v0" }, "refId": "A" } }], "queryOptions": {}, "transformations": [] } }, + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "grafanacloud-projairi-prom" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum by (gen_ai_request_model) (rate(gen_ai_client_operation_count_total{service_name=~\"$service\", deployment_environment=~\"$env\", gen_ai_request_model!=\"\"}[$__rate_interval]))", + "legendFormat": "{{gen_ai_request_model}}" + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, "description": "", "id": 20, "links": [], "title": "LLM Request Rate by Model", - "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "red", "value": 80 }] }, "unit": "reqps" }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.0.0-23630096546" } + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.0.0-23630096546" + } } }, "panel-21": { "kind": "Panel", "spec": { - "data": { "kind": "QueryGroup", "spec": { "queries": [{ "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (gen_ai_operation_name) (rate(gen_ai_client_operation_count_total{service_name=~\"$service\", deployment_environment=~\"$env\", gen_ai_operation_name!=\"\"}[$__rate_interval]))", "legendFormat": "{{gen_ai_operation_name}}" }, "version": "v0" }, "refId": "A" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (airi_gen_ai_operation_kind) (rate(gen_ai_client_operation_count_total{service_name=~\"$service\", deployment_environment=~\"$env\", airi_gen_ai_operation_kind!=\"\"}[$__rate_interval]))", "legendFormat": "{{airi_gen_ai_operation_kind}}" }, "version": "v0" }, "refId": "B" } }], "queryOptions": {}, "transformations": [] } }, + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "grafanacloud-projairi-prom" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum by (gen_ai_operation_name) (rate(gen_ai_client_operation_count_total{service_name=~\"$service\", deployment_environment=~\"$env\", gen_ai_operation_name!=\"\"}[$__rate_interval]))", + "legendFormat": "{{gen_ai_operation_name}}" + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "grafanacloud-projairi-prom" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum by (airi_gen_ai_operation_kind) (rate(gen_ai_client_operation_count_total{service_name=~\"$service\", deployment_environment=~\"$env\", airi_gen_ai_operation_kind!=\"\"}[$__rate_interval]))", + "legendFormat": "{{airi_gen_ai_operation_kind}}" + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, "description": "", "id": 21, "links": [], "title": "LLM Request Rate by Type", - "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "red", "value": 80 }] }, "unit": "reqps" }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.0.0-23630096546" } + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.0.0-23630096546" + } } }, "panel-22": { "kind": "Panel", "spec": { - "data": { "kind": "QueryGroup", "spec": { "queries": [{ "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "histogram_quantile(0.95, sum by (le, gen_ai_request_model) (rate(gen_ai_client_operation_duration_seconds_bucket{service_name=~\"$service\", deployment_environment=~\"$env\", gen_ai_request_model!=\"\"}[$__rate_interval])))", "legendFormat": "{{gen_ai_request_model}}" }, "version": "v0" }, "refId": "A" } }], "queryOptions": {}, "transformations": [] } }, + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "grafanacloud-projairi-prom" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "histogram_quantile(0.95, sum by (le, gen_ai_request_model) (rate(gen_ai_client_operation_duration_seconds_bucket{service_name=~\"$service\", deployment_environment=~\"$env\", gen_ai_request_model!=\"\"}[$__rate_interval])))", + "legendFormat": "{{gen_ai_request_model}}" + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, "description": "", "id": 22, "links": [], "title": "LLM Latency by Model (P95)", - "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "red", "value": 80 }] }, "unit": "s" }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.0.0-23630096546" } + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.0.0-23630096546" + } } }, "panel-23": { "kind": "Panel", "spec": { - "data": { "kind": "QueryGroup", "spec": { "queries": [{ "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (gen_ai_request_model) (rate(airi_billing_flux_consumed_total{service_name=~\"$service\", deployment_environment=~\"$env\", gen_ai_request_model!=\"\"}[$__rate_interval]))", "legendFormat": "{{gen_ai_request_model}}" }, "version": "v0" }, "refId": "A" } }], "queryOptions": {}, "transformations": [] } }, - "description": "", + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "grafanacloud-projairi-prom" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum by (gen_ai_request_model) (rate(gen_ai_client_token_usage_input_total{service_name=~\"$service\", deployment_environment=~\"$env\", gen_ai_request_model!=\"\"}[$__rate_interval]))", + "legendFormat": "{{gen_ai_request_model}} (input)" + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "grafanacloud-projairi-prom" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum by (gen_ai_request_model) (rate(gen_ai_client_token_usage_output_total{service_name=~\"$service\", deployment_environment=~\"$env\", gen_ai_request_model!=\"\"}[$__rate_interval]))", + "legendFormat": "{{gen_ai_request_model}} (output)" + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Prompt + completion tokens per model, stacked. More directly meaningful than Flux which is an abstracted billing unit.", "id": 23, "links": [], - "title": "Flux Consumed by Model", - "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "red", "value": 80 }] }, "unit": "short" }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.0.0-23630096546" } + "title": "Total Tokens by Model", + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.0.0-23630096546" + } } }, "panel-24": { "kind": "Panel", "spec": { - "data": { "kind": "QueryGroup", "spec": { "queries": [{ "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (gen_ai_request_model) (rate(gen_ai_client_token_usage_input_total{service_name=~\"$service\", deployment_environment=~\"$env\", gen_ai_request_model!=\"\"}[$__rate_interval]))", "legendFormat": "{{gen_ai_request_model}}" }, "version": "v0" }, "refId": "A" } }], "queryOptions": {}, "transformations": [] } }, + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "grafanacloud-projairi-prom" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum by (gen_ai_request_model) (rate(gen_ai_client_token_usage_input_total{service_name=~\"$service\", deployment_environment=~\"$env\", gen_ai_request_model!=\"\"}[$__rate_interval]))", + "legendFormat": "{{gen_ai_request_model}}" + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, "description": "", "id": 24, "links": [], "title": "Prompt Tokens by Model", - "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "red", "value": 80 }] }, "unit": "short" }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.0.0-23630096546" } + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.0.0-23630096546" + } } }, "panel-25": { "kind": "Panel", "spec": { - "data": { "kind": "QueryGroup", "spec": { "queries": [{ "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (gen_ai_request_model) (rate(gen_ai_client_token_usage_output_total{service_name=~\"$service\", deployment_environment=~\"$env\", gen_ai_request_model!=\"\"}[$__rate_interval]))", "legendFormat": "{{gen_ai_request_model}}" }, "version": "v0" }, "refId": "A" } }], "queryOptions": {}, "transformations": [] } }, + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "grafanacloud-projairi-prom" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum by (gen_ai_request_model) (rate(gen_ai_client_token_usage_output_total{service_name=~\"$service\", deployment_environment=~\"$env\", gen_ai_request_model!=\"\"}[$__rate_interval]))", + "legendFormat": "{{gen_ai_request_model}}" + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, "description": "", "id": 25, "links": [], "title": "Completion Tokens by Model", - "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "red", "value": 80 }] }, "unit": "short" }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.0.0-23630096546" } + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.0.0-23630096546" + } } }, "panel-40": { @@ -1392,7 +2034,9 @@ "spec": { "hidden": false, "query": { - "datasource": { "name": "grafanacloud-projairi-prom" }, + "datasource": { + "name": "grafanacloud-projairi-prom" + }, "group": "prometheus", "kind": "DataQuery", "spec": { @@ -1419,13 +2063,24 @@ "spec": { "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": 0 }, - { "color": "yellow", "value": 1 }, - { "color": "red", "value": 10 } + { + "color": "green", + "value": 0 + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "red", + "value": 10 + } ] }, "unit": "currencyUSD" @@ -1438,7 +2093,13 @@ "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "showPercentChange": false, "textMode": "auto", "wideLayout": true @@ -1460,7 +2121,9 @@ "spec": { "hidden": false, "query": { - "datasource": { "name": "grafanacloud-projairi-prom" }, + "datasource": { + "name": "grafanacloud-projairi-prom" + }, "group": "prometheus", "kind": "DataQuery", "spec": { @@ -1487,11 +2150,16 @@ "spec": { "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "thresholds": { "mode": "absolute", "steps": [ - { "color": "green", "value": 0 } + { + "color": "green", + "value": 0 + } ] }, "unit": "short" @@ -1504,7 +2172,13 @@ "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", - "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, "showPercentChange": false, "textMode": "auto", "wideLayout": true @@ -1526,7 +2200,9 @@ "spec": { "hidden": false, "query": { - "datasource": { "name": "grafanacloud-projairi-prom" }, + "datasource": { + "name": "grafanacloud-projairi-prom" + }, "group": "prometheus", "kind": "DataQuery", "spec": { @@ -1553,7 +2229,9 @@ "spec": { "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, @@ -1565,27 +2243,62 @@ "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", - "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, - "scaleDistribution": { "type": "linear" }, + "scaleDistribution": { + "type": "linear" + }, "showPoints": "auto", "showValues": false, "spanNulls": false, - "stacking": { "group": "A", "mode": "none" }, - "thresholdsStyle": { "mode": "off" } + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] }, - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "red", "value": 80 }] }, "unit": "short" }, "overrides": [] }, "options": { - "annotations": { "clustering": -1, "multiLane": false }, - "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, - "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } } }, "version": "13.0.0-23630096546" @@ -1604,7 +2317,9 @@ "spec": { "hidden": false, "query": { - "datasource": { "name": "grafanacloud-projairi-prom" }, + "datasource": { + "name": "grafanacloud-projairi-prom" + }, "group": "prometheus", "kind": "DataQuery", "spec": { @@ -1631,7 +2346,9 @@ "spec": { "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, @@ -1643,27 +2360,62 @@ "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", - "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, - "scaleDistribution": { "type": "linear" }, + "scaleDistribution": { + "type": "linear" + }, "showPoints": "auto", "showValues": false, "spanNulls": false, - "stacking": { "group": "A", "mode": "none" }, - "thresholdsStyle": { "mode": "off" } + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] }, - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "red", "value": 80 }] }, "unit": "short" }, "overrides": [] }, "options": { - "annotations": { "clustering": -1, "multiLane": false }, - "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, - "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } } }, "version": "13.0.0-23630096546" @@ -1682,7 +2434,9 @@ "spec": { "hidden": false, "query": { - "datasource": { "name": "grafanacloud-projairi-prom" }, + "datasource": { + "name": "grafanacloud-projairi-prom" + }, "group": "prometheus", "kind": "DataQuery", "spec": { @@ -1699,7 +2453,9 @@ "spec": { "hidden": false, "query": { - "datasource": { "name": "grafanacloud-projairi-prom" }, + "datasource": { + "name": "grafanacloud-projairi-prom" + }, "group": "prometheus", "kind": "DataQuery", "spec": { @@ -1726,7 +2482,9 @@ "spec": { "fieldConfig": { "defaults": { - "color": { "mode": "palette-classic" }, + "color": { + "mode": "palette-classic" + }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, @@ -1738,27 +2496,62 @@ "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", - "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, - "scaleDistribution": { "type": "linear" }, + "scaleDistribution": { + "type": "linear" + }, "showPoints": "auto", "showValues": false, "spanNulls": false, - "stacking": { "group": "A", "mode": "none" }, - "thresholdsStyle": { "mode": "off" } + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] }, - "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "red", "value": 80 }] }, "unit": "short" }, "overrides": [] }, "options": { - "annotations": { "clustering": -1, "multiLane": false }, - "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, - "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } } }, "version": "13.0.0-23630096546" @@ -1768,221 +2561,2213 @@ "panel-34": { "kind": "Panel", "spec": { - "data": { "kind": "QueryGroup", "spec": { "queries": [{ "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "100 * sum(rate(auth_failures_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval])) / clamp_min(sum(rate(auth_attempts_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval])), 1)", "legendFormat": "fail %" }, "version": "v0" }, "refId": "A" } }], "queryOptions": {}, "transformations": [] } }, + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "grafanacloud-projairi-prom" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "100 * sum(rate(auth_failures_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval])) / clamp_min(sum(rate(auth_attempts_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval])), 1)", + "legendFormat": "fail %" + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, "description": "auth_failures / auth_attempts; high values mean credential stuffing or upstream OAuth breakage.", "id": 34, "links": [], "title": "Auth Failure Rate %", - "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "yellow", "value": 10 }, { "color": "red", "value": 30 }] }, "unit": "percent" }, "overrides": [] }, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true } }, "version": "13.0.0-23630096546" } + "vizConfig": { + "group": "stat", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "yellow", + "value": 10 + }, + { + "color": "red", + "value": 30 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + } + }, + "version": "13.0.0-23630096546" + } } }, "panel-50": { "kind": "Panel", "spec": { - "data": { "kind": "QueryGroup", "spec": { "queries": [{ "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(ws_connections_active{service_name=~\"$service\", deployment_environment=~\"$env\"})", "legendFormat": "active" }, "version": "v0" }, "refId": "A" } }], "queryOptions": {}, "transformations": [] } }, + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "grafanacloud-projairi-prom" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum(ws_connections_active{service_name=~\"$service\", deployment_environment=~\"$env\"})", + "legendFormat": "active" + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, "description": "Sum of active WS connections across all instances.", "id": 50, "links": [], "title": "WS Connections Active", - "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "yellow", "value": 1000 }, { "color": "red", "value": 10000 }] }, "unit": "short" }, "overrides": [] }, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true } }, "version": "13.0.0-23630096546" } + "vizConfig": { + "group": "stat", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "yellow", + "value": 1000 + }, + { + "color": "red", + "value": 10000 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + } + }, + "version": "13.0.0-23630096546" + } } }, "panel-51": { "kind": "Panel", "spec": { - "data": { "kind": "QueryGroup", "spec": { "queries": [{ "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(rate(ws_messages_sent_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))", "legendFormat": "sent/s" }, "version": "v0" }, "refId": "A" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(rate(ws_messages_received_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))", "legendFormat": "received/s" }, "version": "v0" }, "refId": "B" } }], "queryOptions": {}, "transformations": [] } }, + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "grafanacloud-projairi-prom" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum(rate(ws_messages_sent_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))", + "legendFormat": "sent/s" + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "grafanacloud-projairi-prom" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum(rate(ws_messages_received_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))", + "legendFormat": "received/s" + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, "description": "WebSocket throughput in both directions.", "id": 51, "links": [], "title": "WS Messages I/O", - "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "red", "value": 80 }] }, "unit": "ops" }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.0.0-23630096546" } + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.0.0-23630096546" + } } }, "panel-52": { "kind": "Panel", "spec": { - "data": { "kind": "QueryGroup", "spec": { "queries": [{ "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(rate(chat_messages_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))", "legendFormat": "chat msgs/s" }, "version": "v0" }, "refId": "A" } }], "queryOptions": {}, "transformations": [] } }, + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "grafanacloud-projairi-prom" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum(rate(chat_messages_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))", + "legendFormat": "chat msgs/s" + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, "description": "Chat messages persisted to chat sync (chats.pushMessages).", "id": 52, "links": [], "title": "Chat Messages Rate", - "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "red", "value": 80 }] }, "unit": "ops" }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.0.0-23630096546" } + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.0.0-23630096546" + } } }, "panel-53": { "kind": "Panel", "spec": { - "data": { "kind": "QueryGroup", "spec": { "queries": [{ "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(rate(character_created_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))", "legendFormat": "created/s" }, "version": "v0" }, "refId": "A" } }, { "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(rate(character_deleted_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))", "legendFormat": "deleted/s" }, "version": "v0" }, "refId": "B" } }], "queryOptions": {}, "transformations": [] } }, + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "grafanacloud-projairi-prom" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum(rate(character_created_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))", + "legendFormat": "created/s" + }, + "version": "v0" + }, + "refId": "A" + } + }, + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "grafanacloud-projairi-prom" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum(rate(character_deleted_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))", + "legendFormat": "deleted/s" + }, + "version": "v0" + }, + "refId": "B" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, "description": "", "id": 53, "links": [], "title": "Character Lifecycle", - "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "red", "value": 80 }] }, "unit": "ops" }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.0.0-23630096546" } + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.0.0-23630096546" + } } }, "panel-54": { "kind": "Panel", "spec": { - "data": { "kind": "QueryGroup", "spec": { "queries": [{ "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (action) (rate(character_engagement_total{service_name=~\"$service\", deployment_environment=~\"$env\", action!=\"\"}[$__rate_interval]))", "legendFormat": "{{action}}" }, "version": "v0" }, "refId": "A" } }], "queryOptions": {}, "transformations": [] } }, + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "grafanacloud-projairi-prom" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum by (action) (rate(character_engagement_total{service_name=~\"$service\", deployment_environment=~\"$env\", action!=\"\"}[$__rate_interval]))", + "legendFormat": "{{action}}" + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, "description": "Like / unlike / bookmark / unbookmark actions on characters.", "id": 54, "links": [], "title": "Character Engagement", - "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "red", "value": 80 }] }, "unit": "ops" }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.0.0-23630096546" } + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.0.0-23630096546" + } } }, "panel-55": { "kind": "Panel", "spec": { - "data": { "kind": "QueryGroup", "spec": { "queries": [{ "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (event_type) (rate(stripe_events_total{service_name=~\"$service\", deployment_environment=~\"$env\", event_type!=\"\"}[$__rate_interval]))", "legendFormat": "{{event_type}}" }, "version": "v0" }, "refId": "A" } }], "queryOptions": {}, "transformations": [] } }, + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "grafanacloud-projairi-prom" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum by (event_type) (rate(stripe_events_total{service_name=~\"$service\", deployment_environment=~\"$env\", event_type!=\"\"}[$__rate_interval]))", + "legendFormat": "{{event_type}}" + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, "description": "All Stripe webhook events grouped by event.type.", "id": 55, "links": [], "title": "Stripe Events by Type", - "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "red", "value": 80 }] }, "unit": "ops" }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.0.0-23630096546" } + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.0.0-23630096546" + } } }, "panel-56": { "kind": "Panel", "spec": { - "data": { "kind": "QueryGroup", "spec": { "queries": [{ "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (event_type) (rate(stripe_subscription_event_total{service_name=~\"$service\", deployment_environment=~\"$env\", event_type!=\"\"}[$__rate_interval]))", "legendFormat": "{{event_type}}" }, "version": "v0" }, "refId": "A" } }], "queryOptions": {}, "transformations": [] } }, + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "grafanacloud-projairi-prom" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum by (event_type) (rate(stripe_subscription_event_total{service_name=~\"$service\", deployment_environment=~\"$env\", event_type!=\"\"}[$__rate_interval]))", + "legendFormat": "{{event_type}}" + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, "description": "Subscription lifecycle events (created/updated/deleted).", "id": 56, "links": [], "title": "Subscription Events by Type", - "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "red", "value": 80 }] }, "unit": "ops" }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.0.0-23630096546" } + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.0.0-23630096546" + } } }, "panel-57": { "kind": "Panel", "spec": { - "data": { "kind": "QueryGroup", "spec": { "queries": [{ "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum(rate(stripe_payment_failed_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))", "legendFormat": "failed/s" }, "version": "v0" }, "refId": "A" } }], "queryOptions": {}, "transformations": [] } }, + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "grafanacloud-projairi-prom" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum(rate(stripe_payment_failed_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))", + "legendFormat": "failed/s" + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, "description": "invoice.payment_failed events. Spikes mean dunning storms or card decline cascades.", "id": 57, "links": [], "title": "Payment Failures", - "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "yellow", "value": 0.01 }, { "color": "red", "value": 0.1 }] }, "unit": "ops" }, "overrides": [] }, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true } }, "version": "13.0.0-23630096546" } + "vizConfig": { + "group": "stat", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "yellow", + "value": 0.01 + }, + { + "color": "red", + "value": 0.1 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + } + }, + "version": "13.0.0-23630096546" + } } }, "panel-58": { "kind": "Panel", "spec": { - "data": { "kind": "QueryGroup", "spec": { "queries": [{ "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "100 * sum(increase(stripe_checkout_completed_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__range])) / clamp_min(sum(increase(stripe_checkout_created_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__range])), 1)", "legendFormat": "completed %" }, "version": "v0" }, "refId": "A" } }], "queryOptions": {}, "transformations": [] } }, + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "grafanacloud-projairi-prom" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "100 * sum(increase(stripe_checkout_completed_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__range])) / clamp_min(sum(increase(stripe_checkout_created_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__range])), 1)", + "legendFormat": "completed %" + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, "description": "Checkouts completed / created over the dashboard time range. Drops can flag price-page bugs or payment-method outages.", "id": 58, "links": [], "title": "Checkout Conversion", - "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": 0 }, { "color": "yellow", "value": 30 }, { "color": "green", "value": 60 }] }, "unit": "percent" }, "overrides": [] }, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true } }, "version": "13.0.0-23630096546" } + "vizConfig": { + "group": "stat", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": 0 + }, + { + "color": "yellow", + "value": 30 + }, + { + "color": "green", + "value": 60 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + } + }, + "version": "13.0.0-23630096546" + } } }, "panel-60": { "kind": "Panel", "spec": { - "data": { "kind": "QueryGroup", "spec": { "queries": [{ "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (source) (rate(airi_billing_flux_credited_total{service_name=~\"$service\", deployment_environment=~\"$env\", source!=\"\"}[$__rate_interval]))", "legendFormat": "{{source}}" }, "version": "v0" }, "refId": "A" } }], "queryOptions": {}, "transformations": [] } }, + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "grafanacloud-projairi-prom" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum by (source) (rate(airi_billing_flux_credited_total{service_name=~\"$service\", deployment_environment=~\"$env\", source!=\"\"}[$__rate_interval]))", + "legendFormat": "{{source}}" + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, "description": "Flux added to user balances, broken down by source: stripe.checkout / stripe.invoice / promo / admin grants.", "id": 60, "links": [], "title": "Flux Credited by Source", - "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }] }, "unit": "short" }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.0.0-23630096546" } + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.0.0-23630096546" + } } }, "panel-61": { "kind": "Panel", "spec": { - "data": { "kind": "QueryGroup", "spec": { "queries": [{ "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (currency) (rate(airi_stripe_revenue_minor_unit_total{service_name=~\"$service\", deployment_environment=~\"$env\", currency!=\"\"}[$__rate_interval])) / 100", "legendFormat": "{{currency}}" }, "version": "v0" }, "refId": "A" } }], "queryOptions": {}, "transformations": [] } }, + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "grafanacloud-projairi-prom" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum by (currency) (rate(airi_stripe_revenue_minor_unit_total{service_name=~\"$service\", deployment_environment=~\"$env\", currency!=\"\"}[$__rate_interval])) / 100", + "legendFormat": "{{currency}}" + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, "description": "Real revenue per second, in major currency unit (cents -> dollars / equivalent). Always grouped by currency — cross-currency sums are meaningless.", "id": 61, "links": [], "title": "Stripe Revenue Rate (major unit/s)", - "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }] }, "unit": "short" }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.0.0-23630096546" } + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.0.0-23630096546" + } } }, "panel-62": { "kind": "Panel", "spec": { - "data": { "kind": "QueryGroup", "spec": { "queries": [{ "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (gen_ai_request_model) (rate(airi_billing_flux_unbilled_total{service_name=~\"$service\", deployment_environment=~\"$env\", gen_ai_request_model!=\"\"}[$__rate_interval]))", "legendFormat": "{{gen_ai_request_model}}" }, "version": "v0" }, "refId": "A" } }], "queryOptions": {}, "transformations": [] } }, + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "grafanacloud-projairi-prom" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum by (gen_ai_request_model) (rate(airi_billing_flux_unbilled_total{service_name=~\"$service\", deployment_environment=~\"$env\", gen_ai_request_model!=\"\"}[$__rate_interval]))", + "legendFormat": "{{gen_ai_request_model}}" + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, "description": "Flux that should have been debited but wasn't (debit failure after streaming response sent). Real revenue leak — anything above 0 sustained should page.", "id": 62, "links": [], "title": "Flux Unbilled (revenue leak)", - "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 30, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "normal" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "red", "value": 0.001 }] }, "unit": "short" }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.0.0-23630096546" } + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 0.001 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.0.0-23630096546" + } } }, "panel-63": { "kind": "Panel", "spec": { - "data": { "kind": "QueryGroup", "spec": { "queries": [{ "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (gen_ai_request_model, stage) (rate(airi_gen_ai_stream_interrupted_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))", "legendFormat": "{{gen_ai_request_model}} ({{stage}})" }, "version": "v0" }, "refId": "A" } }], "queryOptions": {}, "transformations": [] } }, + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "grafanacloud-projairi-prom" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum by (gen_ai_request_model, stage) (rate(airi_gen_ai_stream_interrupted_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))", + "legendFormat": "{{gen_ai_request_model}} ({{stage}})" + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, "description": "Streaming responses that died mid-flight. `before_first_chunk` = upstream blew up before any token; `mid_stream` = partial delivery.", "id": 63, "links": [], "title": "Stream Interruptions", - "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "red", "value": 0.5 }] }, "unit": "ops" }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.0.0-23630096546" } + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 0.5 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.0.0-23630096546" + } } }, "panel-64": { "kind": "Panel", "spec": { - "data": { "kind": "QueryGroup", "spec": { "queries": [{ "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "histogram_quantile(0.95, sum by (le, gen_ai_request_model) (rate(gen_ai_client_first_token_duration_seconds_bucket{service_name=~\"$service\", deployment_environment=~\"$env\", gen_ai_request_model!=\"\"}[$__rate_interval])))", "legendFormat": "{{gen_ai_request_model}}" }, "version": "v0" }, "refId": "A" } }], "queryOptions": {}, "transformations": [] } }, + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "grafanacloud-projairi-prom" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "histogram_quantile(0.95, sum by (le, gen_ai_request_model) (rate(gen_ai_client_first_token_duration_seconds_bucket{service_name=~\"$service\", deployment_environment=~\"$env\", gen_ai_request_model!=\"\"}[$__rate_interval])))", + "legendFormat": "{{gen_ai_request_model}}" + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, "description": "Time from request start to first streamed token (TTFB). Tracks streaming chat experience independently from total operation duration.", "id": 64, "links": [], "title": "First Token P95 Latency", - "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "yellow", "value": 1 }, { "color": "red", "value": 3 }] }, "unit": "s" }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.0.0-23630096546" } + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "red", + "value": 3 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.0.0-23630096546" + } } }, "panel-65": { "kind": "Panel", "spec": { - "data": { "kind": "QueryGroup", "spec": { "queries": [{ "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (model) (rate(airi_billing_tts_chars_total{service_name=~\"$service\", deployment_environment=~\"$env\", model!=\"\"}[$__rate_interval]))", "legendFormat": "{{model}}" }, "version": "v0" }, "refId": "A" } }], "queryOptions": {}, "transformations": [] } }, + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "grafanacloud-projairi-prom" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum by (model) (rate(airi_billing_tts_chars_total{service_name=~\"$service\", deployment_environment=~\"$env\", model!=\"\"}[$__rate_interval]))", + "legendFormat": "{{model}}" + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, "description": "TTS input characters processed per second — billing base before whole-Flux rounding. Spikes here without proportional flux_consumed = debt accumulating in Redis.", "id": 65, "links": [], "title": "TTS Chars Rate", - "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }] }, "unit": "short" }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.0.0-23630096546" } + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.0.0-23630096546" + } } }, "panel-66": { "kind": "Panel", "spec": { - "data": { "kind": "QueryGroup", "spec": { "queries": [{ "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (meter, reason) (rate(airi_billing_tts_preflight_rejections_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))", "legendFormat": "{{meter}} ({{reason}})" }, "version": "v0" }, "refId": "A" } }], "queryOptions": {}, "transformations": [] } }, + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "grafanacloud-projairi-prom" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum by (meter, reason) (rate(airi_billing_tts_preflight_rejections_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))", + "legendFormat": "{{meter}} ({{reason}})" + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, "description": "Pre-flight rejections from flux-meter assertCanAfford. Volume here = users hitting paywall before upstream call.", "id": 66, "links": [], "title": "TTS Preflight Rejections", - "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }] }, "unit": "ops" }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.0.0-23630096546" } + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.0.0-23630096546" + } } }, "panel-67": { "kind": "Panel", "spec": { - "data": { "kind": "QueryGroup", "spec": { "queries": [{ "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (template) (rate(airi_email_send_total{service_name=~\"$service\", deployment_environment=~\"$env\", template!=\"\"}[$__rate_interval]))", "legendFormat": "{{template}}" }, "version": "v0" }, "refId": "A" } }], "queryOptions": {}, "transformations": [] } }, + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "grafanacloud-projairi-prom" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum by (template) (rate(airi_email_send_total{service_name=~\"$service\", deployment_environment=~\"$env\", template!=\"\"}[$__rate_interval]))", + "legendFormat": "{{template}}" + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, "description": "Resend accepted emails by template (verification / password_reset / magic_link / change_email / delete_account).", "id": 67, "links": [], "title": "Email Send Rate", - "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }] }, "unit": "ops" }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.0.0-23630096546" } + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.0.0-23630096546" + } } }, "panel-68": { "kind": "Panel", "spec": { - "data": { "kind": "QueryGroup", "spec": { "queries": [{ "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "100 * sum(rate(airi_email_failures_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval])) / clamp_min(sum(rate(airi_email_send_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval])) + sum(rate(airi_email_failures_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval])), 1)", "legendFormat": "fail %" }, "version": "v0" }, "refId": "A" } }], "queryOptions": {}, "transformations": [] } }, + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "grafanacloud-projairi-prom" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "100 * sum(rate(airi_email_failures_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval])) / clamp_min(sum(rate(airi_email_send_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval])) + sum(rate(airi_email_failures_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval])), 1)", + "legendFormat": "fail %" + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, "description": "Email failures / total attempts. >5% means Resend / DNS / suppression list issues blocking auth flows.", "id": 68, "links": [], "title": "Email Failure Rate %", - "vizConfig": { "group": "stat", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "yellow", "value": 1 }, { "color": "red", "value": 5 }] }, "unit": "percent" }, "overrides": [] }, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true } }, "version": "13.0.0-23630096546" } + "vizConfig": { + "group": "stat", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "red", + "value": 5 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + } + }, + "version": "13.0.0-23630096546" + } } }, "panel-69": { "kind": "Panel", "spec": { - "data": { "kind": "QueryGroup", "spec": { "queries": [{ "kind": "PanelQuery", "spec": { "hidden": false, "query": { "datasource": { "name": "grafanacloud-projairi-prom" }, "group": "prometheus", "kind": "DataQuery", "spec": { "expr": "sum by (route, key_type) (rate(airi_rate_limit_blocked_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))", "legendFormat": "{{route}} ({{key_type}})" }, "version": "v0" }, "refId": "A" } }], "queryOptions": {}, "transformations": [] } }, + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "grafanacloud-projairi-prom" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "sum by (route, key_type) (rate(airi_rate_limit_blocked_total{service_name=~\"$service\", deployment_environment=~\"$env\"}[$__rate_interval]))", + "legendFormat": "{{route}} ({{key_type}})" + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, "description": "Requests denied by rate limiter, broken down by route + key type (user vs ip). Sustained ip-keyed traffic = unauthenticated abuse / scraping.", "id": 69, "links": [], "title": "Rate Limit Blocked", - "vizConfig": { "group": "timeseries", "kind": "VizConfig", "spec": { "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 20, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "smooth", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "showValues": false, "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": 0 }, { "color": "red", "value": 1 }] }, "unit": "ops" }, "overrides": [] }, "options": { "annotations": { "clustering": -1, "multiLane": false }, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } } }, "version": "13.0.0-23630096546" } + "vizConfig": { + "group": "timeseries", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "options": { + "annotations": { + "clustering": -1, + "multiLane": false + }, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + } + }, + "version": "13.0.0-23630096546" + } } } }, @@ -1998,31 +4783,58 @@ "kind": "GridLayout", "spec": { "items": [ - { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-1" }, "height": 8, "width": 8, "x": 0, "y": 0 } }, - { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-2" }, "height": 8, "width": 8, "x": 8, "y": 0 } }, - { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-3" }, "height": 8, "width": 8, "x": 16, "y": 0 } }, - { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-4" }, "height": 8, "width": 12, "x": 0, "y": 8 } }, - { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-5" }, "height": 8, "width": 12, "x": 12, "y": 8 } }, - { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-6" }, "height": 8, "width": 12, "x": 0, "y": 16 } }, - { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-30" }, "height": 8, "width": 12, "x": 12, "y": 16 } } - ] - } - }, - "title": "HTTP Overview" - } - }, - { - "kind": "RowsLayoutRow", - "spec": { - "collapse": false, - "layout": { - "kind": "GridLayout", - "spec": { - "items": [ - { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-31" }, "height": 8, "width": 6, "x": 0, "y": 0 } }, - { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-34" }, "height": 8, "width": 6, "x": 6, "y": 0 } }, - { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-32" }, "height": 8, "width": 6, "x": 12, "y": 0 } }, - { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-33" }, "height": 8, "width": 6, "x": 18, "y": 0 } } + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-31" + }, + "height": 8, + "width": 6, + "x": 0, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-34" + }, + "height": 8, + "width": 6, + "x": 6, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-32" + }, + "height": 8, + "width": 6, + "x": 12, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-33" + }, + "height": 8, + "width": 6, + "x": 18, + "y": 0 + } + } ] } }, @@ -2037,11 +4849,71 @@ "kind": "GridLayout", "spec": { "items": [ - { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-50" }, "height": 8, "width": 6, "x": 0, "y": 0 } }, - { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-51" }, "height": 8, "width": 9, "x": 6, "y": 0 } }, - { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-52" }, "height": 8, "width": 9, "x": 15, "y": 0 } }, - { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-53" }, "height": 8, "width": 12, "x": 0, "y": 8 } }, - { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-54" }, "height": 8, "width": 12, "x": 12, "y": 8 } } + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-50" + }, + "height": 8, + "width": 6, + "x": 0, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-51" + }, + "height": 8, + "width": 9, + "x": 6, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-52" + }, + "height": 8, + "width": 9, + "x": 15, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-53" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 8 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-54" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 8 + } + } ] } }, @@ -2056,12 +4928,84 @@ "kind": "GridLayout", "spec": { "items": [ - { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-40" }, "height": 8, "width": 8, "x": 0, "y": 0 } }, - { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-41" }, "height": 8, "width": 8, "x": 8, "y": 0 } }, - { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-44" }, "height": 8, "width": 8, "x": 16, "y": 0 } }, - { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-42" }, "height": 8, "width": 8, "x": 0, "y": 8 } }, - { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-43" }, "height": 8, "width": 8, "x": 8, "y": 8 } }, - { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-60" }, "height": 8, "width": 8, "x": 16, "y": 8 } } + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-40" + }, + "height": 8, + "width": 8, + "x": 0, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-41" + }, + "height": 8, + "width": 8, + "x": 8, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-44" + }, + "height": 8, + "width": 8, + "x": 16, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-42" + }, + "height": 8, + "width": 8, + "x": 0, + "y": 8 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-43" + }, + "height": 8, + "width": 8, + "x": 8, + "y": 8 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-60" + }, + "height": 8, + "width": 8, + "x": 16, + "y": 8 + } + } ] } }, @@ -2076,11 +5020,71 @@ "kind": "GridLayout", "spec": { "items": [ - { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-57" }, "height": 8, "width": 6, "x": 0, "y": 0 } }, - { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-58" }, "height": 8, "width": 6, "x": 6, "y": 0 } }, - { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-55" }, "height": 8, "width": 12, "x": 12, "y": 0 } }, - { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-56" }, "height": 8, "width": 12, "x": 0, "y": 8 } }, - { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-61" }, "height": 8, "width": 12, "x": 12, "y": 8 } } + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-57" + }, + "height": 8, + "width": 6, + "x": 0, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-58" + }, + "height": 8, + "width": 6, + "x": 6, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-55" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-56" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 8 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-61" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 8 + } + } ] } }, @@ -2095,10 +5099,58 @@ "kind": "GridLayout", "spec": { "items": [ - { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-7" }, "height": 8, "width": 12, "x": 0, "y": 0 } }, - { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-8" }, "height": 8, "width": 12, "x": 12, "y": 0 } }, - { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-9" }, "height": 8, "width": 12, "x": 0, "y": 8 } }, - { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-10" }, "height": 8, "width": 12, "x": 12, "y": 8 } } + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-7" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-8" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-9" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 8 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-10" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 8 + } + } ] } }, @@ -2113,19 +5165,175 @@ "kind": "GridLayout", "spec": { "items": [ - { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-17" }, "height": 8, "width": 8, "x": 0, "y": 0 } }, - { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-18" }, "height": 8, "width": 8, "x": 8, "y": 0 } }, - { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-19" }, "height": 8, "width": 8, "x": 16, "y": 0 } }, - { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-20" }, "height": 8, "width": 12, "x": 0, "y": 8 } }, - { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-21" }, "height": 8, "width": 12, "x": 12, "y": 8 } }, - { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-22" }, "height": 8, "width": 12, "x": 0, "y": 16 } }, - { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-64" }, "height": 8, "width": 12, "x": 12, "y": 16 } }, - { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-23" }, "height": 8, "width": 12, "x": 0, "y": 24 } }, - { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-62" }, "height": 8, "width": 12, "x": 12, "y": 24 } }, - { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-63" }, "height": 8, "width": 12, "x": 0, "y": 32 } }, - { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-65" }, "height": 8, "width": 12, "x": 12, "y": 32 } }, - { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-24" }, "height": 8, "width": 12, "x": 0, "y": 40 } }, - { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-25" }, "height": 8, "width": 12, "x": 12, "y": 40 } } + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-17" + }, + "height": 8, + "width": 8, + "x": 0, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-18" + }, + "height": 8, + "width": 8, + "x": 8, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-19" + }, + "height": 8, + "width": 8, + "x": 16, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-20" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 8 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-21" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 8 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-22" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 16 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-64" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 16 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-23" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 24 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-62" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 24 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-63" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 32 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-65" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 32 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-24" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 40 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-25" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 40 + } + } ] } }, @@ -2140,10 +5348,58 @@ "kind": "GridLayout", "spec": { "items": [ - { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-68" }, "height": 8, "width": 6, "x": 0, "y": 0 } }, - { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-67" }, "height": 8, "width": 18, "x": 6, "y": 0 } }, - { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-69" }, "height": 8, "width": 12, "x": 0, "y": 8 } }, - { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-66" }, "height": 8, "width": 12, "x": 12, "y": 8 } } + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-68" + }, + "height": 8, + "width": 6, + "x": 0, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-67" + }, + "height": 8, + "width": 18, + "x": 6, + "y": 0 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-69" + }, + "height": 8, + "width": 12, + "x": 0, + "y": 8 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-66" + }, + "height": 8, + "width": 12, + "x": 12, + "y": 8 + } + } ] } }, @@ -2158,7 +5414,19 @@ "kind": "GridLayout", "spec": { "items": [ - { "kind": "GridLayoutItem", "spec": { "element": { "kind": "ElementReference", "name": "panel-16" }, "height": 10, "width": 24, "x": 0, "y": 0 } } + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-16" + }, + "height": 10, + "width": 24, + "x": 0, + "y": 0 + } + } ] } }, @@ -2171,10 +5439,25 @@ "links": [], "liveNow": false, "preload": false, - "tags": ["airi", "observability", "grafana-cloud"], + "tags": [ + "airi", + "observability", + "grafana-cloud" + ], "timeSettings": { "autoRefresh": "", - "autoRefreshIntervals": ["5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d"], + "autoRefreshIntervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], "fiscalYearStartMonth": 0, "from": "now-1h", "hideTimepicker": false, @@ -2187,21 +5470,28 @@ "kind": "QueryVariable", "spec": { "allowCustomValue": true, - "current": { "text": "All", "value": "$__all" }, - "definition": "label_values(http_server_request_duration_seconds_count, deployment_environment)", + "current": { + "text": "All", + "value": "$__all" + }, + "definition": "label_values(target_info, deployment_environment)", "hide": "dontHide", "includeAll": true, "multi": false, "name": "env", "options": [], "query": { - "datasource": { "name": "grafanacloud-projairi-prom" }, + "datasource": { + "name": "grafanacloud-projairi-prom" + }, "group": "prometheus", "kind": "DataQuery", - "spec": { "__legacyStringValue": "label_values(http_server_request_duration_seconds_count, deployment_environment)" }, + "spec": { + "__legacyStringValue": "label_values(target_info, deployment_environment)" + }, "version": "v0" }, - "refresh": "onTimeRangeChanged", + "refresh": "onDashboardLoad", "regex": "", "regexApplyTo": "value", "skipUrlSync": false, @@ -2212,21 +5502,32 @@ "kind": "QueryVariable", "spec": { "allowCustomValue": true, - "current": { "text": ["server"], "value": ["server"] }, - "definition": "label_values(http_server_request_duration_seconds_count{deployment_environment=~\"$env\"}, service_name)", + "current": { + "text": [ + "server" + ], + "value": [ + "server" + ] + }, + "definition": "label_values(target_info{deployment_environment=~\"$env\"}, service_name)", "hide": "dontHide", "includeAll": true, "multi": true, "name": "service", "options": [], "query": { - "datasource": { "name": "grafanacloud-projairi-prom" }, + "datasource": { + "name": "grafanacloud-projairi-prom" + }, "group": "prometheus", "kind": "DataQuery", - "spec": { "__legacyStringValue": "label_values(http_server_request_duration_seconds_count{deployment_environment=~\"$env\"}, service_name)" }, + "spec": { + "__legacyStringValue": "label_values(target_info{deployment_environment=~\"$env\"}, service_name)" + }, "version": "v0" }, - "refresh": "onTimeRangeChanged", + "refresh": "onDashboardLoad", "regex": "", "regexApplyTo": "value", "skipUrlSync": false, diff --git a/apps/server/src/libs/otel.ts b/apps/server/src/libs/otel.ts index d386ca56e..eeff9973d 100644 --- a/apps/server/src/libs/otel.ts +++ b/apps/server/src/libs/otel.ts @@ -1,4 +1,4 @@ -import type { Counter, Histogram, UpDownCounter } from '@opentelemetry/api' +import type { Counter, Histogram, ObservableGauge, UpDownCounter } from '@opentelemetry/api' import type { Env } from './env' @@ -77,7 +77,24 @@ export interface EngagementMetrics { characterCreated: Counter characterDeleted: Counter characterEngagement: Counter - wsConnectionsActive: UpDownCounter + /** + * Pull-based gauge for active WebSocket connections. + * + * Use when: + * - Querying current concurrent WS connections in Grafana / alerts. + * + * Why ObservableGauge instead of UpDownCounter: + * - UpDownCounter is delta-based (+1 / -1) and drifts when disconnect + * handlers miss (process crash, SIGKILL, TCP RST, network blackhole). + * - ObservableGauge runs a callback at every export interval and reports + * the live registry size, so a missed -1 self-corrects on the next + * scrape instead of leaking forever. + * + * Expects: + * - Caller (`createChatWsHandlers`) registers exactly one callback via + * `addCallback`. Multiple callbacks would double-count. + */ + wsConnectionsActive: ObservableGauge wsMessagesSent: Counter wsMessagesReceived: Counter } @@ -258,8 +275,15 @@ export function initOtel(env: Env): OtelInstance | undefined { characterEngagement: meter.createCounter(METRIC_CHARACTER_ENGAGEMENT, { description: 'Number of character engagement actions (like/bookmark)', }), - wsConnectionsActive: meter.createUpDownCounter(METRIC_WS_CONNECTIONS_ACTIVE, { - description: 'Active WebSocket connections', + // NOTICE: + // ObservableGauge — caller (chat-ws factory) registers a callback that + // reads the live connection registry on each export interval. UpDownCounter + // was previously used but drifted: missed `-1` on process crash / SIGKILL / + // TCP RST left the counter stuck high until Prom staleness expired the + // dead instance's series (~5 min). The pull-based gauge self-corrects on + // the next scrape because there is no delta state to leak. + wsConnectionsActive: meter.createObservableGauge(METRIC_WS_CONNECTIONS_ACTIVE, { + description: 'Active WebSocket connections (live registry size, scraped per export interval)', }), wsMessagesSent: meter.createCounter(METRIC_WS_MESSAGES_SENT, { description: 'Messages sent via WebSocket', diff --git a/apps/server/src/routes/chat-ws/index.ts b/apps/server/src/routes/chat-ws/index.ts index 737800a64..b822215e5 100644 --- a/apps/server/src/routes/chat-ws/index.ts +++ b/apps/server/src/routes/chat-ws/index.ts @@ -60,6 +60,17 @@ export function createChatWsHandlers( // Dedicated subscriber connection (ioredis requires a separate connection for subscribe mode) const sub = redis.duplicate() + // Pull-based active-connection gauge: walk the local registry on each + // export interval and report the actual live count. Registered exactly + // once per process here (factory runs once via injeca); duplicate + // registration would double-count. + metrics?.wsConnectionsActive.addCallback((result) => { + let total = 0 + for (const conns of userConnections.values()) + total += conns.size + result.observe(total) + }) + sub.on('message', (_channel: string, message: string) => { try { const data = parseChatBroadcastMessage(message) @@ -113,13 +124,11 @@ export function createChatWsHandlers( addConnection(userId, ctx) ensureSubscribed(userId) log.withFields({ userId }).log('WS connected') - metrics?.wsConnectionsActive.add(1) ctx.on(wsDisconnectedEvent, () => { removeConnection(userId, ctx) maybeUnsubscribe(userId) log.withFields({ userId }).log('WS disconnected') - metrics?.wsConnectionsActive.add(-1) }) // RPC: send messages