feat(analytics): add official provider ops metrics
Build on existing pricing funnel events with official provider, TTS usage, second-turn activation, and Flux balance bucket metrics.
This commit is contained in:
Lovehsigure_520
@@ -38,6 +38,9 @@ Live dashboard created on 2026-06-30:
|
||||
- Current cards:
|
||||
- Text card: `AIRI product analytics runbook`
|
||||
- Funnel: `Chat activation funnel`
|
||||
- Trend: `Official provider usage`
|
||||
- Funnel: `Official TTS activation`
|
||||
- Trend: `Paywall exposure`
|
||||
- Trend: `Provider config failures`
|
||||
- Trend: `TTS voice selection and preview`
|
||||
- Trend: `Top selected TTS voices`
|
||||
@@ -52,6 +55,7 @@ Steps:
|
||||
|
||||
1. `chat_activation_started`
|
||||
2. `chat_activation_succeeded`
|
||||
3. `second_turn_started`
|
||||
|
||||
Breakdowns:
|
||||
|
||||
@@ -67,6 +71,26 @@ Watch for:
|
||||
|
||||
- Official provider conversion lower than custom provider conversion.
|
||||
- Large drop after `chat_activation_started`.
|
||||
- First-turn success but weak `second_turn_started` conversion.
|
||||
|
||||
### Insight 1b: Official Provider Selection
|
||||
|
||||
Type: Trends
|
||||
|
||||
Events:
|
||||
|
||||
- `official_provider_selected`
|
||||
|
||||
Breakdowns:
|
||||
|
||||
- `provider_id`
|
||||
- `source`
|
||||
- `auto_selected`
|
||||
|
||||
Watch for:
|
||||
|
||||
- Official provider auto-selection is present but users do not reach `second_turn_started`.
|
||||
- A single provider id dominates errors or activation drop-off.
|
||||
|
||||
### Insight 2: Chat Activation Failures
|
||||
|
||||
@@ -144,6 +168,10 @@ Events:
|
||||
- `voice_selected`
|
||||
- `voice_preview_played`
|
||||
- `voice_pack_bound`
|
||||
- `official_tts_exposed`
|
||||
- `official_tts_preview_started`
|
||||
- `official_tts_preview_succeeded`
|
||||
- `official_tts_auto_enabled`
|
||||
|
||||
Breakdowns:
|
||||
|
||||
@@ -160,6 +188,11 @@ Use PostHog or SQL when grouping by:
|
||||
- `voice_id`
|
||||
- `voice_pack_id`
|
||||
|
||||
Watch for:
|
||||
|
||||
- Users see official TTS but do not preview it.
|
||||
- Official TTS preview succeeds but chat auto TTS is not triggered later.
|
||||
|
||||
### Insight 6: Voice Input Friction
|
||||
|
||||
Type: Funnel
|
||||
@@ -216,6 +249,8 @@ The `Product Analytics` row includes:
|
||||
- `Product Failure %`
|
||||
- `TTS Success %`
|
||||
- `TTS Failed / Blocked (range)`
|
||||
- `TTS Blocked by Reason`
|
||||
- `TTS Blocked by Flux Bucket`
|
||||
- `Top Product Actions (range)`
|
||||
- `Product Event Rate`
|
||||
- `TTS Event Rate by Source`
|
||||
@@ -230,6 +265,8 @@ Live import status:
|
||||
- `Product Failure %`
|
||||
- `TTS Success %`
|
||||
- `TTS Failed / Blocked (range)`
|
||||
- `TTS Blocked by Reason`
|
||||
- `TTS Blocked by Flux Bucket`
|
||||
- `Top Product Actions (range)`
|
||||
- `Product Event Rate`
|
||||
- `TTS Event Rate by Source`
|
||||
@@ -242,6 +279,7 @@ Permission notes from the import retry:
|
||||
- API confirmation returned `403 Access denied`: `You'll need additional permissions to perform this action. Permissions needed: any of dashboards:create, dashboards:write`.
|
||||
- The logged-in Grafana user `1260907335@qq.com` has org role `Viewer`; API metadata for `/d/rbr55dn/airi-server-overview` reports `canSave=false`, `canEdit=false`, `canAdmin=false`.
|
||||
- After permissions were updated, the generated dashboard was imported from Microsoft Edge. Grafana assigned the imported dashboard UID `ad8qbp5` instead of overwriting the earlier `rbr55dn` dashboard, so the imported dashboard was renamed to `AIRI Server Overview - Product Analytics` to avoid ambiguity.
|
||||
- On 2026-07-01, the live `ad8qbp5` dashboard was updated to include `TTS Blocked by Reason` and `TTS Blocked by Flux Bucket`. The live dashboard uses panel id `105` for the Flux bucket panel because id `103` was already occupied by the imported `User Engagement` row.
|
||||
|
||||
Regenerate after dashboard changes:
|
||||
|
||||
@@ -298,8 +336,11 @@ clamp_min(sum(increase(airi_product_events_total{feature!="", action!=""}[15m]))
|
||||
|
||||
## Verification Checklist
|
||||
|
||||
- PostHog can show `chat_activation_started -> chat_activation_succeeded` by `provider_mode`.
|
||||
- PostHog can show `chat_activation_started -> chat_activation_succeeded -> second_turn_started` by `provider_mode`.
|
||||
- PostHog can show `official_provider_selected` by `provider_id`, `source`, and `auto_selected`.
|
||||
- PostHog can show `voice_selected` by `voice_type` and `tts_provider_id`.
|
||||
- PostHog can show official TTS exposure / preview / auto-enabled events.
|
||||
- PostHog can show `paywall_seen` by `flux_balance_bucket`.
|
||||
- PostHog can show `feedback_submitted` and `bug_report_submitted`.
|
||||
- Grafana dashboard JSON contains `TTS Success %`, `TTS Failed / Blocked (range)`, and `TTS Event Rate by Source`.
|
||||
- Grafana product analytics panels use only bounded labels: `feature`, `action`, `status`, `source`.
|
||||
- Grafana dashboard JSON contains `TTS Success %`, `TTS Failed / Blocked (range)`, `TTS Blocked by Reason`, `TTS Blocked by Flux Bucket`, and `TTS Event Rate by Source`.
|
||||
- Grafana product analytics panels use only bounded labels: `feature`, `action`, `status`, `source`, `reason`, `flux_balance_bucket`.
|
||||
|
||||
@@ -147,6 +147,8 @@ Grafana 当前不能回答：
|
||||
| `chat_activation_started` | frontend | PostHog | 用户进入首次聊天路径或点击发送第一条消息前 |
|
||||
| `chat_activation_succeeded` | frontend | PostHog | 首次消息完成并看到 assistant response |
|
||||
| `chat_activation_failed` | frontend | PostHog | 首次消息未完成，包含配置、网络、鉴权、余额、模型等失败 |
|
||||
| `official_provider_selected` | frontend | PostHog | 官方 Provider 被默认落地或在设置页被手动选择，记录 provider id 与是否自动选择 |
|
||||
| `second_turn_started` | frontend | PostHog | 同一会话开始第二轮对话 |
|
||||
|
||||
字段：
|
||||
|
||||
@@ -159,6 +161,8 @@ Grafana 当前不能回答：
|
||||
| `time_to_first_message_ms` | success only | 从 app start 或 onboarding complete 到首次成功 |
|
||||
| `error_code` | failed only | 稳定错误码 |
|
||||
| `failure_stage` | failed only | `provider_config` / `model_list` / `message_send` / `llm_response` / `tts` |
|
||||
| `turn_index` | second turn only | 固定为 `2`，用于首轮成功后的二轮启动 |
|
||||
| `auto_selected` | official provider only | 官方默认 Provider 自动落地时为 `true` |
|
||||
|
||||
推荐看板：
|
||||
|
||||
@@ -216,6 +220,9 @@ Grafana 当前不能回答：
|
||||
| `voice_selected` | frontend | PostHog | 用户选择音色，包括官方默认落地时的 baseline |
|
||||
| `voice_preview_played` | frontend | PostHog | 用户试听音色 |
|
||||
| `voice_pack_bound` | frontend | PostHog | 用户把 Voice Pack 绑定到角色 |
|
||||
| `official_tts_exposed` | frontend | PostHog | 官方 TTS 设置入口或激活入口被展示 |
|
||||
| `official_tts_preview_started` / `official_tts_preview_succeeded` | frontend | PostHog | 官方 TTS 试听开始 / 成功 |
|
||||
| `official_tts_auto_enabled` | frontend | PostHog | 聊天自动语音实际触发官方 TTS |
|
||||
| `speech_requested` | server | Postgres/Grafana | REST / WS TTS 请求开始；沿用现有 `feature = tts` action |
|
||||
| `speech_succeeded` | server | Postgres/Grafana | REST / WS TTS 交付成功；沿用现有 `feature = tts` action |
|
||||
| `speech_failed` | server | Postgres/Grafana | TTS 上游、配置、路由失败；沿用现有 `feature = tts` action |
|
||||
@@ -456,6 +463,7 @@ Grafana 当前不能回答：
|
||||
| `tts_stop_clicked` | 保留用户停止行为 | 不代表音色选择；音色选择用 `voice_selected` |
|
||||
| `speech_requested` / `speech_succeeded` / `speech_failed` / `speech_blocked` | 服务端 TTS truth | 继续沿用 `feature = tts`，只补 metadata 字段 |
|
||||
| `pricing_page_viewed` / `plan_selected` / `checkout_started` | 保留付费漏斗前段 | 真正 payment completed 仍以 Stripe / Postgres 为准 |
|
||||
| `paywall_seen` | 付费漏斗入口 | 用 `flux_balance_bucket` 分层，不上报精确余额 |
|
||||
|
||||
## 数据卫生
|
||||
|
||||
@@ -859,15 +867,15 @@ Prometheus label 不放 `user_id`、`session_id`、`voice_pack_id`、自定义
|
||||
|
||||
已接入代码：
|
||||
|
||||
- Chat activation：`chat_activation_started`、`chat_activation_succeeded`、`chat_activation_failed`。
|
||||
- Chat activation：`chat_activation_started`、`chat_activation_succeeded`、`chat_activation_failed`、`second_turn_started`；官方 Provider 选择事件为 `official_provider_selected`，实际聊天使用口径看 activation events 的 `provider_mode = official`。
|
||||
- Model list：`model_list_loaded`、`model_list_failed`。
|
||||
- Provider config：`provider_config_started`、`provider_config_succeeded`、`provider_config_failed`。
|
||||
- TTS voice：`tts_provider_selected`、`voice_selected`、`voice_preview_played`、`voice_pack_bound`。
|
||||
- TTS 服务端 metadata：REST / WS TTS `product_events.metadata` 已补 `voice_id`、`voice_type`、`voice_pack_id`。
|
||||
- TTS voice：`tts_provider_selected`、`voice_selected`、`voice_preview_played`、`voice_pack_bound`、`official_tts_exposed`、`official_tts_preview_started`、`official_tts_preview_succeeded`、`official_tts_auto_enabled`。
|
||||
- TTS 服务端 metadata：REST / WS TTS `product_events.metadata` 已补 `voice_id`、`voice_type`、`voice_pack_id`、`block_reason`、`failure_reason`、`flux_balance_bucket`。
|
||||
- Voice input：`voice_input_started`、`microphone_permission_requested`、`microphone_permission_denied`、`audio_device_unavailable`、`voice_input_cancelled`。
|
||||
- STT：保留 `stt_started`、`stt_succeeded`、`stt_failed`，并将失败码收敛到稳定枚举，避免上报 raw error。
|
||||
- Feedback：`feedback_submitted` / `bug_report_submitted` 的低基数字段与 analytics API 已定义；产品内反馈提交入口与服务端收件流程拆到单独 PR。
|
||||
- Grafana Dashboard：`Product Analytics` 行已补 TTS success、TTS failed / blocked、TTS event rate by source 面板，并保留 voice drilldown 在 Postgres metadata / PostHog，不进入 Prometheus labels。
|
||||
- Grafana Dashboard：`Product Analytics` 行已补 TTS success、TTS failed / blocked、TTS event rate by source、TTS blocked by reason、TTS blocked by Flux bucket 面板，并保留 voice drilldown 在 Postgres metadata / PostHog，不进入 Prometheus labels。
|
||||
- Dashboard setup 文档：`product-analytics-dashboard-setup.md` 已补 PostHog insights、Grafana panels、PostHog / Grafana alert 配置建议。
|
||||
- 上线冒烟文档：`verifications/product-analytics-smoke.md` 已补 PostHog、Postgres、Grafana 三层验证步骤。
|
||||
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
# Verification: Product Analytics Smoke Test
|
||||
|
||||
Status: **code-level instrumentation verified; live PostHog dashboard created; Grafana dashboard imported; alert setup pending**
|
||||
Status: **code-level instrumentation verified; live PostHog dashboard updated; Grafana dashboard updated; alert setup pending**
|
||||
Owner: Community / Product Analytics
|
||||
Last updated: 2026-06-30
|
||||
Last updated: 2026-07-01
|
||||
Related:
|
||||
- [`product-analytics-instrumentation.md`](../product-analytics-instrumentation.md)
|
||||
- [`product-analytics-dashboard-setup.md`](../product-analytics-dashboard-setup.md)
|
||||
@@ -12,7 +12,7 @@ Related:
|
||||
|
||||
- **场景**：验证新增埋点能回答“用户是否能正常开始聊天”“Provider 配置卡在哪里”“哪个 TTS 音色被选择 / 实际播放”“语音输入卡在哪里”“用户是否提交反馈”。
|
||||
- **预期**：PostHog 能看到前端 journey events；Postgres `product_events` 能看到服务端 TTS metadata；Grafana 能看到低基数 server-side product health。
|
||||
- **当前状态**：代码与 dashboard JSON 已验证；线上 PostHog dashboard 已创建；线上 Grafana `AIRI Server Overview - Product Analytics` (`ad8qbp5`) 已导入完整 Product Analytics row；alert 仍需人工配置。
|
||||
- **当前状态**：代码与 dashboard JSON 已验证；线上 PostHog dashboard 已补官方 Provider / 官方 TTS / paywall 卡片；线上 Grafana `AIRI Server Overview - Product Analytics` (`ad8qbp5`) 已补 TTS blocked reason / Flux bucket 面板；alert 仍需人工配置。
|
||||
|
||||
## 已经由代码验证
|
||||
|
||||
@@ -35,12 +35,14 @@ Action:
|
||||
1. Use a fresh or test account.
|
||||
2. Start with an official provider.
|
||||
3. Send the first chat message and wait for the assistant response.
|
||||
4. Send a second message in the same session.
|
||||
|
||||
Expected PostHog events:
|
||||
|
||||
```text
|
||||
chat_activation_started
|
||||
chat_activation_succeeded
|
||||
second_turn_started
|
||||
```
|
||||
|
||||
Required properties:
|
||||
@@ -50,14 +52,42 @@ provider_mode = official
|
||||
provider_id = <official provider id>
|
||||
model_id = <selected model id>
|
||||
surface = web | mobile | electron
|
||||
turn_index = 2
|
||||
```
|
||||
|
||||
Fail if:
|
||||
|
||||
- `chat_activation_started` appears but `chat_activation_succeeded` never appears for a successful chat.
|
||||
- The second message is sent but `second_turn_started` does not appear.
|
||||
- `provider_mode` is missing or always `unknown`.
|
||||
- `surface` is missing.
|
||||
|
||||
### 1b. PostHog: official provider selection
|
||||
|
||||
Action:
|
||||
|
||||
1. Sign in with an account that has no active chat provider yet, or switch the chat provider to the official provider in settings.
|
||||
|
||||
Expected PostHog events:
|
||||
|
||||
```text
|
||||
official_provider_selected
|
||||
```
|
||||
|
||||
Required properties:
|
||||
|
||||
```text
|
||||
provider_mode = official
|
||||
provider_id = <official provider id>
|
||||
source = default_auto | settings
|
||||
auto_selected = true | false
|
||||
```
|
||||
|
||||
Fail if:
|
||||
|
||||
- Default official provider bootstrap reports `auto_selected = false`.
|
||||
- Manual settings selection reports `auto_selected = true`.
|
||||
|
||||
### 2. PostHog: provider config failure
|
||||
|
||||
Action:
|
||||
@@ -99,6 +129,9 @@ Expected PostHog events:
|
||||
|
||||
```text
|
||||
tts_provider_selected
|
||||
official_tts_exposed
|
||||
official_tts_preview_started
|
||||
official_tts_preview_succeeded
|
||||
voice_selected
|
||||
voice_preview_played
|
||||
```
|
||||
@@ -116,8 +149,35 @@ source = settings | manual_preview
|
||||
Fail if:
|
||||
|
||||
- `voice_selected` is missing, because this blocks “哪个 TTS 音色比较多”的核心问题。
|
||||
- Official TTS preview succeeds in the UI but `official_tts_preview_succeeded` is missing.
|
||||
- Official default voice is indistinguishable from custom configured voice.
|
||||
|
||||
### 3b. PostHog: official TTS auto playback
|
||||
|
||||
Action:
|
||||
|
||||
1. Enable chat auto TTS with an official TTS provider.
|
||||
2. Send a chat message and wait for an assistant response that triggers speech playback.
|
||||
|
||||
Expected PostHog events:
|
||||
|
||||
```text
|
||||
official_tts_auto_enabled
|
||||
```
|
||||
|
||||
Required properties:
|
||||
|
||||
```text
|
||||
tts_provider_id = <official provider id>
|
||||
tts_model_id = <model id>
|
||||
source = chat_auto_tts
|
||||
enabled = true
|
||||
```
|
||||
|
||||
Fail if:
|
||||
|
||||
- Chat auto TTS plays through the official provider but `official_tts_auto_enabled` is missing.
|
||||
|
||||
### 4. PostHog: voice input friction
|
||||
|
||||
Action:
|
||||
@@ -143,6 +203,32 @@ Fail if:
|
||||
- Permission denied or device unavailable is only visible as a generic `stt_failed`.
|
||||
- `error_code` contains raw browser error text.
|
||||
|
||||
### 4b. PostHog: paywall exposure
|
||||
|
||||
Action:
|
||||
|
||||
1. Open the Flux / plan purchase entry.
|
||||
2. Use an account with a known low or zero Flux balance if possible.
|
||||
|
||||
Expected PostHog events:
|
||||
|
||||
```text
|
||||
paywall_seen
|
||||
```
|
||||
|
||||
Required properties:
|
||||
|
||||
```text
|
||||
surface = settings_flux
|
||||
reason = manual_topup
|
||||
flux_balance_bucket = zero | 1_100 | 101_1000 | 1001_10000 | 10000_plus | unknown
|
||||
```
|
||||
|
||||
Fail if:
|
||||
|
||||
- The purchase entry is visible but `paywall_seen` is missing.
|
||||
- A precise balance is sent instead of the bounded `flux_balance_bucket`.
|
||||
|
||||
### 5. Postgres: server-side TTS metadata
|
||||
|
||||
Action:
|
||||
@@ -176,6 +262,8 @@ Expected:
|
||||
- `speech_requested` and `speech_succeeded` rows exist for successful TTS.
|
||||
- `voice_id` is present when the request provided a selected voice.
|
||||
- `voice_type` distinguishes official default / selected / custom / voice pack where available.
|
||||
- Blocked rows include bounded `block_reason` and `flux_balance_bucket`.
|
||||
- Failed rows include bounded `failure_reason`.
|
||||
|
||||
Fail if:
|
||||
|
||||
@@ -197,6 +285,8 @@ Product Events (range)
|
||||
Product Failure %
|
||||
TTS Success %
|
||||
TTS Failed / Blocked (range)
|
||||
TTS Blocked by Reason
|
||||
TTS Blocked by Flux Bucket
|
||||
Top Product Actions (range)
|
||||
Product Event Rate
|
||||
TTS Event Rate by Source
|
||||
@@ -211,7 +301,7 @@ sum(increase(airi_product_events_total{feature="tts"}[1h]))
|
||||
Expected:
|
||||
|
||||
- Query returns a non-zero value after TTS smoke actions.
|
||||
- Legends only use bounded labels: `feature`, `action`, `status`, `source`.
|
||||
- Legends only use bounded labels: `feature`, `action`, `status`, `source`, `reason`, `flux_balance_bucket`.
|
||||
|
||||
Fail if:
|
||||
|
||||
@@ -262,14 +352,16 @@ ORDER BY event_count DESC;
|
||||
| Item | Pass condition |
|
||||
|---|---|
|
||||
| Activation | PostHog funnel shows `chat_activation_started -> chat_activation_succeeded` by `provider_mode` |
|
||||
| Retention proxy | PostHog shows `second_turn_started` for the second message in a successful session |
|
||||
| Official provider | PostHog shows `official_provider_selected` by `provider_id` and `source` |
|
||||
| Provider config | Failed custom config emits `provider_config_failed` with bounded `error_code` |
|
||||
| TTS voice | PostHog can rank `voice_selected` by `voice_id`; Postgres can rank actual `speech_succeeded` by metadata voice |
|
||||
| TTS voice | PostHog can rank `voice_selected` by `voice_id`; official TTS exposure / preview / auto playback events appear; Postgres can rank actual `speech_succeeded` by metadata voice |
|
||||
| Voice input | Permission / device / cancel paths are distinguishable |
|
||||
| Feedback | Feedback event API exists with bounded fields; product feedback UI/server submission is split into a separate PR |
|
||||
| Grafana | Product Analytics row renders and uses only bounded Prometheus labels |
|
||||
| Grafana | Product Analytics row renders TTS reason / Flux bucket panels and uses only bounded Prometheus labels |
|
||||
|
||||
## Known Pending Work
|
||||
|
||||
- PostHog dashboard and alerts still need to be created inside the PostHog account.
|
||||
- Updated Grafana JSON still needs to be imported or deployed to the production Grafana workspace.
|
||||
- PostHog dashboard cards are created, but the official provider / official TTS / paywall cards need deployed traffic before they show meaningful data.
|
||||
- Updated Grafana panels are deployed to the production Grafana workspace, but alert rules still need to be configured.
|
||||
- Discord / QQ ingestion and daily / weekly automation scripts are intentionally excluded from this pass.
|
||||
|
||||
@@ -1562,6 +1562,172 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"panel-102": {
|
||||
"kind": "Panel",
|
||||
"spec": {
|
||||
"data": {
|
||||
"kind": "QueryGroup",
|
||||
"spec": {
|
||||
"queries": [
|
||||
{
|
||||
"kind": "PanelQuery",
|
||||
"spec": {
|
||||
"hidden": false,
|
||||
"query": {
|
||||
"datasource": {
|
||||
"name": "grafanacloud-projairi-prom"
|
||||
},
|
||||
"group": "prometheus",
|
||||
"kind": "DataQuery",
|
||||
"spec": {
|
||||
"expr": "topk(12, sum by (reason, source) (increase(airi_product_events_total{service_name=~\"$service\", deployment_environment=~\"$env\", feature!=\"\", action!=\"\", feature=\"tts\", action=\"speech_blocked\", status=\"blocked\"}[$__range])))",
|
||||
"legendFormat": "{{reason}} · {{source}}",
|
||||
"instant": true,
|
||||
"range": false
|
||||
},
|
||||
"version": "v0"
|
||||
},
|
||||
"refId": "A"
|
||||
}
|
||||
}
|
||||
],
|
||||
"queryOptions": {},
|
||||
"transformations": []
|
||||
}
|
||||
},
|
||||
"description": "Blocked TTS events over the dashboard range, grouped by bounded product reason and source. Today this mostly shows insufficient balance; new policy/provider/preflight buckets can be added without exposing user, voice, or model labels.",
|
||||
"id": 102,
|
||||
"links": [],
|
||||
"title": "TTS Blocked by Reason",
|
||||
"vizConfig": {
|
||||
"group": "bargauge",
|
||||
"kind": "VizConfig",
|
||||
"spec": {
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": 0
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "short",
|
||||
"noValue": "0"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"displayMode": "gradient",
|
||||
"maxVizHeight": 300,
|
||||
"minVizHeight": 12,
|
||||
"minVizWidth": 8,
|
||||
"namePlacement": "auto",
|
||||
"orientation": "horizontal",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"showUnfilled": true,
|
||||
"sizing": "auto",
|
||||
"valueMode": "color"
|
||||
}
|
||||
},
|
||||
"version": "13.0.0-23630096546"
|
||||
}
|
||||
}
|
||||
},
|
||||
"panel-103": {
|
||||
"kind": "Panel",
|
||||
"spec": {
|
||||
"data": {
|
||||
"kind": "QueryGroup",
|
||||
"spec": {
|
||||
"queries": [
|
||||
{
|
||||
"kind": "PanelQuery",
|
||||
"spec": {
|
||||
"hidden": false,
|
||||
"query": {
|
||||
"datasource": {
|
||||
"name": "grafanacloud-projairi-prom"
|
||||
},
|
||||
"group": "prometheus",
|
||||
"kind": "DataQuery",
|
||||
"spec": {
|
||||
"expr": "topk(8, sum by (flux_balance_bucket, source) (increase(airi_product_events_total{service_name=~\"$service\", deployment_environment=~\"$env\", feature!=\"\", action!=\"\", feature=\"tts\", action=\"speech_blocked\", status=\"blocked\", flux_balance_bucket!=\"\"}[$__range])))",
|
||||
"legendFormat": "{{flux_balance_bucket}} · {{source}}",
|
||||
"instant": true,
|
||||
"range": false
|
||||
},
|
||||
"version": "v0"
|
||||
},
|
||||
"refId": "A"
|
||||
}
|
||||
}
|
||||
],
|
||||
"queryOptions": {},
|
||||
"transformations": []
|
||||
}
|
||||
},
|
||||
"description": "Blocked TTS events over the dashboard range, grouped by coarse Flux balance bucket. This helps separate truly empty accounts from low-balance accounts without exposing exact user balances.",
|
||||
"id": 103,
|
||||
"links": [],
|
||||
"title": "TTS Blocked by Flux Bucket",
|
||||
"vizConfig": {
|
||||
"group": "bargauge",
|
||||
"kind": "VizConfig",
|
||||
"spec": {
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"color": {
|
||||
"mode": "thresholds"
|
||||
},
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{
|
||||
"color": "green",
|
||||
"value": 0
|
||||
}
|
||||
]
|
||||
},
|
||||
"unit": "short",
|
||||
"noValue": "0"
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"options": {
|
||||
"displayMode": "gradient",
|
||||
"maxVizHeight": 300,
|
||||
"minVizHeight": 12,
|
||||
"minVizWidth": 8,
|
||||
"namePlacement": "auto",
|
||||
"orientation": "horizontal",
|
||||
"reduceOptions": {
|
||||
"calcs": [
|
||||
"lastNotNull"
|
||||
],
|
||||
"fields": "",
|
||||
"values": false
|
||||
},
|
||||
"showUnfilled": true,
|
||||
"sizing": "auto",
|
||||
"valueMode": "color"
|
||||
}
|
||||
},
|
||||
"version": "13.0.0-23630096546"
|
||||
}
|
||||
}
|
||||
},
|
||||
"panel-16": {
|
||||
"kind": "Panel",
|
||||
"spec": {
|
||||
@@ -4564,6 +4730,32 @@
|
||||
"x": 12,
|
||||
"y": 9
|
||||
}
|
||||
},
|
||||
{
|
||||
"kind": "GridLayoutItem",
|
||||
"spec": {
|
||||
"element": {
|
||||
"kind": "ElementReference",
|
||||
"name": "panel-102"
|
||||
},
|
||||
"height": 5,
|
||||
"width": 12,
|
||||
"x": 0,
|
||||
"y": 13
|
||||
}
|
||||
},
|
||||
{
|
||||
"kind": "GridLayoutItem",
|
||||
"spec": {
|
||||
"element": {
|
||||
"kind": "ElementReference",
|
||||
"name": "panel-103"
|
||||
},
|
||||
"height": 5,
|
||||
"width": 12,
|
||||
"x": 12,
|
||||
"y": 13
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -64,6 +64,8 @@ describe('grafana dashboard builder', () => {
|
||||
expect(panelTitle('panel-99')).toBe('TTS Success %')
|
||||
expect(panelTitle('panel-100')).toBe('TTS Failed / Blocked (range)')
|
||||
expect(panelTitle('panel-101')).toBe('TTS Event Rate by Source')
|
||||
expect(panelTitle('panel-102')).toBe('TTS Blocked by Reason')
|
||||
expect(panelTitle('panel-103')).toBe('TTS Blocked by Flux Bucket')
|
||||
})
|
||||
|
||||
/**
|
||||
@@ -80,6 +82,8 @@ describe('grafana dashboard builder', () => {
|
||||
dashboard.elements['panel-99'],
|
||||
dashboard.elements['panel-100'],
|
||||
dashboard.elements['panel-101'],
|
||||
dashboard.elements['panel-102'],
|
||||
dashboard.elements['panel-103'],
|
||||
]).join('\n')
|
||||
|
||||
expect(productPanelExpressions).not.toContain('voice_id')
|
||||
|
||||
@@ -613,6 +613,34 @@ elements['panel-101'] = timeseriesPanel(
|
||||
{ unit: 'eps', fillOpacity: 15 },
|
||||
)
|
||||
|
||||
elements['panel-102'] = barGaugePanel(
|
||||
102,
|
||||
'TTS Blocked by Reason',
|
||||
'Blocked TTS events over the dashboard range, grouped by bounded product reason and source. Today this mostly shows insufficient balance; new policy/provider/preflight buckets can be added without exposing user, voice, or model labels.',
|
||||
[query(
|
||||
`topk(12, sum by (reason, source) (increase(airi_product_events_total{${PRODUCT_EVENT_FILTER}, feature="tts", action="speech_blocked", status="blocked"}[$__range])))`,
|
||||
'{{reason}} · {{source}}',
|
||||
'A',
|
||||
PROM,
|
||||
{ instant: true },
|
||||
)],
|
||||
{ unit: 'short', noValue: '0' },
|
||||
)
|
||||
|
||||
elements['panel-103'] = barGaugePanel(
|
||||
103,
|
||||
'TTS Blocked by Flux Bucket',
|
||||
'Blocked TTS events over the dashboard range, grouped by coarse Flux balance bucket. This helps separate truly empty accounts from low-balance accounts without exposing exact user balances.',
|
||||
[query(
|
||||
`topk(8, sum by (flux_balance_bucket, source) (increase(airi_product_events_total{${PRODUCT_EVENT_FILTER}, feature="tts", action="speech_blocked", status="blocked", flux_balance_bucket!=""}[$__range])))`,
|
||||
'{{flux_balance_bucket}} · {{source}}',
|
||||
'A',
|
||||
PROM,
|
||||
{ instant: true },
|
||||
)],
|
||||
{ unit: 'short', noValue: '0' },
|
||||
)
|
||||
|
||||
// --- Row 2: HTTP — traffic ranking, error trend, latency trend -------------
|
||||
elements['panel-16'] = barGaugePanel(
|
||||
16,
|
||||
@@ -955,6 +983,8 @@ const rows = [
|
||||
item('panel-97', 0, 5, 12, 8),
|
||||
item('panel-98', 12, 5, 12, 4),
|
||||
item('panel-101', 12, 9, 12, 4),
|
||||
item('panel-102', 0, 13, 12, 5),
|
||||
item('panel-103', 12, 13, 12, 5),
|
||||
]),
|
||||
// Row 3: HTTP — full-width error breakdown on top, then traffic ranking +
|
||||
// latency trend side by side.
|
||||
|
||||
@@ -322,7 +322,9 @@ describe('audio-speech-ws route', () => {
|
||||
reason: 'insufficient_balance',
|
||||
metadata: expect.objectContaining({
|
||||
trigger: 'auto',
|
||||
block_reason: 'insufficient_balance',
|
||||
balance_state: 'insufficient',
|
||||
flux_balance_bucket: 'zero',
|
||||
}),
|
||||
}))
|
||||
})
|
||||
|
||||
@@ -11,6 +11,7 @@ import WebSocket from 'ws'
|
||||
import { useLogger } from '@guiiai/logg'
|
||||
import { context as otelContext, SpanStatusCode, trace } from '@opentelemetry/api'
|
||||
|
||||
import { fluxBalanceBucket } from '../../services/domain/flux-balance'
|
||||
import { ApiError } from '../../utils/error'
|
||||
import { nanoid } from '../../utils/id'
|
||||
import {
|
||||
@@ -94,6 +95,7 @@ export function createSessionState(
|
||||
let closed = false
|
||||
let billed = false
|
||||
let totalInputChars = 0
|
||||
let preflightFluxBalance: number | undefined
|
||||
let modelLabel = STREAM_MODEL_LABEL_FALLBACK
|
||||
let voiceLabel: string | undefined
|
||||
/**
|
||||
@@ -141,6 +143,7 @@ export function createSessionState(
|
||||
// afford the worst-case session.
|
||||
try {
|
||||
const flux = await opts.fluxService.getFlux(userId)
|
||||
preflightFluxBalance = flux.flux
|
||||
await opts.ttsMeter.assertCanAfford(userId, STREAMING_PREFLIGHT_CHARS_ESTIMATE, flux.flux)
|
||||
}
|
||||
catch (err) {
|
||||
@@ -508,7 +511,9 @@ export function createSessionState(
|
||||
model: modelLabel,
|
||||
reason: 'insufficient_balance',
|
||||
metadata: {
|
||||
block_reason: 'insufficient_balance',
|
||||
balance_state: 'insufficient',
|
||||
flux_balance_bucket: fluxBalanceBucket(preflightFluxBalance),
|
||||
billing_units: STREAMING_PREFLIGHT_CHARS_ESTIMATE,
|
||||
close_code: code,
|
||||
duration_ms: Date.now() - startedAt,
|
||||
|
||||
@@ -975,6 +975,7 @@ describe('v1CompletionsRoutes', () => {
|
||||
action: 'speech_failed',
|
||||
reason: 'TOO_MANY_REQUESTS',
|
||||
metadata: expect.objectContaining({
|
||||
failure_reason: 'TOO_MANY_REQUESTS',
|
||||
http_status: 429,
|
||||
}),
|
||||
}),
|
||||
@@ -1012,7 +1013,9 @@ describe('v1CompletionsRoutes', () => {
|
||||
reason: 'insufficient_balance',
|
||||
metadata: expect.objectContaining({
|
||||
trigger: 'manual',
|
||||
block_reason: 'insufficient_balance',
|
||||
balance_state: 'insufficient',
|
||||
flux_balance_bucket: 'zero',
|
||||
}),
|
||||
}))
|
||||
})
|
||||
@@ -1058,7 +1061,9 @@ describe('v1CompletionsRoutes', () => {
|
||||
reason: 'insufficient_balance',
|
||||
metadata: expect.objectContaining({
|
||||
trigger: 'auto',
|
||||
block_reason: 'insufficient_balance',
|
||||
balance_state: 'insufficient',
|
||||
flux_balance_bucket: 'zero',
|
||||
}),
|
||||
}))
|
||||
})
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
export type FluxBalanceBucket = 'zero' | '1_100' | '101_1000' | '1001_10000' | '10000_plus' | 'unknown'
|
||||
|
||||
/**
|
||||
* Normalizes exact Flux balance values into analytics-safe buckets.
|
||||
*
|
||||
* Before:
|
||||
* - 0
|
||||
* - 42
|
||||
* - 1200
|
||||
*
|
||||
* After:
|
||||
* - "zero"
|
||||
* - "1_100"
|
||||
* - "1001_10000"
|
||||
*/
|
||||
export function fluxBalanceBucket(balance: number | null | undefined): FluxBalanceBucket {
|
||||
if (balance == null || Number.isNaN(balance))
|
||||
return 'unknown'
|
||||
if (balance <= 0)
|
||||
return 'zero'
|
||||
if (balance <= 100)
|
||||
return '1_100'
|
||||
if (balance <= 1000)
|
||||
return '101_1000'
|
||||
if (balance <= 10000)
|
||||
return '1001_10000'
|
||||
return '10000_plus'
|
||||
}
|
||||
@@ -18,6 +18,7 @@ import {
|
||||
AIRI_ATTR_GEN_AI_OPERATION_KIND,
|
||||
GEN_AI_ATTR_REQUEST_MODEL,
|
||||
} from '../../../utils/observability'
|
||||
import { fluxBalanceBucket } from '../flux-balance'
|
||||
|
||||
const tracer = trace.getTracer('v1-completions')
|
||||
|
||||
@@ -153,7 +154,9 @@ export function createOpenAiSpeechService(deps: OpenAiSpeechServiceDeps) {
|
||||
metadata: {
|
||||
input_chars: inputText.length,
|
||||
billing_units: billingUnits,
|
||||
block_reason: 'insufficient_balance',
|
||||
balance_state: 'insufficient',
|
||||
flux_balance_bucket: fluxBalanceBucket(flux.flux),
|
||||
trigger: analytics.trigger,
|
||||
...voiceMetadata,
|
||||
},
|
||||
@@ -230,6 +233,7 @@ export function createOpenAiSpeechService(deps: OpenAiSpeechServiceDeps) {
|
||||
metadata: {
|
||||
http_status: failure.status,
|
||||
duration_ms: Date.now() - startedAt,
|
||||
failure_reason: failure.reason,
|
||||
trigger: analytics.trigger,
|
||||
...voiceMetadata,
|
||||
},
|
||||
@@ -257,6 +261,7 @@ export function createOpenAiSpeechService(deps: OpenAiSpeechServiceDeps) {
|
||||
metadata: {
|
||||
http_status: response.status,
|
||||
duration_ms: durationMs,
|
||||
failure_reason: 'upstream_error',
|
||||
trigger: analytics.trigger,
|
||||
...voiceMetadata,
|
||||
},
|
||||
|
||||
@@ -114,6 +114,7 @@ describe('productEventService', () => {
|
||||
metadata: {
|
||||
trigger: 'auto',
|
||||
balance_state: 'insufficient',
|
||||
flux_balance_bucket: 'zero',
|
||||
},
|
||||
})
|
||||
|
||||
@@ -131,6 +132,8 @@ describe('productEventService', () => {
|
||||
action: 'speech_blocked',
|
||||
status: 'blocked',
|
||||
source: 'chat_auto_tts',
|
||||
reason: 'insufficient_balance',
|
||||
flux_balance_bucket: 'zero',
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@@ -75,6 +75,27 @@ export interface ProductEventAggregateRow {
|
||||
distinctUsers: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds bounded Prometheus labels from product event inputs.
|
||||
*/
|
||||
function metricLabels(input: ProductEventInput): Record<string, string> {
|
||||
const attrs: Record<string, string> = {
|
||||
feature: input.feature,
|
||||
action: input.action,
|
||||
status: input.status,
|
||||
}
|
||||
if (input.source)
|
||||
attrs.source = input.source
|
||||
if (input.reason)
|
||||
attrs.reason = input.reason
|
||||
|
||||
const fluxBalanceBucket = input.metadata?.flux_balance_bucket
|
||||
if (typeof fluxBalanceBucket === 'string')
|
||||
attrs.flux_balance_bucket = fluxBalanceBucket
|
||||
|
||||
return attrs
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates AIRI's first-party product analytics event writer.
|
||||
*
|
||||
@@ -109,14 +130,7 @@ export function createProductEventService(db: Database, metrics?: ProductMetrics
|
||||
createdAt: input.createdAt,
|
||||
})
|
||||
|
||||
const attrs: Record<string, string> = {
|
||||
feature: input.feature,
|
||||
action: input.action,
|
||||
status: input.status,
|
||||
}
|
||||
if (input.source)
|
||||
attrs.source = input.source
|
||||
metrics?.events.add(1, attrs)
|
||||
metrics?.events.add(1, metricLabels(input))
|
||||
}
|
||||
catch (err) {
|
||||
logger.withError(err).withFields({
|
||||
|
||||
Reference in New Issue
Block a user