diff --git a/apps/server/docs/ai-context/product-analytics-dashboard-setup.md b/apps/server/docs/ai-context/product-analytics-dashboard-setup.md index d46773787..dadc86833 100644 --- a/apps/server/docs/ai-context/product-analytics-dashboard-setup.md +++ b/apps/server/docs/ai-context/product-analytics-dashboard-setup.md @@ -38,6 +38,9 @@ Live dashboard created on 2026-06-30: - Current cards: - Text card: `AIRI product analytics runbook` - Funnel: `Chat activation funnel` + - Trend: `Official provider usage` + - Funnel: `Official TTS activation` + - Trend: `Paywall exposure` - Trend: `Provider config failures` - Trend: `TTS voice selection and preview` - Trend: `Top selected TTS voices` @@ -52,6 +55,7 @@ Steps: 1. `chat_activation_started` 2. `chat_activation_succeeded` +3. `second_turn_started` Breakdowns: @@ -67,6 +71,26 @@ Watch for: - Official provider conversion lower than custom provider conversion. - Large drop after `chat_activation_started`. +- First-turn success but weak `second_turn_started` conversion. + +### Insight 1b: Official Provider Selection + +Type: Trends + +Events: + +- `official_provider_selected` + +Breakdowns: + +- `provider_id` +- `source` +- `auto_selected` + +Watch for: + +- Official provider auto-selection is present but users do not reach `second_turn_started`. +- A single provider id dominates errors or activation drop-off. ### Insight 2: Chat Activation Failures @@ -144,6 +168,10 @@ Events: - `voice_selected` - `voice_preview_played` - `voice_pack_bound` +- `official_tts_exposed` +- `official_tts_preview_started` +- `official_tts_preview_succeeded` +- `official_tts_auto_enabled` Breakdowns: @@ -160,6 +188,11 @@ Use PostHog or SQL when grouping by: - `voice_id` - `voice_pack_id` +Watch for: + +- Users see official TTS but do not preview it. +- Official TTS preview succeeds but chat auto TTS is not triggered later. + ### Insight 6: Voice Input Friction Type: Funnel @@ -216,6 +249,8 @@ The `Product Analytics` row includes: - `Product Failure %` - `TTS Success %` - `TTS Failed / Blocked (range)` +- `TTS Blocked by Reason` +- `TTS Blocked by Flux Bucket` - `Top Product Actions (range)` - `Product Event Rate` - `TTS Event Rate by Source` @@ -230,6 +265,8 @@ Live import status: - `Product Failure %` - `TTS Success %` - `TTS Failed / Blocked (range)` + - `TTS Blocked by Reason` + - `TTS Blocked by Flux Bucket` - `Top Product Actions (range)` - `Product Event Rate` - `TTS Event Rate by Source` @@ -242,6 +279,7 @@ Permission notes from the import retry: - API confirmation returned `403 Access denied`: `You'll need additional permissions to perform this action. Permissions needed: any of dashboards:create, dashboards:write`. - The logged-in Grafana user `1260907335@qq.com` has org role `Viewer`; API metadata for `/d/rbr55dn/airi-server-overview` reports `canSave=false`, `canEdit=false`, `canAdmin=false`. - After permissions were updated, the generated dashboard was imported from Microsoft Edge. Grafana assigned the imported dashboard UID `ad8qbp5` instead of overwriting the earlier `rbr55dn` dashboard, so the imported dashboard was renamed to `AIRI Server Overview - Product Analytics` to avoid ambiguity. +- On 2026-07-01, the live `ad8qbp5` dashboard was updated to include `TTS Blocked by Reason` and `TTS Blocked by Flux Bucket`. The live dashboard uses panel id `105` for the Flux bucket panel because id `103` was already occupied by the imported `User Engagement` row. Regenerate after dashboard changes: @@ -298,8 +336,11 @@ clamp_min(sum(increase(airi_product_events_total{feature!="", action!=""}[15m])) ## Verification Checklist -- PostHog can show `chat_activation_started -> chat_activation_succeeded` by `provider_mode`. +- PostHog can show `chat_activation_started -> chat_activation_succeeded -> second_turn_started` by `provider_mode`. +- PostHog can show `official_provider_selected` by `provider_id`, `source`, and `auto_selected`. - PostHog can show `voice_selected` by `voice_type` and `tts_provider_id`. +- PostHog can show official TTS exposure / preview / auto-enabled events. +- PostHog can show `paywall_seen` by `flux_balance_bucket`. - PostHog can show `feedback_submitted` and `bug_report_submitted`. -- Grafana dashboard JSON contains `TTS Success %`, `TTS Failed / Blocked (range)`, and `TTS Event Rate by Source`. -- Grafana product analytics panels use only bounded labels: `feature`, `action`, `status`, `source`. +- Grafana dashboard JSON contains `TTS Success %`, `TTS Failed / Blocked (range)`, `TTS Blocked by Reason`, `TTS Blocked by Flux Bucket`, and `TTS Event Rate by Source`. +- Grafana product analytics panels use only bounded labels: `feature`, `action`, `status`, `source`, `reason`, `flux_balance_bucket`. diff --git a/apps/server/docs/ai-context/product-analytics-instrumentation.md b/apps/server/docs/ai-context/product-analytics-instrumentation.md index 93c736972..53555391f 100644 --- a/apps/server/docs/ai-context/product-analytics-instrumentation.md +++ b/apps/server/docs/ai-context/product-analytics-instrumentation.md @@ -147,6 +147,8 @@ Grafana 当前不能回答： | `chat_activation_started` | frontend | PostHog | 用户进入首次聊天路径或点击发送第一条消息前 | | `chat_activation_succeeded` | frontend | PostHog | 首次消息完成并看到 assistant response | | `chat_activation_failed` | frontend | PostHog | 首次消息未完成，包含配置、网络、鉴权、余额、模型等失败 | +| `official_provider_selected` | frontend | PostHog | 官方 Provider 被默认落地或在设置页被手动选择，记录 provider id 与是否自动选择 | +| `second_turn_started` | frontend | PostHog | 同一会话开始第二轮对话 | 字段： @@ -159,6 +161,8 @@ Grafana 当前不能回答： | `time_to_first_message_ms` | success only | 从 app start 或 onboarding complete 到首次成功 | | `error_code` | failed only | 稳定错误码 | | `failure_stage` | failed only | `provider_config` / `model_list` / `message_send` / `llm_response` / `tts` | +| `turn_index` | second turn only | 固定为 `2`，用于首轮成功后的二轮启动 | +| `auto_selected` | official provider only | 官方默认 Provider 自动落地时为 `true` | 推荐看板： @@ -216,6 +220,9 @@ Grafana 当前不能回答： | `voice_selected` | frontend | PostHog | 用户选择音色，包括官方默认落地时的 baseline | | `voice_preview_played` | frontend | PostHog | 用户试听音色 | | `voice_pack_bound` | frontend | PostHog | 用户把 Voice Pack 绑定到角色 | +| `official_tts_exposed` | frontend | PostHog | 官方 TTS 设置入口或激活入口被展示 | +| `official_tts_preview_started` / `official_tts_preview_succeeded` | frontend | PostHog | 官方 TTS 试听开始 / 成功 | +| `official_tts_auto_enabled` | frontend | PostHog | 聊天自动语音实际触发官方 TTS | | `speech_requested` | server | Postgres/Grafana | REST / WS TTS 请求开始；沿用现有 `feature = tts` action | | `speech_succeeded` | server | Postgres/Grafana | REST / WS TTS 交付成功；沿用现有 `feature = tts` action | | `speech_failed` | server | Postgres/Grafana | TTS 上游、配置、路由失败；沿用现有 `feature = tts` action | @@ -456,6 +463,7 @@ Grafana 当前不能回答： | `tts_stop_clicked` | 保留用户停止行为 | 不代表音色选择；音色选择用 `voice_selected` | | `speech_requested` / `speech_succeeded` / `speech_failed` / `speech_blocked` | 服务端 TTS truth | 继续沿用 `feature = tts`，只补 metadata 字段 | | `pricing_page_viewed` / `plan_selected` / `checkout_started` | 保留付费漏斗前段 | 真正 payment completed 仍以 Stripe / Postgres 为准 | +| `paywall_seen` | 付费漏斗入口 | 用 `flux_balance_bucket` 分层，不上报精确余额 | ## 数据卫生 @@ -859,15 +867,15 @@ Prometheus label 不放 `user_id`、`session_id`、`voice_pack_id`、自定义 已接入代码： -- Chat activation：`chat_activation_started`、`chat_activation_succeeded`、`chat_activation_failed`。 +- Chat activation：`chat_activation_started`、`chat_activation_succeeded`、`chat_activation_failed`、`second_turn_started`；官方 Provider 选择事件为 `official_provider_selected`，实际聊天使用口径看 activation events 的 `provider_mode = official`。 - Model list：`model_list_loaded`、`model_list_failed`。 - Provider config：`provider_config_started`、`provider_config_succeeded`、`provider_config_failed`。 -- TTS voice：`tts_provider_selected`、`voice_selected`、`voice_preview_played`、`voice_pack_bound`。 -- TTS 服务端 metadata：REST / WS TTS `product_events.metadata` 已补 `voice_id`、`voice_type`、`voice_pack_id`。 +- TTS voice：`tts_provider_selected`、`voice_selected`、`voice_preview_played`、`voice_pack_bound`、`official_tts_exposed`、`official_tts_preview_started`、`official_tts_preview_succeeded`、`official_tts_auto_enabled`。 +- TTS 服务端 metadata：REST / WS TTS `product_events.metadata` 已补 `voice_id`、`voice_type`、`voice_pack_id`、`block_reason`、`failure_reason`、`flux_balance_bucket`。 - Voice input：`voice_input_started`、`microphone_permission_requested`、`microphone_permission_denied`、`audio_device_unavailable`、`voice_input_cancelled`。 - STT：保留 `stt_started`、`stt_succeeded`、`stt_failed`，并将失败码收敛到稳定枚举，避免上报 raw error。 - Feedback：`feedback_submitted` / `bug_report_submitted` 的低基数字段与 analytics API 已定义；产品内反馈提交入口与服务端收件流程拆到单独 PR。 -- Grafana Dashboard：`Product Analytics` 行已补 TTS success、TTS failed / blocked、TTS event rate by source 面板，并保留 voice drilldown 在 Postgres metadata / PostHog，不进入 Prometheus labels。 +- Grafana Dashboard：`Product Analytics` 行已补 TTS success、TTS failed / blocked、TTS event rate by source、TTS blocked by reason、TTS blocked by Flux bucket 面板，并保留 voice drilldown 在 Postgres metadata / PostHog，不进入 Prometheus labels。 - Dashboard setup 文档：`product-analytics-dashboard-setup.md` 已补 PostHog insights、Grafana panels、PostHog / Grafana alert 配置建议。 - 上线冒烟文档：`verifications/product-analytics-smoke.md` 已补 PostHog、Postgres、Grafana 三层验证步骤。 diff --git a/apps/server/docs/ai-context/verifications/product-analytics-smoke.md b/apps/server/docs/ai-context/verifications/product-analytics-smoke.md index 57c99d632..35948a7b3 100644 --- a/apps/server/docs/ai-context/verifications/product-analytics-smoke.md +++ b/apps/server/docs/ai-context/verifications/product-analytics-smoke.md @@ -1,8 +1,8 @@ # Verification: Product Analytics Smoke Test -Status: **code-level instrumentation verified; live PostHog dashboard created; Grafana dashboard imported; alert setup pending** +Status: **code-level instrumentation verified; live PostHog dashboard updated; Grafana dashboard updated; alert setup pending** Owner: Community / Product Analytics -Last updated: 2026-06-30 +Last updated: 2026-07-01 Related: - [`product-analytics-instrumentation.md`](../product-analytics-instrumentation.md) - [`product-analytics-dashboard-setup.md`](../product-analytics-dashboard-setup.md) @@ -12,7 +12,7 @@ Related: - **场景**：验证新增埋点能回答“用户是否能正常开始聊天”“Provider 配置卡在哪里”“哪个 TTS 音色被选择 / 实际播放”“语音输入卡在哪里”“用户是否提交反馈”。 - **预期**：PostHog 能看到前端 journey events；Postgres `product_events` 能看到服务端 TTS metadata；Grafana 能看到低基数 server-side product health。 -- **当前状态**：代码与 dashboard JSON 已验证；线上 PostHog dashboard 已创建；线上 Grafana `AIRI Server Overview - Product Analytics` (`ad8qbp5`) 已导入完整 Product Analytics row；alert 仍需人工配置。 +- **当前状态**：代码与 dashboard JSON 已验证；线上 PostHog dashboard 已补官方 Provider / 官方 TTS / paywall 卡片；线上 Grafana `AIRI Server Overview - Product Analytics` (`ad8qbp5`) 已补 TTS blocked reason / Flux bucket 面板；alert 仍需人工配置。 ## 已经由代码验证 @@ -35,12 +35,14 @@ Action: 1. Use a fresh or test account. 2. Start with an official provider. 3. Send the first chat message and wait for the assistant response. +4. Send a second message in the same session. Expected PostHog events: ```text chat_activation_started chat_activation_succeeded +second_turn_started ``` Required properties: @@ -50,14 +52,42 @@ provider_mode = official provider_id = model_id = surface = web | mobile | electron +turn_index = 2 ``` Fail if: - `chat_activation_started` appears but `chat_activation_succeeded` never appears for a successful chat. +- The second message is sent but `second_turn_started` does not appear. - `provider_mode` is missing or always `unknown`. - `surface` is missing. +### 1b. PostHog: official provider selection + +Action: + +1. Sign in with an account that has no active chat provider yet, or switch the chat provider to the official provider in settings. + +Expected PostHog events: + +```text +official_provider_selected +``` + +Required properties: + +```text +provider_mode = official +provider_id = +source = default_auto | settings +auto_selected = true | false +``` + +Fail if: + +- Default official provider bootstrap reports `auto_selected = false`. +- Manual settings selection reports `auto_selected = true`. + ### 2. PostHog: provider config failure Action: @@ -99,6 +129,9 @@ Expected PostHog events: ```text tts_provider_selected +official_tts_exposed +official_tts_preview_started +official_tts_preview_succeeded voice_selected voice_preview_played ``` @@ -116,8 +149,35 @@ source = settings | manual_preview Fail if: - `voice_selected` is missing, because this blocks “哪个 TTS 音色比较多”的核心问题。 +- Official TTS preview succeeds in the UI but `official_tts_preview_succeeded` is missing. - Official default voice is indistinguishable from custom configured voice. +### 3b. PostHog: official TTS auto playback + +Action: + +1. Enable chat auto TTS with an official TTS provider. +2. Send a chat message and wait for an assistant response that triggers speech playback. + +Expected PostHog events: + +```text +official_tts_auto_enabled +``` + +Required properties: + +```text +tts_provider_id = +tts_model_id = +source = chat_auto_tts +enabled = true +``` + +Fail if: + +- Chat auto TTS plays through the official provider but `official_tts_auto_enabled` is missing. + ### 4. PostHog: voice input friction Action: @@ -143,6 +203,32 @@ Fail if: - Permission denied or device unavailable is only visible as a generic `stt_failed`. - `error_code` contains raw browser error text. +### 4b. PostHog: paywall exposure + +Action: + +1. Open the Flux / plan purchase entry. +2. Use an account with a known low or zero Flux balance if possible. + +Expected PostHog events: + +```text +paywall_seen +``` + +Required properties: + +```text +surface = settings_flux +reason = manual_topup +flux_balance_bucket = zero | 1_100 | 101_1000 | 1001_10000 | 10000_plus | unknown +``` + +Fail if: + +- The purchase entry is visible but `paywall_seen` is missing. +- A precise balance is sent instead of the bounded `flux_balance_bucket`. + ### 5. Postgres: server-side TTS metadata Action: @@ -176,6 +262,8 @@ Expected: - `speech_requested` and `speech_succeeded` rows exist for successful TTS. - `voice_id` is present when the request provided a selected voice. - `voice_type` distinguishes official default / selected / custom / voice pack where available. +- Blocked rows include bounded `block_reason` and `flux_balance_bucket`. +- Failed rows include bounded `failure_reason`. Fail if: @@ -197,6 +285,8 @@ Product Events (range) Product Failure % TTS Success % TTS Failed / Blocked (range) +TTS Blocked by Reason +TTS Blocked by Flux Bucket Top Product Actions (range) Product Event Rate TTS Event Rate by Source @@ -211,7 +301,7 @@ sum(increase(airi_product_events_total{feature="tts"}[1h])) Expected: - Query returns a non-zero value after TTS smoke actions. -- Legends only use bounded labels: `feature`, `action`, `status`, `source`. +- Legends only use bounded labels: `feature`, `action`, `status`, `source`, `reason`, `flux_balance_bucket`. Fail if: @@ -262,14 +352,16 @@ ORDER BY event_count DESC; | Item | Pass condition | |---|---| | Activation | PostHog funnel shows `chat_activation_started -> chat_activation_succeeded` by `provider_mode` | +| Retention proxy | PostHog shows `second_turn_started` for the second message in a successful session | +| Official provider | PostHog shows `official_provider_selected` by `provider_id` and `source` | | Provider config | Failed custom config emits `provider_config_failed` with bounded `error_code` | -| TTS voice | PostHog can rank `voice_selected` by `voice_id`; Postgres can rank actual `speech_succeeded` by metadata voice | +| TTS voice | PostHog can rank `voice_selected` by `voice_id`; official TTS exposure / preview / auto playback events appear; Postgres can rank actual `speech_succeeded` by metadata voice | | Voice input | Permission / device / cancel paths are distinguishable | | Feedback | Feedback event API exists with bounded fields; product feedback UI/server submission is split into a separate PR | -| Grafana | Product Analytics row renders and uses only bounded Prometheus labels | +| Grafana | Product Analytics row renders TTS reason / Flux bucket panels and uses only bounded Prometheus labels | ## Known Pending Work -- PostHog dashboard and alerts still need to be created inside the PostHog account. -- Updated Grafana JSON still needs to be imported or deployed to the production Grafana workspace. +- PostHog dashboard cards are created, but the official provider / official TTS / paywall cards need deployed traffic before they show meaningful data. +- Updated Grafana panels are deployed to the production Grafana workspace, but alert rules still need to be configured. - Discord / QQ ingestion and daily / weekly automation scripts are intentionally excluded from this pass. diff --git a/apps/server/otel/grafana/dashboards/airi-server-overview-cloud.json b/apps/server/otel/grafana/dashboards/airi-server-overview-cloud.json index 0806a459f..9d1aaf0b4 100644 --- a/apps/server/otel/grafana/dashboards/airi-server-overview-cloud.json +++ b/apps/server/otel/grafana/dashboards/airi-server-overview-cloud.json @@ -1562,6 +1562,172 @@ } } }, + "panel-102": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "grafanacloud-projairi-prom" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "topk(12, sum by (reason, source) (increase(airi_product_events_total{service_name=~\"$service\", deployment_environment=~\"$env\", feature!=\"\", action!=\"\", feature=\"tts\", action=\"speech_blocked\", status=\"blocked\"}[$__range])))", + "legendFormat": "{{reason}} · {{source}}", + "instant": true, + "range": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Blocked TTS events over the dashboard range, grouped by bounded product reason and source. Today this mostly shows insufficient balance; new policy/provider/preflight buckets can be added without exposing user, voice, or model labels.", + "id": 102, + "links": [], + "title": "TTS Blocked by Reason", + "vizConfig": { + "group": "bargauge", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "short", + "noValue": "0" + }, + "overrides": [] + }, + "options": { + "displayMode": "gradient", + "maxVizHeight": 300, + "minVizHeight": 12, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + } + }, + "version": "13.0.0-23630096546" + } + } + }, + "panel-103": { + "kind": "Panel", + "spec": { + "data": { + "kind": "QueryGroup", + "spec": { + "queries": [ + { + "kind": "PanelQuery", + "spec": { + "hidden": false, + "query": { + "datasource": { + "name": "grafanacloud-projairi-prom" + }, + "group": "prometheus", + "kind": "DataQuery", + "spec": { + "expr": "topk(8, sum by (flux_balance_bucket, source) (increase(airi_product_events_total{service_name=~\"$service\", deployment_environment=~\"$env\", feature!=\"\", action!=\"\", feature=\"tts\", action=\"speech_blocked\", status=\"blocked\", flux_balance_bucket!=\"\"}[$__range])))", + "legendFormat": "{{flux_balance_bucket}} · {{source}}", + "instant": true, + "range": false + }, + "version": "v0" + }, + "refId": "A" + } + } + ], + "queryOptions": {}, + "transformations": [] + } + }, + "description": "Blocked TTS events over the dashboard range, grouped by coarse Flux balance bucket. This helps separate truly empty accounts from low-balance accounts without exposing exact user balances.", + "id": 103, + "links": [], + "title": "TTS Blocked by Flux Bucket", + "vizConfig": { + "group": "bargauge", + "kind": "VizConfig", + "spec": { + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "short", + "noValue": "0" + }, + "overrides": [] + }, + "options": { + "displayMode": "gradient", + "maxVizHeight": 300, + "minVizHeight": 12, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + } + }, + "version": "13.0.0-23630096546" + } + } + }, "panel-16": { "kind": "Panel", "spec": { @@ -4564,6 +4730,32 @@ "x": 12, "y": 9 } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-102" + }, + "height": 5, + "width": 12, + "x": 0, + "y": 13 + } + }, + { + "kind": "GridLayoutItem", + "spec": { + "element": { + "kind": "ElementReference", + "name": "panel-103" + }, + "height": 5, + "width": 12, + "x": 12, + "y": 13 + } } ] } diff --git a/apps/server/otel/grafana/dashboards/build.test.ts b/apps/server/otel/grafana/dashboards/build.test.ts index 4073c0cfb..0b82e47ee 100644 --- a/apps/server/otel/grafana/dashboards/build.test.ts +++ b/apps/server/otel/grafana/dashboards/build.test.ts @@ -64,6 +64,8 @@ describe('grafana dashboard builder', () => { expect(panelTitle('panel-99')).toBe('TTS Success %') expect(panelTitle('panel-100')).toBe('TTS Failed / Blocked (range)') expect(panelTitle('panel-101')).toBe('TTS Event Rate by Source') + expect(panelTitle('panel-102')).toBe('TTS Blocked by Reason') + expect(panelTitle('panel-103')).toBe('TTS Blocked by Flux Bucket') }) /** @@ -80,6 +82,8 @@ describe('grafana dashboard builder', () => { dashboard.elements['panel-99'], dashboard.elements['panel-100'], dashboard.elements['panel-101'], + dashboard.elements['panel-102'], + dashboard.elements['panel-103'], ]).join('\n') expect(productPanelExpressions).not.toContain('voice_id') diff --git a/apps/server/otel/grafana/dashboards/build.ts b/apps/server/otel/grafana/dashboards/build.ts index d39252e91..0b7fc543c 100644 --- a/apps/server/otel/grafana/dashboards/build.ts +++ b/apps/server/otel/grafana/dashboards/build.ts @@ -613,6 +613,34 @@ elements['panel-101'] = timeseriesPanel( { unit: 'eps', fillOpacity: 15 }, ) +elements['panel-102'] = barGaugePanel( + 102, + 'TTS Blocked by Reason', + 'Blocked TTS events over the dashboard range, grouped by bounded product reason and source. Today this mostly shows insufficient balance; new policy/provider/preflight buckets can be added without exposing user, voice, or model labels.', + [query( + `topk(12, sum by (reason, source) (increase(airi_product_events_total{${PRODUCT_EVENT_FILTER}, feature="tts", action="speech_blocked", status="blocked"}[$__range])))`, + '{{reason}} · {{source}}', + 'A', + PROM, + { instant: true }, + )], + { unit: 'short', noValue: '0' }, +) + +elements['panel-103'] = barGaugePanel( + 103, + 'TTS Blocked by Flux Bucket', + 'Blocked TTS events over the dashboard range, grouped by coarse Flux balance bucket. This helps separate truly empty accounts from low-balance accounts without exposing exact user balances.', + [query( + `topk(8, sum by (flux_balance_bucket, source) (increase(airi_product_events_total{${PRODUCT_EVENT_FILTER}, feature="tts", action="speech_blocked", status="blocked", flux_balance_bucket!=""}[$__range])))`, + '{{flux_balance_bucket}} · {{source}}', + 'A', + PROM, + { instant: true }, + )], + { unit: 'short', noValue: '0' }, +) + // --- Row 2: HTTP — traffic ranking, error trend, latency trend ------------- elements['panel-16'] = barGaugePanel( 16, @@ -955,6 +983,8 @@ const rows = [ item('panel-97', 0, 5, 12, 8), item('panel-98', 12, 5, 12, 4), item('panel-101', 12, 9, 12, 4), + item('panel-102', 0, 13, 12, 5), + item('panel-103', 12, 13, 12, 5), ]), // Row 3: HTTP — full-width error breakdown on top, then traffic ranking + // latency trend side by side. diff --git a/apps/server/src/routes/audio-speech-ws/route.test.ts b/apps/server/src/routes/audio-speech-ws/route.test.ts index 855467c26..9345e9f2b 100644 --- a/apps/server/src/routes/audio-speech-ws/route.test.ts +++ b/apps/server/src/routes/audio-speech-ws/route.test.ts @@ -322,7 +322,9 @@ describe('audio-speech-ws route', () => { reason: 'insufficient_balance', metadata: expect.objectContaining({ trigger: 'auto', + block_reason: 'insufficient_balance', balance_state: 'insufficient', + flux_balance_bucket: 'zero', }), })) }) diff --git a/apps/server/src/routes/audio-speech-ws/session.ts b/apps/server/src/routes/audio-speech-ws/session.ts index 83f45b43d..f62ed2a17 100644 --- a/apps/server/src/routes/audio-speech-ws/session.ts +++ b/apps/server/src/routes/audio-speech-ws/session.ts @@ -11,6 +11,7 @@ import WebSocket from 'ws' import { useLogger } from '@guiiai/logg' import { context as otelContext, SpanStatusCode, trace } from '@opentelemetry/api' +import { fluxBalanceBucket } from '../../services/domain/flux-balance' import { ApiError } from '../../utils/error' import { nanoid } from '../../utils/id' import { @@ -94,6 +95,7 @@ export function createSessionState( let closed = false let billed = false let totalInputChars = 0 + let preflightFluxBalance: number | undefined let modelLabel = STREAM_MODEL_LABEL_FALLBACK let voiceLabel: string | undefined /** @@ -141,6 +143,7 @@ export function createSessionState( // afford the worst-case session. try { const flux = await opts.fluxService.getFlux(userId) + preflightFluxBalance = flux.flux await opts.ttsMeter.assertCanAfford(userId, STREAMING_PREFLIGHT_CHARS_ESTIMATE, flux.flux) } catch (err) { @@ -508,7 +511,9 @@ export function createSessionState( model: modelLabel, reason: 'insufficient_balance', metadata: { + block_reason: 'insufficient_balance', balance_state: 'insufficient', + flux_balance_bucket: fluxBalanceBucket(preflightFluxBalance), billing_units: STREAMING_PREFLIGHT_CHARS_ESTIMATE, close_code: code, duration_ms: Date.now() - startedAt, diff --git a/apps/server/src/routes/openai/v1/route.test.ts b/apps/server/src/routes/openai/v1/route.test.ts index e6bf7cd79..cca0bb2ef 100644 --- a/apps/server/src/routes/openai/v1/route.test.ts +++ b/apps/server/src/routes/openai/v1/route.test.ts @@ -975,6 +975,7 @@ describe('v1CompletionsRoutes', () => { action: 'speech_failed', reason: 'TOO_MANY_REQUESTS', metadata: expect.objectContaining({ + failure_reason: 'TOO_MANY_REQUESTS', http_status: 429, }), }), @@ -1012,7 +1013,9 @@ describe('v1CompletionsRoutes', () => { reason: 'insufficient_balance', metadata: expect.objectContaining({ trigger: 'manual', + block_reason: 'insufficient_balance', balance_state: 'insufficient', + flux_balance_bucket: 'zero', }), })) }) @@ -1058,7 +1061,9 @@ describe('v1CompletionsRoutes', () => { reason: 'insufficient_balance', metadata: expect.objectContaining({ trigger: 'auto', + block_reason: 'insufficient_balance', balance_state: 'insufficient', + flux_balance_bucket: 'zero', }), })) }) diff --git a/apps/server/src/services/domain/flux-balance.ts b/apps/server/src/services/domain/flux-balance.ts new file mode 100644 index 000000000..f2f5033f4 --- /dev/null +++ b/apps/server/src/services/domain/flux-balance.ts @@ -0,0 +1,28 @@ +export type FluxBalanceBucket = 'zero' | '1_100' | '101_1000' | '1001_10000' | '10000_plus' | 'unknown' + +/** + * Normalizes exact Flux balance values into analytics-safe buckets. + * + * Before: + * - 0 + * - 42 + * - 1200 + * + * After: + * - "zero" + * - "1_100" + * - "1001_10000" + */ +export function fluxBalanceBucket(balance: number | null | undefined): FluxBalanceBucket { + if (balance == null || Number.isNaN(balance)) + return 'unknown' + if (balance <= 0) + return 'zero' + if (balance <= 100) + return '1_100' + if (balance <= 1000) + return '101_1000' + if (balance <= 10000) + return '1001_10000' + return '10000_plus' +} diff --git a/apps/server/src/services/domain/openai-speech/index.ts b/apps/server/src/services/domain/openai-speech/index.ts index 0193ad41f..7403d1f9e 100644 --- a/apps/server/src/services/domain/openai-speech/index.ts +++ b/apps/server/src/services/domain/openai-speech/index.ts @@ -18,6 +18,7 @@ import { AIRI_ATTR_GEN_AI_OPERATION_KIND, GEN_AI_ATTR_REQUEST_MODEL, } from '../../../utils/observability' +import { fluxBalanceBucket } from '../flux-balance' const tracer = trace.getTracer('v1-completions') @@ -153,7 +154,9 @@ export function createOpenAiSpeechService(deps: OpenAiSpeechServiceDeps) { metadata: { input_chars: inputText.length, billing_units: billingUnits, + block_reason: 'insufficient_balance', balance_state: 'insufficient', + flux_balance_bucket: fluxBalanceBucket(flux.flux), trigger: analytics.trigger, ...voiceMetadata, }, @@ -230,6 +233,7 @@ export function createOpenAiSpeechService(deps: OpenAiSpeechServiceDeps) { metadata: { http_status: failure.status, duration_ms: Date.now() - startedAt, + failure_reason: failure.reason, trigger: analytics.trigger, ...voiceMetadata, }, @@ -257,6 +261,7 @@ export function createOpenAiSpeechService(deps: OpenAiSpeechServiceDeps) { metadata: { http_status: response.status, duration_ms: durationMs, + failure_reason: 'upstream_error', trigger: analytics.trigger, ...voiceMetadata, }, diff --git a/apps/server/src/services/domain/product-events.test.ts b/apps/server/src/services/domain/product-events.test.ts index 61914d824..73bec67c6 100644 --- a/apps/server/src/services/domain/product-events.test.ts +++ b/apps/server/src/services/domain/product-events.test.ts @@ -114,6 +114,7 @@ describe('productEventService', () => { metadata: { trigger: 'auto', balance_state: 'insufficient', + flux_balance_bucket: 'zero', }, }) @@ -131,6 +132,8 @@ describe('productEventService', () => { action: 'speech_blocked', status: 'blocked', source: 'chat_auto_tts', + reason: 'insufficient_balance', + flux_balance_bucket: 'zero', }) }) }) diff --git a/apps/server/src/services/domain/product-events.ts b/apps/server/src/services/domain/product-events.ts index 7790af3fd..a1f4cef91 100644 --- a/apps/server/src/services/domain/product-events.ts +++ b/apps/server/src/services/domain/product-events.ts @@ -75,6 +75,27 @@ export interface ProductEventAggregateRow { distinctUsers: number } +/** + * Builds bounded Prometheus labels from product event inputs. + */ +function metricLabels(input: ProductEventInput): Record { + const attrs: Record = { + feature: input.feature, + action: input.action, + status: input.status, + } + if (input.source) + attrs.source = input.source + if (input.reason) + attrs.reason = input.reason + + const fluxBalanceBucket = input.metadata?.flux_balance_bucket + if (typeof fluxBalanceBucket === 'string') + attrs.flux_balance_bucket = fluxBalanceBucket + + return attrs +} + /** * Creates AIRI's first-party product analytics event writer. * @@ -109,14 +130,7 @@ export function createProductEventService(db: Database, metrics?: ProductMetrics createdAt: input.createdAt, }) - const attrs: Record = { - feature: input.feature, - action: input.action, - status: input.status, - } - if (input.source) - attrs.source = input.source - metrics?.events.add(1, attrs) + metrics?.events.add(1, metricLabels(input)) } catch (err) { logger.withError(err).withFields({ diff --git a/packages/core-agent/src/runtime/chat-orchestrator-runtime.test.ts b/packages/core-agent/src/runtime/chat-orchestrator-runtime.test.ts index 1f2d90d9d..738013a10 100644 --- a/packages/core-agent/src/runtime/chat-orchestrator-runtime.test.ts +++ b/packages/core-agent/src/runtime/chat-orchestrator-runtime.test.ts @@ -332,6 +332,7 @@ describe('createChatOrchestratorRuntime', () => { expect(harness.telemetry.chatActivationStarted).toEqual([{ model: 'gpt-test', provider: 'mock-provider', + sessionId: 'session-1', source: 'voice', }]) expect(harness.telemetry.chatActivationSucceeded).toEqual([{ @@ -359,6 +360,7 @@ describe('createChatOrchestratorRuntime', () => { expect(harness.telemetry.chatActivationStarted).toEqual([{ model: 'gpt-test', provider: 'mock-provider', + sessionId: 'session-1', source: 'text', }]) expect(harness.telemetry.chatActivationSucceeded).toEqual([]) diff --git a/packages/core-agent/src/runtime/chat-orchestrator-runtime.ts b/packages/core-agent/src/runtime/chat-orchestrator-runtime.ts index 239106db3..221f05e31 100644 --- a/packages/core-agent/src/runtime/chat-orchestrator-runtime.ts +++ b/packages/core-agent/src/runtime/chat-orchestrator-runtime.ts @@ -192,6 +192,7 @@ export interface ChatOrchestratorRuntimeDeps { onTrackFirstMessage?: () => void /** Called when a user starts a chat activation attempt. */ onChatActivationStarted?: (event: { + sessionId: string source: 'text' | 'voice' model: string provider: string @@ -247,6 +248,10 @@ export interface ChatOrchestratorRuntimeDeps { sessionId: string message: Extract & { id: string } messageText: string + source: 'text' | 'voice' + model: string + provider: string + turnIndex: number }) => void /** Called after the assistant message has been finalized into session history. */ onAssistantMessageAppended?: (event: { @@ -426,6 +431,7 @@ export function createChatOrchestratorRuntime(deps: ChatOrchestratorRuntimeDeps) const activeProvider = deps.getActiveProvider?.() ?? '' deps.onTrackFirstMessage?.() deps.onChatActivationStarted?.({ + sessionId, source: sendSource, model: options.model, provider: activeProvider, @@ -475,6 +481,7 @@ export function createChatOrchestratorRuntime(deps: ChatOrchestratorRuntimeDeps) id: userMessageId, } deps.session.appendSessionMessage(sessionId, userMessage) + const userTurnIndex = deps.session.getSessionMessages(sessionId).filter(message => message.role === 'user').length // Cloud sync v1: only the raw text part round-trips; image attachments // and other non-text parts stay local. @@ -482,6 +489,10 @@ export function createChatOrchestratorRuntime(deps: ChatOrchestratorRuntimeDeps) sessionId, message: userMessage, messageText: sendingMessage, + source: sendSource, + model: options.model, + provider: activeProvider, + turnIndex: userTurnIndex, }) const sessionMessagesForSend = deps.session.getSessionMessages(sessionId) diff --git a/packages/stage-pages/src/pages/settings/flux.vue b/packages/stage-pages/src/pages/settings/flux.vue index 74aea2dc0..551af2b0c 100644 --- a/packages/stage-pages/src/pages/settings/flux.vue +++ b/packages/stage-pages/src/pages/settings/flux.vue @@ -1,4 +1,6 @@