diff --git a/apps/server/docs/ai-context/product-analytics-instrumentation.md b/apps/server/docs/ai-context/product-analytics-instrumentation.md index 74d095151..0dec0f760 100644 --- a/apps/server/docs/ai-context/product-analytics-instrumentation.md +++ b/apps/server/docs/ai-context/product-analytics-instrumentation.md @@ -180,6 +180,35 @@ Grafana 当前不能回答： - 新用户 activation conversion 24h 环比下降超过 15%。 - `provider_mode = official` 的 activation failure 上升，优先排查官方 Provider。 +### AI generation 用量事实 + +`$ai_generation` 是 token / usage / 成本覆盖率事实来源；不要把 Prompt、回复正文、API Key、raw endpoint 发到 PostHog。 + +| Field | Required | Notes | +|---|---|---| +| `$ai_trace_id` | yes | 优先使用真实 `conversation_id`；无客户端会话 header 时用 server request id 兜底 | +| `$ai_session_id` | yes | 与 `conversation_id` 保持一致 | +| `$ai_span_id` | yes | 与 `round_id` / generation id 保持一致 | +| `$ai_model` | yes | 原始生成模型；不要跨 Provider 合并 | +| `$ai_provider` | yes | 实际生成 Provider | +| `airi_user_id` | server yes | Better Auth user id，便于和 Pro / 付费事件关联 | +| `conversation_id` | yes | 始终存在；结合 `conversation_id_source` 判断是否真实应用会话 | +| `conversation_id_source` | yes | `client_header` / `client_runtime` / `server_request` | +| `round_id` | yes | 单轮或 request-level generation id | +| `app_surface` | when known | 只表示用户产品端：`web` / `electron` / `mobile`；不要用 `server` 兜底 | +| `capture_surface` | yes | 事件采集端：`client` / `server` | +| `usage_source` | yes | `reported` / `estimated` / `unavailable` | +| `token_usage_available` | yes | token 是否可用于聚合；日报 token 分析先过滤 `true` | +| `cost_usd_source` | yes | `reported` / `estimated` / `unavailable` | +| `cost_usd_known` | yes | `false` 不能按零成本处理，只能计入未知成本覆盖率 | + +日报里的 Pro token / cost： + +- Token 总量、P50、P90：只统计 `token_usage_available = true`。 +- AIRI USD 成本：只统计 `cost_usd_known = true`；`cost_usd_known = false` 单独报 unknown generation count / coverage。 +- 服务端 request fallback：`conversation_id_source = server_request` 只能做 request 级 usage，不能和 `message_round` 当成同一应用会话 join。 +- 模型分析按 `$ai_provider + $ai_model` 看；不要新增 `canonical_model` / `model_family` 把不同供应链揉在一起。 + ### Provider And Model Configuration 目标：定位配置复杂和失败劝退。 @@ -480,9 +509,9 @@ PostHog 线上已经能看到 `model` / `model_id` 存在自由文本风险。 - Provider、model、voice 使用稳定 ID。 - 自定义值不要直接 group-by。 +- 不跨 Provider 合并模型名：`deepseek-chat` 和 `deepseek/deepseek-chat` 可能代表不同供应链 / 成本口径，日报按原始 Provider + model 组合看。 - 自定义模型传： - `provider_id = custom` - - `model_family = custom` - `is_custom_model = true` - `custom_model_hash` 可选，必须单向 hash，不能还原原文。 - 自定义 voice 传： diff --git a/apps/server/src/routes/openai/v1/analytics.ts b/apps/server/src/routes/openai/v1/analytics.ts index 2799b1fc1..85aca5290 100644 --- a/apps/server/src/routes/openai/v1/analytics.ts +++ b/apps/server/src/routes/openai/v1/analytics.ts @@ -6,9 +6,16 @@ export const AIRI_CHAT_APP_SURFACE_HEADER = 'x-airi-app-surface' const CLIENT_CHAT_ANALYTICS_SURFACES = new Set(['web', 'mobile', 'electron']) -export function resolveChatAnalyticsSurface(value: string | undefined): AiGenerationAppSurface { +/** + * Resolves the product runtime from a trusted client hint. + * + * Unknown values are not coerced to `server`: `$ai_generation` uses + * `capture_surface` for the process that emitted the event, while + * `app_surface` stays reserved for the user's actual product runtime. + */ +export function resolveChatAnalyticsSurface(value: string | undefined): AiGenerationAppSurface | undefined { if (CLIENT_CHAT_ANALYTICS_SURFACES.has(value as AiGenerationAppSurface)) return value as AiGenerationAppSurface - return 'server' + return undefined } diff --git a/apps/server/src/routes/openai/v1/operations/chat-completions/index.ts b/apps/server/src/routes/openai/v1/operations/chat-completions/index.ts index caaf4c3ba..103da8569 100644 --- a/apps/server/src/routes/openai/v1/operations/chat-completions/index.ts +++ b/apps/server/src/routes/openai/v1/operations/chat-completions/index.ts @@ -22,7 +22,7 @@ export interface ChatCompletionsOperationRequest { body: Record sessionId?: string roundId?: string - appSurface: AiGenerationAppSurface + appSurface?: AiGenerationAppSurface abortSignal?: AbortSignal } @@ -32,7 +32,7 @@ interface GenerationCaptureInput { requestId: string sessionId?: string roundId?: string - appSurface: AiGenerationAppSurface + appSurface?: AiGenerationAppSurface generationModel: string routeCtxProvider: string usage: UsageInfo @@ -240,13 +240,14 @@ interface ChatModelAliasPlan { function captureGeneration(input: GenerationCaptureInput): void { const generationId = input.roundId ?? input.requestId + const conversationId = input.sessionId ?? input.requestId const totalTokens = input.usage.promptTokens != null && input.usage.completionTokens != null ? input.usage.promptTokens + input.usage.completionTokens : undefined input.deps.productEventService.trackGeneration({ userId: input.userId, - traceId: input.sessionId ?? input.requestId, + traceId: conversationId, generationId, model: input.generationModel, provider: input.routeCtxProvider || 'unknown', @@ -257,9 +258,12 @@ function captureGeneration(input: GenerationCaptureInput): void { inputTokens: input.usage.promptTokens, outputTokens: input.usage.completionTokens, totalTokens, - conversationId: input.sessionId, + costUsdSource: 'unavailable', + conversationId, + conversationIdSource: input.sessionId ? 'client_header' : 'server_request', roundId: generationId, - appSurface: input.appSurface, + ...(input.appSurface && { appSurface: input.appSurface }), + captureSurface: 'server', latencySeconds: input.durationMs / 1000, stream: input.stream, }) @@ -356,7 +360,7 @@ function streamChatCompletion(input: { userId: string sessionId?: string roundId?: string - appSurface: AiGenerationAppSurface + appSurface?: AiGenerationAppSurface requestModel: string generationModel: string routeCtxProvider: string @@ -579,7 +583,7 @@ async function completeNonStreamingChat(input: { userId: string sessionId?: string roundId?: string - appSurface: AiGenerationAppSurface + appSurface?: AiGenerationAppSurface requestModel: string generationModel: string routeCtxProvider: string diff --git a/apps/server/src/routes/openai/v1/route.test.ts b/apps/server/src/routes/openai/v1/route.test.ts index 56111a157..337a8e565 100644 --- a/apps/server/src/routes/openai/v1/route.test.ts +++ b/apps/server/src/routes/openai/v1/route.test.ts @@ -974,14 +974,79 @@ describe('v1CompletionsRoutes', () => { inputTokens: 1, outputTokens: 2, totalTokens: 3, + costUsdSource: 'unavailable', conversationId: 'conversation-1', + conversationIdSource: 'client_header', roundId: 'round-1', appSurface: 'electron', + captureSurface: 'server', latencySeconds: expect.any(Number), stream: false, }) }) + it('uses request-level correlation for server-captured generations without chat headers', async () => { + const llmRouter = createMockLlmRouter({ + route: vi.fn(async (_req, ctx) => { + if (ctx) { + ctx.provider = 'openrouter' + ctx.upstreamModel = 'openai/gpt-4o-mini' + } + return new Response(JSON.stringify({ + choices: [], + usage: { prompt_tokens: 1, completion_tokens: 2, total_tokens: 3 }, + }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }) + }) as any, + }) + const productEventService = createMockProductEventService() + const app = createTestApp( + createMockFluxService(), + createMockConfigKV(), + undefined, + undefined, + undefined, + llmRouter, + createMockLlmTracing(), + productEventService, + ) + + await app.fetch( + new Request('http://localhost/api/v1/openai/chat/completions', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ model: 'chat-auto', messages: [{ role: 'user', content: 'hi' }] }), + }), + { user: testUser } as any, + ) + + expect(productEventService.trackGeneration).toHaveBeenCalledWith({ + userId: 'user-1', + traceId: expect.any(String), + generationId: expect.any(String), + model: 'openai/gpt-4o-mini', + provider: 'openrouter', + providerType: 'official', + usageSource: 'reported', + inputTokens: 1, + outputTokens: 2, + totalTokens: 3, + costUsdSource: 'unavailable', + conversationId: expect.any(String), + conversationIdSource: 'server_request', + roundId: expect.any(String), + captureSurface: 'server', + latencySeconds: expect.any(Number), + stream: false, + }) + const generation = vi.mocked(productEventService.trackGeneration).mock.calls[0]?.[0] + expect(generation?.traceId).toBe(generation?.conversationId) + expect(generation?.roundId).toBe(generation?.generationId) + expect(generation).not.toHaveProperty('appSurface') + }) + it('should not charge flux when upstream returns error', async () => { globalThis.fetch = vi.fn(async () => new Response('{"error":"bad"}', { status: 500, diff --git a/apps/server/src/services/domain/product-events.test.ts b/apps/server/src/services/domain/product-events.test.ts index 875697988..0c32f38b0 100644 --- a/apps/server/src/services/domain/product-events.test.ts +++ b/apps/server/src/services/domain/product-events.test.ts @@ -271,8 +271,10 @@ describe('productEventService', () => { outputTokens: 8, totalTokens: 20, conversationId: 'session-1', + conversationIdSource: 'client_header', roundId: 'round-1', - appSurface: 'server', + appSurface: 'electron', + captureSurface: 'server', }) expect(capture).not.toHaveBeenCalled() @@ -289,11 +291,17 @@ describe('productEventService', () => { $ai_output_tokens: 8, $ai_total_tokens: 20, $insert_id: 'ai-generation:round-1', + airi_user_id: 'user-1', provider_type: 'official', usage_source: 'reported', + token_usage_available: true, + cost_usd_source: 'unavailable', + cost_usd_known: false, conversation_id: 'session-1', + conversation_id_source: 'client_header', round_id: 'round-1', - app_surface: 'server', + app_surface: 'electron', + capture_surface: 'server', }, }) diff --git a/apps/server/src/services/domain/product-events.ts b/apps/server/src/services/domain/product-events.ts index b39b40245..56a6bf0bc 100644 --- a/apps/server/src/services/domain/product-events.ts +++ b/apps/server/src/services/domain/product-events.ts @@ -76,7 +76,17 @@ export interface ProductEventAggregateRow { distinctUsers: number } -export type AiGenerationAppSurface = 'server' | 'web' | 'mobile' | 'electron' +/** Product runtime where the user initiated the AI generation. */ +export type AiGenerationAppSurface = 'web' | 'mobile' | 'electron' + +/** Runtime that captured the `$ai_generation` fact. */ +export type AiGenerationCaptureSurface = 'server' | 'client' + +/** Explains whether `conversation_id` is an app conversation or a server fallback. */ +export type AiGenerationConversationIdSource = 'client_header' | 'server_request' + +/** Explains whether AIRI supplied a trustworthy USD cost for this generation. */ +export type AiGenerationCostUsdSource = 'reported' | 'estimated' | 'unavailable' /** Content-free PostHog AI generation fact keyed to the authenticated user. */ export interface AiGenerationEventInput { @@ -90,9 +100,17 @@ export interface AiGenerationEventInput { inputTokens?: number outputTokens?: number totalTokens?: number - conversationId?: string + totalCostUsd?: number + costUsdSource?: AiGenerationCostUsdSource + /** Always present for joins; `conversationIdSource` tells whether it is request-level fallback. */ + conversationId: string + /** Distinguishes real client conversation ids from server-generated request fallbacks. */ + conversationIdSource: AiGenerationConversationIdSource roundId?: string - appSurface: AiGenerationAppSurface + /** Omitted when the server cannot determine the user's product runtime. */ + appSurface?: AiGenerationAppSurface + /** Defaults to `server` because this service runs in the API process. */ + captureSurface?: AiGenerationCaptureSurface latencySeconds?: number stream?: boolean } @@ -169,21 +187,28 @@ export function createProductEventService(db: Database, metrics?: ProductMetrics event: '$ai_generation', properties: { $ai_trace_id: input.traceId, - ...(input.conversationId && { $ai_session_id: input.conversationId }), + $ai_session_id: input.conversationId, $ai_span_id: input.generationId, $ai_model: input.model, $ai_provider: input.provider, ...(input.inputTokens != null && { $ai_input_tokens: input.inputTokens }), ...(input.outputTokens != null && { $ai_output_tokens: input.outputTokens }), ...(input.totalTokens != null && { $ai_total_tokens: input.totalTokens }), + ...(input.totalCostUsd != null && { $ai_total_cost_usd: input.totalCostUsd }), ...(input.latencySeconds != null && { $ai_latency: input.latencySeconds }), ...(input.stream != null && { $ai_stream: input.stream }), $insert_id: `ai-generation:${input.generationId}`, + airi_user_id: input.userId, provider_type: input.providerType, usage_source: input.usageSource, - ...(input.conversationId && { conversation_id: input.conversationId }), + token_usage_available: input.usageSource !== 'unavailable', + cost_usd_source: input.costUsdSource ?? 'unavailable', + cost_usd_known: input.totalCostUsd != null, + conversation_id: input.conversationId, + conversation_id_source: input.conversationIdSource, ...(input.roundId && { round_id: input.roundId }), - app_surface: input.appSurface, + ...(input.appSurface && { app_surface: input.appSurface }), + capture_surface: input.captureSurface ?? 'server', }, } diff --git a/packages/stage-ui/src/composables/use-analytics.test.ts b/packages/stage-ui/src/composables/use-analytics.test.ts index 282df1c48..01df76ec0 100644 --- a/packages/stage-ui/src/composables/use-analytics.test.ts +++ b/packages/stage-ui/src/composables/use-analytics.test.ts @@ -108,10 +108,15 @@ describe('useAnalytics conversation product events', () => { $ai_total_tokens: 20, $insert_id: 'ai-generation:round-1', app_surface: 'web', + capture_surface: 'client', conversation_id: 'session-1', + conversation_id_source: 'client_runtime', round_id: 'round-1', provider_type: 'custom', usage_source: 'reported', + token_usage_available: true, + cost_usd_source: 'unavailable', + cost_usd_known: false, }) }) @@ -135,10 +140,15 @@ describe('useAnalytics conversation product events', () => { $ai_provider: 'ollama', $insert_id: 'ai-generation:round-2', app_surface: 'web', + capture_surface: 'client', conversation_id: 'session-1', + conversation_id_source: 'client_runtime', round_id: 'round-2', provider_type: 'custom', usage_source: 'unavailable', + token_usage_available: false, + cost_usd_source: 'unavailable', + cost_usd_known: false, }) }) diff --git a/packages/stage-ui/src/composables/use-analytics.ts b/packages/stage-ui/src/composables/use-analytics.ts index 4f4a78b5c..b0e04b928 100644 --- a/packages/stage-ui/src/composables/use-analytics.ts +++ b/packages/stage-ui/src/composables/use-analytics.ts @@ -429,10 +429,15 @@ export function useAnalytics() { ...(totalTokens != null && { $ai_total_tokens: totalTokens }), $insert_id: `ai-generation:${properties.round_id}`, app_surface: getConversationAnalyticsSurface(), + capture_surface: 'client', conversation_id: properties.conversation_id, + conversation_id_source: 'client_runtime', round_id: properties.round_id, provider_type: properties.provider_type, usage_source: properties.usage_source, + token_usage_available: properties.usage_source !== 'unavailable', + cost_usd_source: 'unavailable', + cost_usd_known: false, }) }