nanakura/moeka-project
1
0
Fork 0
Code Issues Pull Requests 1 Actions 2 Packages Projects Releases Wiki Activity

fix(analytics): normalize PostHog event semantics (#2041)

Browse Source
This commit is contained in:
RainbowBird
2026-07-10 13:54:23 +08:00
committed by GitHub
parent 231eccf362
commit 0e4dc69ec0
23 changed files with 808 additions and 412 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/core-agent/src/runtime/chat-orchestrator-runtime.test.ts
+89 -2
View File
@@ -43,6 +43,7 @@ function createHarness() {
llmFirstToken: [] as unknown[],
assistantResponseRendered: [] as unknown[],
messageRound: [] as unknown[],
messageRoundFailed: [] as unknown[],
}
const stream = vi.fn(async (_model: string, _chatProvider: ChatProvider, _messages: Message[], options?: {
onStreamEvent?: (event: StreamEvent) => Promise<void> | void
@@ -100,6 +101,7 @@ function createHarness() {
onLlmFirstToken: event => telemetry.llmFirstToken.push(event),
onAssistantResponseRendered: event => telemetry.assistantResponseRendered.push(event),
onMessageRound: event => telemetry.messageRound.push(event),
onMessageRoundFailed: event => telemetry.messageRoundFailed.push(event),
})
return {
@@ -308,42 +310,86 @@ describe('createChatOrchestratorRuntime', () => {
})
expect(harness.telemetry.messageSendStarted).toEqual([{
conversationId: 'session-1',
roundId: 'user-id',
source: 'voice',
model: 'gpt-test',
turnIndex: 1,
}])
expect(harness.telemetry.llmRequestStarted).toEqual([{
conversationId: 'session-1',
roundId: 'user-id',
model: 'gpt-test',
provider: 'mock-provider',
hasVoice: true,
turnIndex: 1,
}])
expect(harness.telemetry.llmFirstToken).toEqual([{
conversationId: 'session-1',
roundId: 'user-id',
model: 'gpt-test',
ttfbMs: 100,
turnIndex: 1,
}])
expect(harness.telemetry.assistantResponseRendered).toEqual([{
conversationId: 'session-1',
roundId: 'user-id',
model: 'gpt-test',
latencyMs: 250,
turnIndex: 1,
}])
expect(harness.telemetry.messageRound).toEqual([{
conversationId: 'session-1',
roundId: 'user-id',
durationMs: 360,
hasVoice: true,
model: 'gpt-test',
turnIndex: 1,
}])
expect(harness.telemetry.chatActivationStarted).toEqual([{
conversationId: 'session-1',
model: 'gpt-test',
provider: 'mock-provider',
sessionId: 'session-1',
roundId: 'user-id',
source: 'voice',
turnIndex: 1,
}])
expect(harness.telemetry.chatActivationSucceeded).toEqual([{
conversationId: 'session-1',
durationMs: 360,
model: 'gpt-test',
provider: 'mock-provider',
roundId: 'user-id',
source: 'voice',
turnIndex: 1,
}])
expect(harness.telemetry.chatActivationFailed).toEqual([])
})
// ROOT CAUSE:
//
// Activation callbacks were emitted for every chat round, so production
// `chat_activation_*` volume tracked message traffic instead of the first
// successful assistant response in a conversation.
it('emits activation milestones only until the conversation gets its first assistant response', async () => {
const harness = createHarness()
await harness.runtime.ingest('first turn', {
model: 'gpt-test',
chatProvider: provider,
})
await harness.runtime.ingest('second turn', {
model: 'gpt-test',
chatProvider: provider,
})
expect(harness.telemetry.chatActivationStarted).toHaveLength(1)
expect(harness.telemetry.chatActivationSucceeded).toHaveLength(1)
expect(harness.telemetry.chatActivationFailed).toHaveLength(0)
expect(harness.telemetry.messageSendStarted).toHaveLength(2)
expect(harness.telemetry.messageRound).toHaveLength(2)
})
/**
* @example
* await expect(runtime.ingest('hello', { model, chatProvider })).rejects.toThrow('provider rejected')
@@ -358,19 +404,60 @@ describe('createChatOrchestratorRuntime', () => {
})).rejects.toThrow('provider rejected')
expect(harness.telemetry.chatActivationStarted).toEqual([{
conversationId: 'session-1',
model: 'gpt-test',
provider: 'mock-provider',
sessionId: 'session-1',
roundId: 'user-id',
source: 'text',
turnIndex: 1,
}])
expect(harness.telemetry.chatActivationSucceeded).toEqual([])
expect(harness.telemetry.chatActivationFailed).toEqual([{
conversationId: 'session-1',
errorCode: 'llm_response_failed',
failureStage: 'llm_response',
model: 'gpt-test',
provider: 'mock-provider',
roundId: 'user-id',
source: 'text',
turnIndex: 1,
}])
expect(harness.telemetry.messageRoundFailed).toEqual([{
conversationId: 'session-1',
errorCode: 'llm_response_failed',
failureStage: 'llm_response',
model: 'gpt-test',
provider: 'mock-provider',
roundId: 'user-id',
source: 'text',
turnIndex: 1,
}])
})
it('emits a round failure for later turns without repeating activation failure', async () => {
const harness = createHarness()
await harness.runtime.ingest('first turn succeeds', {
model: 'gpt-test',
chatProvider: provider,
})
harness.stream.mockRejectedValueOnce(new Error('later turn rejected'))
await expect(harness.runtime.ingest('second turn fails', {
model: 'gpt-test',
chatProvider: provider,
})).rejects.toThrow('later turn rejected')
expect(harness.telemetry.chatActivationFailed).toEqual([])
expect(harness.telemetry.messageRoundFailed).toEqual([
expect.objectContaining({
conversationId: 'session-1',
errorCode: 'llm_response_failed',
failureStage: 'llm_response',
roundId: expect.any(String),
turnIndex: 2,
}),
])
})
/**
packages/core-agent/src/runtime/chat-orchestrator-runtime.ts
+83 -29
View File
@@ -156,6 +156,16 @@ export interface ChatOrchestratorRuntimeState {
pendingQueuedSendCount: number
}
/** Correlation keys shared by every analytics milestone from one user-to-assistant round. */
interface ChatRoundCorrelation {
/** Application conversation that owns the round. */
conversationId: string
/** Stable round key; the runtime reuses the persisted user-message ID. */
roundId: string
/** One-based user turn position within the conversation. */
turnIndex: number
}
/**
* Dependency surface used by the platform-agnostic chat orchestrator runtime.
*/
@@ -190,22 +200,21 @@ export interface ChatOrchestratorRuntimeDeps {
onSendSettled?: (event: { sessionId: string }) => void
/** Called when a send starts and the first assistant placeholder is created. */
onTrackFirstMessage?: () => void
/** Called when a user starts a chat activation attempt. */
onChatActivationStarted?: (event: {
sessionId: string
/** Called for attempts made before the conversation has its first assistant response. */
onChatActivationStarted?: (event: ChatRoundCorrelation & {
source: 'text' | 'voice'
model: string
provider: string
}) => void
/** Called after one user-to-assistant message round completes successfully. */
onChatActivationSucceeded?: (event: {
/** Called when the conversation reaches its first successful assistant response. */
onChatActivationSucceeded?: (event: ChatRoundCorrelation & {
source: 'text' | 'voice'
model: string
provider: string
durationMs: number
}) => void
/** Called after a chat activation attempt fails before assistant completion. */
onChatActivationFailed?: (event: {
/** Called when a pre-activation attempt fails before assistant completion. */
onChatActivationFailed?: (event: ChatRoundCorrelation & {
source: 'text' | 'voice'
model: string
provider: string
@@ -213,32 +222,40 @@ export interface ChatOrchestratorRuntimeDeps {
errorCode: 'llm_response_failed'
}) => void
/** Called when a user message send begins. */
onMessageSendStarted?: (event: {
onMessageSendStarted?: (event: ChatRoundCorrelation & {
source: 'text' | 'voice'
model: string
}) => void
/** Called immediately before the provider LLM request starts. */
onLlmRequestStarted?: (event: {
onLlmRequestStarted?: (event: ChatRoundCorrelation & {
model: string
provider: string
hasVoice: boolean
}) => void
/** Called when the first text token arrives from the provider stream. */
onLlmFirstToken?: (event: {
onLlmFirstToken?: (event: ChatRoundCorrelation & {
model: string
ttfbMs: number
}) => void
/** Called after the assistant stream is parsed and rendered into runtime state. */
onAssistantResponseRendered?: (event: {
onAssistantResponseRendered?: (event: ChatRoundCorrelation & {
model: string
latencyMs: number
}) => void
/** Called after one user-to-assistant message round completes successfully. */
onMessageRound?: (event: {
onMessageRound?: (event: ChatRoundCorrelation & {
durationMs: number
hasVoice: boolean
model: string
}) => void
/** Called whenever a user-to-assistant round fails before completion. */
onMessageRoundFailed?: (event: ChatRoundCorrelation & {
source: 'text' | 'voice'
model: string
provider: string
failureStage: 'llm_response'
errorCode: 'llm_response_failed'
}) => void
/** Called for context/prompt lifecycle observability. */
onLifecycle?: (record: ChatOrchestratorLifecycleRecord) => void
/** Called with the final provider prompt projection. */
@@ -251,6 +268,7 @@ export interface ChatOrchestratorRuntimeDeps {
source: 'text' | 'voice'
model: string
provider: string
roundId: string
turnIndex: number
}) => void
/** Called after the assistant message has been finalized into session history. */
@@ -386,6 +404,14 @@ export function createChatOrchestratorRuntime(deps: ChatOrchestratorRuntimeDeps)
deps.session.ensureSession(sessionId)
const existingSessionMessages = deps.session.getSessionMessages(sessionId)
const turnIndex = existingSessionMessages.filter(message => message.role === 'user').length + 1
// Activation measures whether a conversation reaches its first assistant
// response. Later turns still emit message and latency telemetry, but they
// must not inflate the one-time activation milestones.
const isActivationAttempt = !existingSessionMessages.some(message => message.role === 'assistant')
// Datetime is no longer injected through the side-channel context store.
// It is applied at message-assembly time (see below) as a system-prompt
// date anchor + per-message [HH:MM] prefixes, which is more KV-cache
@@ -429,14 +455,25 @@ export function createChatOrchestratorRuntime(deps: ChatOrchestratorRuntimeDeps)
patchForegroundStream(sessionId, buildingMessage)
const sendSource = options.input ? 'voice' : 'text'
const activeProvider = deps.getActiveProvider?.() ?? ''
// The user message is the durable start of a round, so its ID also serves
// as the correlation key for every telemetry milestone emitted by it.
const roundId = createId()
const correlation: ChatRoundCorrelation = {
conversationId: sessionId,
roundId,
turnIndex,
}
deps.onTrackFirstMessage?.()
deps.onChatActivationStarted?.({
sessionId,
source: sendSource,
model: options.model,
provider: activeProvider,
})
if (isActivationAttempt) {
deps.onChatActivationStarted?.({
...correlation,
source: sendSource,
model: options.model,
provider: activeProvider,
})
}
deps.onMessageSendStarted?.({
...correlation,
source: sendSource,
model: options.model,
})
@@ -473,15 +510,13 @@ export function createChatOrchestratorRuntime(deps: ChatOrchestratorRuntimeDeps)
if (shouldAbort())
return
const userMessageId = createId()
const userMessage = {
role: 'user' as const,
content: finalContent,
createdAt: sendingCreatedAt,
id: userMessageId,
id: roundId,
}
deps.session.appendSessionMessage(sessionId, userMessage)
const userTurnIndex = deps.session.getSessionMessages(sessionId).filter(message => message.role === 'user').length
// Cloud sync v1: only the raw text part round-trips; image attachments
// and other non-text parts stay local.
@@ -492,7 +527,8 @@ export function createChatOrchestratorRuntime(deps: ChatOrchestratorRuntimeDeps)
source: sendSource,
model: options.model,
provider: activeProvider,
turnIndex: userTurnIndex,
roundId,
turnIndex,
})
const sessionMessagesForSend = deps.session.getSessionMessages(sessionId)
@@ -644,6 +680,7 @@ export function createChatOrchestratorRuntime(deps: ChatOrchestratorRuntimeDeps)
const llmRequestStartedAt = monotonicNow()
let llmFirstTokenEmitted = false
deps.onLlmRequestStarted?.({
...correlation,
model: options.model,
provider: deps.getActiveProvider() || 'unknown',
hasVoice: !!options.input,
@@ -684,6 +721,7 @@ export function createChatOrchestratorRuntime(deps: ChatOrchestratorRuntimeDeps)
if (!llmFirstTokenEmitted) {
llmFirstTokenEmitted = true
deps.onLlmFirstToken?.({
...correlation,
model: options.model,
ttfbMs: Math.round(monotonicNow() - llmRequestStartedAt),
})
@@ -718,6 +756,7 @@ export function createChatOrchestratorRuntime(deps: ChatOrchestratorRuntimeDeps)
await parser.end()
deps.onAssistantResponseRendered?.({
...correlation,
model: options.model,
latencyMs: Math.round(monotonicNow() - llmRequestStartedAt),
})
@@ -751,26 +790,41 @@ export function createChatOrchestratorRuntime(deps: ChatOrchestratorRuntimeDeps)
resetForegroundStream(sessionId)
const durationMs = Math.round(monotonicNow() - roundStartedAt)
deps.onMessageRound?.({
...correlation,
durationMs,
hasVoice: !!options.input,
model: options.model,
})
deps.onChatActivationSucceeded?.({
durationMs,
source: sendSource,
model: options.model,
provider: activeProvider,
})
if (isActivationAttempt) {
deps.onChatActivationSucceeded?.({
...correlation,
durationMs,
source: sendSource,
model: options.model,
provider: activeProvider,
})
}
}
catch (error) {
console.error('Error sending message:', error)
deps.onChatActivationFailed?.({
deps.onMessageRoundFailed?.({
...correlation,
source: sendSource,
model: options.model,
provider: activeProvider,
failureStage: 'llm_response',
errorCode: 'llm_response_failed',
})
if (isActivationAttempt) {
deps.onChatActivationFailed?.({
...correlation,
source: sendSource,
model: options.model,
provider: activeProvider,
failureStage: 'llm_response',
errorCode: 'llm_response_failed',
})
}
throw error
}
finally {
packages/stage-pages/src/pages/settings/flux.vue
+10 -4
View File
@@ -284,11 +284,11 @@ onMounted(async () => {
// PostHog funnel step 1: pricing surface view. Today this is an in-app
// settings page (already-authenticated users); when we add a public
// pricing landing page the surface label changes but the event stays the
// pricing landing page the entry-surface label changes but the event stays the
// same, so the funnel definition in PostHog doesn't need re-wiring.
if (!fluxPurchaseDisabled) {
trackPaywallSeen({
surface: 'settings_flux',
entry_surface: 'settings_flux',
reason: 'manual_topup',
flux_balance_bucket: fluxBalanceBucket(credits.value),
})
@@ -317,7 +317,10 @@ async function handleBuy(stripePriceId: string) {
current_plan: 'flux',
trigger: 'manual_topup',
})
trackPlanSelected(stripePriceId, { currency: selectedCurrency.value })
trackPlanSelected(stripePriceId, {
currency: selectedCurrency.value,
entry_surface: 'settings_flux',
})
try {
const res = await client.api.v1.stripe.checkout.$post({ json: { stripePriceId, currency: selectedCurrency.value } })
if (!res.ok) {
@@ -330,7 +333,10 @@ async function handleBuy(stripePriceId: string) {
// PostHog funnel step 3: about to redirect to Stripe. Capture before
// the page nav so the event is sent (PostHog's beforeunload handler
// would otherwise race the navigation).
trackCheckoutStarted(stripePriceId, { currency: selectedCurrency.value })
trackCheckoutStarted(stripePriceId, {
currency: selectedCurrency.value,
entry_surface: 'settings_flux',
})
// Electron renderer runs from file:// and cannot navigate to Stripe in-window
// (the settings window would load checkout.stripe.com and never come back).
// window.open routes through setWindowOpenHandler -> shell.openExternal, so the
packages/stage-ui/src/composables/use-analytics.test.ts
+128 -131
View File
@@ -70,7 +70,7 @@ describe('useAnalytics conversation product events', () => {
analyticsMocks.isPosthogAvailableInBuildMock.mockClear()
})
it('infers the web surface for browser conversation actions', () => {
it('uses app_surface for the web runtime without occupying the event entry surface', () => {
const analytics = useAnalytics()
analytics.trackTtsStopClicked({
@@ -78,7 +78,7 @@ describe('useAnalytics conversation product events', () => {
})
expect(analyticsMocks.posthogCaptureMock).toHaveBeenCalledWith('tts_stop_clicked', {
surface: 'web',
app_surface: 'web',
reason: 'manual-chat',
})
})
@@ -94,7 +94,7 @@ describe('useAnalytics conversation product events', () => {
})
expect(analyticsMocks.posthogCaptureMock).toHaveBeenCalledWith('chat_session_selected', {
surface: 'mobile',
app_surface: 'mobile',
source: 'sessions_drawer',
message_count: 4,
cloud_synced: true,
@@ -118,36 +118,42 @@ describe('useAnalytics conversation product events', () => {
})
expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(1, 'chat_message_deleted', {
surface: 'electron',
app_surface: 'electron',
source: 'history',
message_role: 'assistant',
})
expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(2, 'chat_messages_cleared', {
surface: 'electron',
app_surface: 'electron',
source: 'chat_controls',
message_count: 3,
})
expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(3, 'chat_message_retried', {
surface: 'electron',
app_surface: 'electron',
source: 'history',
})
})
/**
* @example
* analytics.trackChatActivationStarted({ provider_mode: 'official', provider_id: 'official-provider', model_id: 'gpt-test', source: 'text' })
* expect(posthog.capture).toHaveBeenCalledWith('chat_activation_started', expect.objectContaining({ surface: 'web' }))
* analytics.trackChatActivationStarted({ conversation_id: 'session-1', round_id: 'round-1', turn_index: 1, provider_mode: 'official', provider_id: 'official-provider', model_id: 'gpt-test', source: 'text' })
* expect(posthog.capture).toHaveBeenCalledWith('chat_activation_started', expect.objectContaining({ app_surface: 'web' }))
*/
it('emits chat activation milestones with inferred surface and normalized fields', () => {
const analytics = useAnalytics()
analytics.trackChatActivationStarted({
conversation_id: 'session-1',
round_id: 'round-1',
turn_index: 1,
provider_mode: 'official',
provider_id: 'official-provider',
model_id: 'gpt-test',
source: 'text',
})
analytics.trackChatActivationSucceeded({
conversation_id: 'session-1',
round_id: 'round-1',
turn_index: 1,
provider_mode: 'official',
provider_id: 'official-provider',
model_id: 'gpt-test',
@@ -155,6 +161,9 @@ describe('useAnalytics conversation product events', () => {
source: 'voice',
})
analytics.trackChatActivationFailed({
conversation_id: 'session-1',
round_id: 'round-1',
turn_index: 1,
provider_mode: 'custom',
provider_id: 'openai-compatible',
model_id: 'custom',
@@ -164,14 +173,20 @@ describe('useAnalytics conversation product events', () => {
})
expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(1, 'chat_activation_started', {
surface: 'web',
app_surface: 'web',
conversation_id: 'session-1',
round_id: 'round-1',
turn_index: 1,
provider_mode: 'official',
provider_id: 'official-provider',
model_id: 'gpt-test',
source: 'text',
})
expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(2, 'chat_activation_succeeded', {
surface: 'web',
app_surface: 'web',
conversation_id: 'session-1',
round_id: 'round-1',
turn_index: 1,
provider_mode: 'official',
provider_id: 'official-provider',
model_id: 'gpt-test',
@@ -179,7 +194,10 @@ describe('useAnalytics conversation product events', () => {
source: 'voice',
})
expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(3, 'chat_activation_failed', {
surface: 'web',
app_surface: 'web',
conversation_id: 'session-1',
round_id: 'round-1',
turn_index: 1,
provider_mode: 'custom',
provider_id: 'openai-compatible',
model_id: 'custom',
@@ -225,13 +243,13 @@ describe('useAnalytics conversation product events', () => {
})
expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(1, 'tts_provider_selected', {
surface: 'web',
app_surface: 'web',
tts_provider_id: 'official-provider',
tts_model_id: 'stepfun/tts',
source: 'settings',
})
expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(2, 'voice_selected', {
surface: 'web',
app_surface: 'web',
tts_provider_id: 'official-provider',
tts_model_id: 'stepfun/tts',
voice_id: 'longxiaochun_v2',
@@ -239,7 +257,7 @@ describe('useAnalytics conversation product events', () => {
source: 'settings',
})
expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(3, 'voice_preview_played', {
surface: 'web',
app_surface: 'web',
tts_provider_id: 'official-provider',
tts_model_id: 'stepfun/tts',
voice_id: 'longxiaochun_v2',
@@ -247,7 +265,7 @@ describe('useAnalytics conversation product events', () => {
source: 'manual_preview',
})
expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(4, 'voice_pack_bound', {
surface: 'web',
app_surface: 'web',
tts_provider_id: 'official-provider',
tts_model_id: 'stepfun/tts',
voice_id: 'longxiaochun_v2',
@@ -272,15 +290,17 @@ describe('useAnalytics conversation product events', () => {
model_id: 'chat-auto',
})
analytics.trackSecondTurnStarted({
conversation_id: 'session-1',
provider_id: 'official-provider',
provider_mode: 'official',
model_id: 'chat-auto',
round_id: 'round-2',
source: 'text',
turn_index: 2,
})
expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(1, 'official_provider_selected', {
surface: 'web',
app_surface: 'web',
provider_id: 'official-provider',
provider_mode: 'official',
source: 'default_auto',
@@ -288,10 +308,12 @@ describe('useAnalytics conversation product events', () => {
model_id: 'chat-auto',
})
expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(2, 'second_turn_started', {
surface: 'web',
app_surface: 'web',
conversation_id: 'session-1',
provider_id: 'official-provider',
provider_mode: 'official',
model_id: 'chat-auto',
round_id: 'round-2',
source: 'text',
turn_index: 2,
})
@@ -333,13 +355,13 @@ describe('useAnalytics conversation product events', () => {
})
expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(1, 'official_tts_exposed', {
surface: 'web',
app_surface: 'web',
tts_provider_id: 'official-provider-speech',
tts_model_id: 'stepfun/tts',
source: 'post_first_chat',
})
expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(2, 'official_tts_preview_started', {
surface: 'web',
app_surface: 'web',
tts_provider_id: 'official-provider-speech',
tts_model_id: 'stepfun/tts',
voice_id: 'longxiaochun_v2',
@@ -347,7 +369,7 @@ describe('useAnalytics conversation product events', () => {
source: 'manual_preview',
})
expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(3, 'official_tts_preview_succeeded', {
surface: 'web',
app_surface: 'web',
tts_provider_id: 'official-provider-speech',
tts_model_id: 'stepfun/tts',
voice_id: 'longxiaochun_v2',
@@ -356,7 +378,7 @@ describe('useAnalytics conversation product events', () => {
duration_ms: 320,
})
expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(4, 'official_tts_auto_enabled', {
surface: 'web',
app_surface: 'web',
tts_provider_id: 'official-provider-speech',
tts_model_id: 'stepfun/tts',
source: 'settings',
@@ -368,23 +390,55 @@ describe('useAnalytics conversation product events', () => {
const analytics = useAnalytics()
analytics.trackPaywallSeen({
surface: 'settings_flux',
entry_surface: 'settings_flux',
reason: 'manual_topup',
flux_balance_bucket: '1_100',
})
expect(analyticsMocks.posthogCaptureMock).toHaveBeenCalledWith('paywall_seen', {
surface: 'settings_flux',
app_surface: 'web',
entry_surface: 'settings_flux',
reason: 'manual_topup',
flux_balance_bucket: '1_100',
})
})
it('uses entry_surface across the pricing funnel without emitting surface', () => {
const analytics = useAnalytics()
analytics.trackPricingViewed('settings_flux', 'one_time')
analytics.trackPlanSelected('price-1', {
currency: 'USD',
entry_surface: 'settings_flux',
})
analytics.trackCheckoutStarted('price-1', {
currency: 'USD',
entry_surface: 'settings_flux',
})
expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(1, 'pricing_page_viewed', {
entry_surface: 'settings_flux',
plan_period: 'one_time',
})
expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(2, 'plan_selected', {
currency: 'USD',
entry_surface: 'settings_flux',
plan_id: 'price-1',
})
expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(3, 'checkout_started', {
currency: 'USD',
entry_surface: 'settings_flux',
plan_id: 'price-1',
}, {
send_instantly: true,
transport: 'sendBeacon',
})
})
/**
* @example
* analytics.trackMicrophonePermissionDenied({ stt_provider_id: 'browser-web-speech-api' })
* expect(posthog.capture).toHaveBeenCalledWith('microphone_permission_denied', expect.objectContaining({ surface: 'web' }))
* expect(posthog.capture).toHaveBeenCalledWith('microphone_permission_denied', expect.objectContaining({ app_surface: 'web' }))
*/
it('emits voice input friction events with low-cardinality error fields', () => {
const analytics = useAnalytics()
@@ -409,29 +463,29 @@ describe('useAnalytics conversation product events', () => {
})
expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(1, 'voice_input_started', {
surface: 'web',
app_surface: 'web',
stt_provider_id: 'browser-web-speech-api',
})
expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(2, 'voice_input_used', {
surface: 'web',
app_surface: 'web',
stt_provider_id: 'browser-web-speech-api',
})
expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(3, 'microphone_permission_requested', {
surface: 'web',
app_surface: 'web',
stt_provider_id: 'browser-web-speech-api',
})
expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(4, 'microphone_permission_denied', {
surface: 'web',
app_surface: 'web',
stt_provider_id: 'browser-web-speech-api',
error_code: 'permission_denied',
})
expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(5, 'audio_device_unavailable', {
surface: 'web',
app_surface: 'web',
stt_provider_id: 'browser-web-speech-api',
error_code: 'device_unavailable',
})
expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(6, 'voice_input_cancelled', {
surface: 'web',
app_surface: 'web',
stt_provider_id: 'browser-web-speech-api',
duration_ms: 420,
})
@@ -459,14 +513,14 @@ describe('useAnalytics conversation product events', () => {
})
expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(1, 'model_list_loaded', {
surface: 'web',
app_surface: 'web',
provider_id: 'official-provider',
provider_mode: 'official',
model_count: 3,
duration_ms: 25,
})
expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(2, 'model_list_failed', {
surface: 'web',
app_surface: 'web',
provider_id: 'openai-compatible',
provider_mode: 'custom',
error_code: 'provider_error',
@@ -502,20 +556,20 @@ describe('useAnalytics conversation product events', () => {
})
expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(1, 'provider_config_started', {
surface: 'web',
app_surface: 'web',
provider_id: 'openai-compatible',
provider_mode: 'custom',
step: 'settings_auto_validate',
})
expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(2, 'provider_config_succeeded', {
surface: 'web',
app_surface: 'web',
provider_id: 'official-provider',
provider_mode: 'official',
step: 'manual_chat_ping',
duration_ms: 18,
})
expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(3, 'provider_config_completed', {
surface: 'web',
app_surface: 'web',
provider_id: 'official-provider',
provider_mode: 'official',
provider_type: 'official',
@@ -526,12 +580,12 @@ describe('useAnalytics conversation product events', () => {
success: true,
})
expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(4, 'official_provider_enabled', {
surface: 'web',
app_surface: 'web',
provider_name: 'official-provider',
entry: 'settings',
})
expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(5, 'provider_config_failed', {
surface: 'web',
app_surface: 'web',
provider_id: 'openai-compatible',
provider_mode: 'custom',
step: 'settings_auto_validate',
@@ -540,14 +594,9 @@ describe('useAnalytics conversation product events', () => {
})
})
it('emits P0 activation, chat, quota, and feature events using canonical names', () => {
it('emits P0 onboarding, message, quota, and feature events using canonical names', () => {
const analytics = useAnalytics()
analytics.trackSignupCompleted({
source: 'google',
locale: 'en',
utm_source: 'launch',
})
analytics.trackOnboardingStarted({
entry: 'app_start',
})
@@ -556,41 +605,19 @@ describe('useAnalytics conversation product events', () => {
selected_provider_id: 'official-provider',
selected_use_case: 'role_chat',
})
analytics.trackChatStarted({
conversation_id: 'session-1',
provider_type: 'official',
provider_name: 'official-provider',
model: 'gpt-test',
entry: 'chat',
is_paid_user: true,
})
analytics.trackMessageSent({
conversation_id: 'session-1',
provider_type: 'official',
provider_name: 'official-provider',
model: 'gpt-test',
message_id: 'message-1',
round_id: 'message-1',
turn_index: 1,
message_index: 2,
message_length: 24,
has_attachment: false,
mode: 'text',
})
analytics.trackAssistantResponseCompleted({
conversation_id: 'session-1',
provider_type: 'official',
provider_name: 'official-provider',
model: 'gpt-test',
latency_ms: 350,
completion_length: 120,
})
analytics.trackChatFailed({
conversation_id: 'session-1',
provider_type: 'custom',
provider_name: 'openai-compatible',
model: 'custom',
failure_stage: 'llm_response',
error_code: 'provider_error',
})
analytics.trackQuotaLimitReached({
limit_type: 'flux',
current_usage: 0,
@@ -609,73 +636,43 @@ describe('useAnalytics conversation product events', () => {
success: true,
})
expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(1, 'signup_completed', {
source: 'google',
locale: 'en',
utm_source: 'launch',
})
expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(2, 'onboarding_started', {
surface: 'web',
expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(1, 'onboarding_started', {
app_surface: 'web',
entry: 'app_start',
})
expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(3, 'onboarding_completed', {
surface: 'web',
expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(2, 'onboarding_completed', {
app_surface: 'web',
selected_provider_type: 'official',
selected_provider_id: 'official-provider',
selected_use_case: 'role_chat',
})
expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(4, 'chat_started', {
surface: 'web',
conversation_id: 'session-1',
provider_type: 'official',
provider_name: 'official-provider',
model: 'gpt-test',
entry: 'chat',
is_paid_user: true,
})
expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(5, 'message_sent', {
surface: 'web',
expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(3, 'message_sent', {
app_surface: 'web',
conversation_id: 'session-1',
provider_type: 'official',
provider_name: 'official-provider',
model: 'gpt-test',
message_id: 'message-1',
round_id: 'message-1',
turn_index: 1,
message_index: 2,
message_length: 24,
has_attachment: false,
mode: 'text',
})
expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(6, 'assistant_response_completed', {
surface: 'web',
conversation_id: 'session-1',
provider_type: 'official',
provider_name: 'official-provider',
model: 'gpt-test',
latency_ms: 350,
completion_length: 120,
})
expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(7, 'chat_failed', {
surface: 'web',
conversation_id: 'session-1',
provider_type: 'custom',
provider_name: 'openai-compatible',
model: 'custom',
failure_stage: 'llm_response',
error_code: 'provider_error',
})
expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(8, 'quota_limit_reached', {
expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(4, 'quota_limit_reached', {
limit_type: 'flux',
current_usage: 0,
limit_value: 0,
entry: 'pricing',
})
expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(9, 'upgrade_clicked', {
expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(5, 'upgrade_clicked', {
source_page: 'settings_flux',
current_plan: 'flux',
trigger: 'manual_topup',
})
expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(10, 'feature_used', {
surface: 'web',
expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(6, 'feature_used', {
app_surface: 'web',
feature_name: 'chat',
business_domain: 'conversation',
entry: 'chat',
@@ -736,43 +733,43 @@ describe('useAnalytics conversation product events', () => {
})
expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(1, 'conversation_created', {
surface: 'web',
app_surface: 'web',
conversation_id: 'session-1',
source: 'new_session',
character_id: 'character-1',
cloud_synced: true,
})
expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(2, 'conversation_renamed', {
surface: 'web',
app_surface: 'web',
conversation_id: 'session-1',
source: 'sessions_drawer',
})
expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(3, 'conversation_shared', {
surface: 'web',
app_surface: 'web',
conversation_id: 'session-1',
source: 'share_button',
})
expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(4, 'conversation_deleted', {
surface: 'web',
app_surface: 'web',
conversation_id: 'session-1',
message_count: 6,
cloud_synced: true,
})
expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(5, 'attachment_uploaded', {
surface: 'web',
app_surface: 'web',
attachment_type: 'image',
size_bytes: 2048,
source: 'chat',
success: true,
})
expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(6, 'preset_used', {
surface: 'web',
app_surface: 'web',
preset_id: 'preset-live2d-1',
preset_type: 'stage_model',
source: 'settings',
})
expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(7, 'provider_switched', {
surface: 'web',
app_surface: 'web',
from_provider: 'openai-compatible',
to_provider: 'official-provider',
from_provider_type: 'custom',
@@ -780,14 +777,14 @@ describe('useAnalytics conversation product events', () => {
reason: 'manual',
})
expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(8, 'settings_changed', {
surface: 'web',
app_surface: 'web',
setting_name: 'analytics_enabled',
previous_value: false,
new_value: true,
source: 'settings',
})
expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(9, 'support_contacted', {
surface: 'web',
app_surface: 'web',
channel: 'discord',
source: 'settings',
category: 'payment',
@@ -821,7 +818,7 @@ describe('useAnalytics conversation product events', () => {
})
expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(1, 'bug_report_submitted', {
surface: 'web',
app_surface: 'web',
source: 'app',
category: 'update',
severity: 'major',
@@ -832,7 +829,7 @@ describe('useAnalytics conversation product events', () => {
screenshot_attached: true,
})
expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(2, 'feedback_submitted', {
surface: 'web',
app_surface: 'web',
source: 'discord',
category: 'voice_input',
severity: 'minor',
@@ -851,21 +848,21 @@ describe('useAnalytics conversation product events', () => {
analytics.trackAccountDeletionRequested()
analytics.trackOauthCallbackFailed({ stage: 'missing_flow_state' })
expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(1, 'password_changed', { surface: 'web' })
expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(2, 'password_reset_requested', { surface: 'web' })
expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(1, 'password_changed', { app_surface: 'web' })
expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(2, 'password_reset_requested', { app_surface: 'web' })
expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(
3,
'oauth_provider_link_started',
{ surface: 'web', provider: 'github' },
{ app_surface: 'web', provider: 'github' },
{ send_instantly: true, transport: 'sendBeacon' },
)
expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(4, 'oauth_provider_unlinked', {
surface: 'web',
app_surface: 'web',
provider: 'google',
})
expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(5, 'account_deletion_requested', { surface: 'web' })
expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(5, 'account_deletion_requested', { app_surface: 'web' })
expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(6, 'oauth_callback_failed', {
surface: 'web',
app_surface: 'web',
stage: 'missing_flow_state',
})
})
@@ -879,16 +876,16 @@ describe('useAnalytics conversation product events', () => {
analytics.trackCharacterUpdated({ character_id: 'character-1' })
expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(1, 'card_edited', {
surface: 'web',
app_surface: 'web',
card_id: 'card-1',
})
expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(2, 'scene_background_set', {
surface: 'web',
app_surface: 'web',
source: 'card_gallery',
cleared: false,
})
expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(3, 'scene_background_set', {
surface: 'web',
app_surface: 'web',
source: 'scene_settings',
cleared: true,
})
@@ -904,11 +901,11 @@ describe('useAnalytics conversation product events', () => {
analytics.trackDataAction({ action: 'app_data_cleared' })
expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(1, 'data_action', {
surface: 'web',
app_surface: 'web',
action: 'chats_exported',
})
expect(analyticsMocks.posthogCaptureMock).toHaveBeenNthCalledWith(2, 'data_action', {
surface: 'web',
app_surface: 'web',
action: 'app_data_cleared',
})
})
packages/stage-ui/src/composables/use-analytics.ts
+97 -126
View File
@@ -51,7 +51,13 @@ export type OauthCallbackFailureStage
| 'parse'
| 'relay_unreachable'
interface ChatActivationBaseProperties {
interface ChatRoundCorrelationProperties {
conversation_id: string
round_id: string
turn_index: number
}
interface ChatActivationBaseProperties extends ChatRoundCorrelationProperties {
provider_mode: ProviderMode
provider_id: string
model_id: string
@@ -167,23 +173,23 @@ export function useAnalytics() {
* - Any UI surface that shows Flux packages / subscription plans renders.
* Current surfaces: `settings_flux` (in-app billing settings). Future
* surfaces (a public pricing landing page, an upsell modal) just pass a
* different `surface` so the funnel split stays clean.
* different `entry_surface` so the funnel split stays clean.
*
* Expects:
* - `surface` is a stable identifier — don't rename without coordinating
* - `entry_surface` is a stable identifier — don't rename without coordinating
* PostHog funnel definitions in `docs/ai-context/metrics-ownership.md`.
*/
function trackPricingViewed(surface: string, planPeriod?: 'monthly' | 'annual' | 'one_time') {
function trackPricingViewed(entrySurface: string, planPeriod?: 'monthly' | 'annual' | 'one_time') {
if (!canCapture())
return
posthog.capture('pricing_page_viewed', { surface, ...(planPeriod && { plan_period: planPeriod }) })
posthog.capture('pricing_page_viewed', { entry_surface: entrySurface, ...(planPeriod && { plan_period: planPeriod }) })
}
/**
* Pricing funnel — step 2. Fires when the user picks a plan/package but
* hasn't yet kicked off the Stripe checkout redirect.
*/
function trackPlanSelected(planId: string, properties?: { price_minor_unit?: number, currency?: string }) {
function trackPlanSelected(planId: string, properties: { entry_surface: string, price_minor_unit?: number, currency?: string }) {
if (!canCapture())
return
posthog.capture('plan_selected', { plan_id: planId, ...properties })
@@ -206,7 +212,7 @@ export function useAnalytics() {
* `apps/server/src/services/domain/product-events.ts`), keyed by the
* Better Auth user id.
*/
function trackCheckoutStarted(planId: string, properties: { checkout_session_id?: string, price_minor_unit?: number, currency?: string }) {
function trackCheckoutStarted(planId: string, properties: { entry_surface: string, checkout_session_id?: string, price_minor_unit?: number, currency?: string }) {
if (!canCapture())
return
posthog.capture(
@@ -217,14 +223,14 @@ export function useAnalytics() {
}
function trackPaywallSeen(properties: {
surface: string
entry_surface: string
reason: 'manual_topup' | 'insufficient_balance' | 'checkout_recovery' | 'unknown'
flux_balance_bucket: FluxBalanceBucket
}) {
if (!canCapture())
return
posthog.capture('paywall_seen', {
surface: properties.surface,
entry_surface: properties.entry_surface,
app_surface: getConversationAnalyticsSurface(),
reason: properties.reason,
flux_balance_bucket: properties.flux_balance_bucket,
@@ -243,7 +249,7 @@ export function useAnalytics() {
return
posthog.capture('oauth_callback_failed', {
...properties,
surface: getConversationAnalyticsSurface(),
app_surface: getConversationAnalyticsSurface(),
})
}
@@ -253,13 +259,13 @@ export function useAnalytics() {
function trackPasswordChanged() {
if (!canCapture())
return
posthog.capture('password_changed', { surface: getConversationAnalyticsSurface() })
posthog.capture('password_changed', { app_surface: getConversationAnalyticsSurface() })
}
function trackPasswordResetRequested() {
if (!canCapture())
return
posthog.capture('password_reset_requested', { surface: getConversationAnalyticsSurface() })
posthog.capture('password_reset_requested', { app_surface: getConversationAnalyticsSurface() })
}
function trackOauthProviderLinkStarted(properties: { provider: string }) {
@@ -272,7 +278,7 @@ export function useAnalytics() {
'oauth_provider_link_started',
{
...properties,
surface: getConversationAnalyticsSurface(),
app_surface: getConversationAnalyticsSurface(),
},
{ send_instantly: true, transport: 'sendBeacon' },
)
@@ -283,7 +289,7 @@ export function useAnalytics() {
return
posthog.capture('oauth_provider_unlinked', {
...properties,
surface: getConversationAnalyticsSurface(),
app_surface: getConversationAnalyticsSurface(),
})
}
@@ -295,21 +301,7 @@ export function useAnalytics() {
function trackAccountDeletionRequested() {
if (!canCapture())
return
posthog.capture('account_deletion_requested', { surface: getConversationAnalyticsSurface() })
}
function trackSignupCompleted(properties: {
source: string
referrer?: string
country?: string
locale?: string
utm_source?: string
utm_medium?: string
utm_campaign?: string
}) {
if (!canCapture())
return
posthog.capture('signup_completed', properties)
posthog.capture('account_deletion_requested', { app_surface: getConversationAnalyticsSurface() })
}
function trackOnboardingStarted(properties: { entry: ProductAnalyticsEntry }) {
@@ -317,7 +309,7 @@ export function useAnalytics() {
return
posthog.capture('onboarding_started', {
...properties,
surface: getConversationAnalyticsSurface(),
app_surface: getConversationAnalyticsSurface(),
})
}
@@ -326,7 +318,7 @@ export function useAnalytics() {
return
posthog.capture('onboarding_completed', {
...properties,
surface: getConversationAnalyticsSurface(),
app_surface: getConversationAnalyticsSurface(),
})
}
@@ -357,7 +349,7 @@ export function useAnalytics() {
from_model: fromModel,
to_model: toModel,
reason,
surface: getConversationAnalyticsSurface(),
app_surface: getConversationAnalyticsSurface(),
})
}
@@ -379,39 +371,55 @@ export function useAnalytics() {
// (per-request volume stays in DB/Grafana). These client emits supply the
// user-facing latency picture (TTFT, render time) the server cannot see.
function trackMessageSendStarted(properties: { source: 'text' | 'voice', model?: string }) {
function trackMessageSendStarted(properties: ChatRoundCorrelationProperties & { source: 'text' | 'voice', model?: string }) {
if (!canCapture())
return
posthog.capture('message_send_started', properties)
}
function trackLlmRequestStarted(properties: { model: string, provider: string, has_voice: boolean }) {
function trackLlmRequestStarted(properties: ChatRoundCorrelationProperties & { model: string, provider: string, has_voice: boolean }) {
if (!canCapture())
return
posthog.capture('llm_request_started', properties)
}
/** First token from a streaming LLM response — perceived responsiveness anchor. */
function trackLlmFirstToken(properties: { model: string, ttfb_ms: number }) {
function trackLlmFirstToken(properties: ChatRoundCorrelationProperties & { model: string, ttfb_ms: number }) {
if (!canCapture())
return
posthog.capture('llm_first_token', properties)
}
/** Stream finished and the UI has fully rendered the assistant message. */
function trackAssistantResponseRendered(properties: { model: string, latency_ms: number }) {
function trackAssistantResponseRendered(properties: ChatRoundCorrelationProperties & { model: string, latency_ms: number }) {
if (!canCapture())
return
posthog.capture('assistant_response_rendered', properties)
}
/** Closing event for one full message round (user send → assistant render). */
function trackMessageRound(properties: { duration_ms: number, has_voice: boolean, model: string }) {
function trackMessageRound(properties: ChatRoundCorrelationProperties & { duration_ms: number, has_voice: boolean, model: string }) {
if (!canCapture())
return
posthog.capture('message_round', properties)
}
/** Canonical failure event for every user-to-assistant round, including post-activation turns. */
function trackMessageRoundFailed(properties: ChatRoundCorrelationProperties & {
provider_id: string
model_id: string
source: 'text' | 'voice'
error_code: string
failure_stage: ChatActivationFailureStage
}) {
if (!canCapture())
return
posthog.capture('message_round_failed', {
...properties,
app_surface: getConversationAnalyticsSurface(),
})
}
// ─── Chat activation events ──────────────────────────────────────────
function trackChatActivationStarted(properties: ChatActivationBaseProperties) {
@@ -419,7 +427,7 @@ export function useAnalytics() {
return
posthog.capture('chat_activation_started', {
...properties,
surface: getConversationAnalyticsSurface(),
app_surface: getConversationAnalyticsSurface(),
})
}
@@ -428,7 +436,7 @@ export function useAnalytics() {
return
posthog.capture('chat_activation_succeeded', {
...properties,
surface: getConversationAnalyticsSurface(),
app_surface: getConversationAnalyticsSurface(),
})
}
@@ -440,19 +448,7 @@ export function useAnalytics() {
return
posthog.capture('chat_activation_failed', {
...properties,
surface: getConversationAnalyticsSurface(),
})
}
function trackChatStarted(properties: ConversationBaseProperties & {
entry: ProductAnalyticsEntry
is_paid_user?: boolean
}) {
if (!canCapture())
return
posthog.capture('chat_started', {
...properties,
surface: getConversationAnalyticsSurface(),
app_surface: getConversationAnalyticsSurface(),
})
}
@@ -467,11 +463,13 @@ export function useAnalytics() {
return
posthog.capture('official_provider_selected', {
...properties,
surface: getConversationAnalyticsSurface(),
app_surface: getConversationAnalyticsSurface(),
})
}
function trackMessageSent(properties: ConversationBaseProperties & {
round_id: string
turn_index: number
message_id?: string
message_index?: number
message_length?: number
@@ -482,40 +480,16 @@ export function useAnalytics() {
return
posthog.capture('message_sent', {
...properties,
surface: getConversationAnalyticsSurface(),
app_surface: getConversationAnalyticsSurface(),
})
}
function trackAssistantResponseCompleted(properties: ConversationBaseProperties & {
latency_ms?: number
completion_length?: number
}) {
if (!canCapture())
return
posthog.capture('assistant_response_completed', {
...properties,
surface: getConversationAnalyticsSurface(),
})
}
function trackChatFailed(properties: ConversationBaseProperties & {
failure_stage: ChatActivationFailureStage
error_code: string
}) {
if (!canCapture())
return
posthog.capture('chat_failed', {
...properties,
surface: getConversationAnalyticsSurface(),
})
}
function trackSecondTurnStarted(properties: ChatActivationBaseProperties & { turn_index: number }) {
function trackSecondTurnStarted(properties: ChatActivationBaseProperties) {
if (!canCapture())
return
posthog.capture('second_turn_started', {
...properties,
surface: getConversationAnalyticsSurface(),
app_surface: getConversationAnalyticsSurface(),
})
}
@@ -529,7 +503,7 @@ export function useAnalytics() {
return
posthog.capture('model_list_loaded', {
...properties,
surface: getConversationAnalyticsSurface(),
app_surface: getConversationAnalyticsSurface(),
})
}
@@ -543,7 +517,7 @@ export function useAnalytics() {
return
posthog.capture('model_list_failed', {
...properties,
surface: getConversationAnalyticsSurface(),
app_surface: getConversationAnalyticsSurface(),
})
}
@@ -552,7 +526,7 @@ export function useAnalytics() {
return
posthog.capture('provider_config_started', {
...properties,
surface: getConversationAnalyticsSurface(),
app_surface: getConversationAnalyticsSurface(),
})
}
@@ -561,7 +535,7 @@ export function useAnalytics() {
return
posthog.capture('provider_config_succeeded', {
...properties,
surface: getConversationAnalyticsSurface(),
app_surface: getConversationAnalyticsSurface(),
})
trackProviderConfigCompleted({
...properties,
@@ -583,7 +557,7 @@ export function useAnalytics() {
return
posthog.capture('provider_config_failed', {
...properties,
surface: getConversationAnalyticsSurface(),
app_surface: getConversationAnalyticsSurface(),
})
}
@@ -599,7 +573,7 @@ export function useAnalytics() {
provider_type: properties.provider_mode,
provider_name: properties.provider_id,
entry_page: properties.step,
surface: getConversationAnalyticsSurface(),
app_surface: getConversationAnalyticsSurface(),
})
}
@@ -611,7 +585,7 @@ export function useAnalytics() {
return
posthog.capture('official_provider_enabled', {
...properties,
surface: getConversationAnalyticsSurface(),
app_surface: getConversationAnalyticsSurface(),
})
}
@@ -622,7 +596,7 @@ export function useAnalytics() {
return
posthog.capture('tts_stop_clicked', {
...properties,
surface: getConversationAnalyticsSurface(),
app_surface: getConversationAnalyticsSurface(),
})
}
@@ -631,7 +605,7 @@ export function useAnalytics() {
return
posthog.capture('chat_session_selected', {
...properties,
surface: getConversationAnalyticsSurface(),
app_surface: getConversationAnalyticsSurface(),
})
}
@@ -640,7 +614,7 @@ export function useAnalytics() {
return
posthog.capture('chat_message_deleted', {
...properties,
surface: getConversationAnalyticsSurface(),
app_surface: getConversationAnalyticsSurface(),
})
}
@@ -649,7 +623,7 @@ export function useAnalytics() {
return
posthog.capture('chat_messages_cleared', {
...properties,
surface: getConversationAnalyticsSurface(),
app_surface: getConversationAnalyticsSurface(),
})
}
@@ -658,7 +632,7 @@ export function useAnalytics() {
return
posthog.capture('chat_message_retried', {
...properties,
surface: getConversationAnalyticsSurface(),
app_surface: getConversationAnalyticsSurface(),
})
}
@@ -672,7 +646,7 @@ export function useAnalytics() {
return
posthog.capture('conversation_created', {
...properties,
surface: getConversationAnalyticsSurface(),
app_surface: getConversationAnalyticsSurface(),
})
}
@@ -684,7 +658,7 @@ export function useAnalytics() {
return
posthog.capture('conversation_renamed', {
...properties,
surface: getConversationAnalyticsSurface(),
app_surface: getConversationAnalyticsSurface(),
})
}
@@ -696,7 +670,7 @@ export function useAnalytics() {
return
posthog.capture('conversation_shared', {
...properties,
surface: getConversationAnalyticsSurface(),
app_surface: getConversationAnalyticsSurface(),
})
}
@@ -709,7 +683,7 @@ export function useAnalytics() {
return
posthog.capture('conversation_deleted', {
...properties,
surface: getConversationAnalyticsSurface(),
app_surface: getConversationAnalyticsSurface(),
})
}
@@ -738,11 +712,11 @@ export function useAnalytics() {
return
posthog.capture('voice_input_started', {
...properties,
surface: getConversationAnalyticsSurface(),
app_surface: getConversationAnalyticsSurface(),
})
posthog.capture('voice_input_used', {
...properties,
surface: getConversationAnalyticsSurface(),
app_surface: getConversationAnalyticsSurface(),
})
}
@@ -751,7 +725,7 @@ export function useAnalytics() {
return
posthog.capture('microphone_permission_requested', {
...properties,
surface: getConversationAnalyticsSurface(),
app_surface: getConversationAnalyticsSurface(),
})
}
@@ -760,7 +734,7 @@ export function useAnalytics() {
return
posthog.capture('microphone_permission_denied', {
...properties,
surface: getConversationAnalyticsSurface(),
app_surface: getConversationAnalyticsSurface(),
})
}
@@ -769,7 +743,7 @@ export function useAnalytics() {
return
posthog.capture('audio_device_unavailable', {
...properties,
surface: getConversationAnalyticsSurface(),
app_surface: getConversationAnalyticsSurface(),
})
}
@@ -778,7 +752,7 @@ export function useAnalytics() {
return
posthog.capture('voice_input_cancelled', {
...properties,
surface: getConversationAnalyticsSurface(),
app_surface: getConversationAnalyticsSurface(),
})
}
@@ -793,7 +767,7 @@ export function useAnalytics() {
return
posthog.capture('bug_report_submitted', {
...properties,
surface: getConversationAnalyticsSurface(),
app_surface: getConversationAnalyticsSurface(),
})
}
@@ -802,7 +776,7 @@ export function useAnalytics() {
return
posthog.capture('feedback_submitted', {
...properties,
surface: getConversationAnalyticsSurface(),
app_surface: getConversationAnalyticsSurface(),
})
}
@@ -848,7 +822,7 @@ export function useAnalytics() {
return
posthog.capture('tts_provider_selected', {
...properties,
surface: getConversationAnalyticsSurface(),
app_surface: getConversationAnalyticsSurface(),
})
}
@@ -861,7 +835,7 @@ export function useAnalytics() {
return
posthog.capture('voice_selected', {
...properties,
surface: getConversationAnalyticsSurface(),
app_surface: getConversationAnalyticsSurface(),
})
}
@@ -874,7 +848,7 @@ export function useAnalytics() {
return
posthog.capture('voice_preview_played', {
...properties,
surface: getConversationAnalyticsSurface(),
app_surface: getConversationAnalyticsSurface(),
})
}
@@ -886,7 +860,7 @@ export function useAnalytics() {
return
posthog.capture('voice_pack_bound', {
...properties,
surface: getConversationAnalyticsSurface(),
app_surface: getConversationAnalyticsSurface(),
})
}
@@ -900,7 +874,7 @@ export function useAnalytics() {
return
posthog.capture('attachment_uploaded', {
...properties,
surface: getConversationAnalyticsSurface(),
app_surface: getConversationAnalyticsSurface(),
})
}
@@ -909,7 +883,7 @@ export function useAnalytics() {
return
posthog.capture('official_tts_exposed', {
...properties,
surface: getConversationAnalyticsSurface(),
app_surface: getConversationAnalyticsSurface(),
})
}
@@ -922,7 +896,7 @@ export function useAnalytics() {
return
posthog.capture('preset_used', {
...properties,
surface: getConversationAnalyticsSurface(),
app_surface: getConversationAnalyticsSurface(),
})
}
@@ -936,7 +910,7 @@ export function useAnalytics() {
return
posthog.capture('official_tts_preview_started', {
...properties,
surface: getConversationAnalyticsSurface(),
app_surface: getConversationAnalyticsSurface(),
})
}
@@ -951,7 +925,7 @@ export function useAnalytics() {
return
posthog.capture('official_tts_preview_succeeded', {
...properties,
surface: getConversationAnalyticsSurface(),
app_surface: getConversationAnalyticsSurface(),
})
}
@@ -966,7 +940,7 @@ export function useAnalytics() {
return
posthog.capture('provider_switched', {
...properties,
surface: getConversationAnalyticsSurface(),
app_surface: getConversationAnalyticsSurface(),
})
}
@@ -980,7 +954,7 @@ export function useAnalytics() {
return
posthog.capture('settings_changed', {
...properties,
surface: getConversationAnalyticsSurface(),
app_surface: getConversationAnalyticsSurface(),
})
}
@@ -993,7 +967,7 @@ export function useAnalytics() {
return
posthog.capture('support_contacted', {
...properties,
surface: getConversationAnalyticsSurface(),
app_surface: getConversationAnalyticsSurface(),
})
}
@@ -1005,7 +979,7 @@ export function useAnalytics() {
return
posthog.capture('official_tts_auto_enabled', {
...properties,
surface: getConversationAnalyticsSurface(),
app_surface: getConversationAnalyticsSurface(),
})
}
@@ -1029,7 +1003,7 @@ export function useAnalytics() {
return
posthog.capture('card_edited', {
...properties,
surface: getConversationAnalyticsSurface(),
app_surface: getConversationAnalyticsSurface(),
})
}
@@ -1039,7 +1013,7 @@ export function useAnalytics() {
return
posthog.capture('scene_background_set', {
...properties,
surface: getConversationAnalyticsSurface(),
app_surface: getConversationAnalyticsSurface(),
})
}
@@ -1097,7 +1071,7 @@ export function useAnalytics() {
posthog.capture('flux_low_warning_shown', properties)
}
function trackFluxTopupClicked(properties: { balance: number, surface: string }) {
function trackFluxTopupClicked(properties: { balance: number, entry_surface: string }) {
if (!canCapture())
return
posthog.capture('flux_topup_clicked', properties)
@@ -1134,7 +1108,7 @@ export function useAnalytics() {
return
posthog.capture('feature_used', {
...properties,
surface: getConversationAnalyticsSurface(),
app_surface: getConversationAnalyticsSurface(),
})
}
@@ -1153,7 +1127,7 @@ export function useAnalytics() {
return
posthog.capture('data_action', {
...properties,
surface: getConversationAnalyticsSurface(),
app_surface: getConversationAnalyticsSurface(),
})
}
@@ -1243,7 +1217,6 @@ export function useAnalytics() {
trackPlanSelected,
trackCheckoutStarted,
trackPaywallSeen,
trackSignupCompleted,
trackOauthCallbackFailed,
trackPasswordChanged,
trackPasswordResetRequested,
@@ -1261,11 +1234,9 @@ export function useAnalytics() {
trackLlmRequestStarted,
trackLlmFirstToken,
trackAssistantResponseRendered,
trackAssistantResponseCompleted,
trackMessageRound,
trackChatStarted,
trackMessageRoundFailed,
trackMessageSent,
trackChatFailed,
trackChatActivationStarted,
trackChatActivationSucceeded,
trackChatActivationFailed,
packages/stage-ui/src/stores/analytics/index.ts
+4 -4
View File
@@ -59,7 +59,7 @@ export const useSharedAnalyticsStore = defineStore('analytics-shared', () => {
previous_value: previousEnabled,
new_value: enabled,
source: 'settings',
surface: analyticsSurface(),
app_surface: analyticsSurface(),
})
}
@@ -71,7 +71,7 @@ export const useSharedAnalyticsStore = defineStore('analytics-shared', () => {
previous_value: previousEnabled,
new_value: enabled,
source: 'settings',
surface: analyticsSurface(),
app_surface: analyticsSurface(),
})
}
@@ -189,7 +189,7 @@ export const useSharedAnalyticsStore = defineStore('analytics-shared', () => {
from_provider_type: providerMode(prev.provider),
to_provider_type: providerMode(next.provider),
reason: 'manual',
surface: analyticsSurface(),
app_surface: analyticsSurface(),
})
}
@@ -204,7 +204,7 @@ export const useSharedAnalyticsStore = defineStore('analytics-shared', () => {
to_model: next.model,
provider: next.provider,
reason: 'manual',
surface: analyticsSurface(),
app_surface: analyticsSurface(),
})
}
},
packages/stage-ui/src/stores/analytics/posthog.test.ts
+35
View File
@@ -0,0 +1,35 @@
import { describe, expect, it, vi } from 'vitest'
import { ensurePosthogInitialized } from './posthog'
const posthogMocks = vi.hoisted(() => ({
init: vi.fn(),
register: vi.fn(),
}))
vi.mock('posthog-js', () => ({
default: posthogMocks,
}))
vi.mock('@proj-airi/stage-shared', () => ({
isStageCapacitor: () => false,
isStageTamagotchi: () => false,
}))
vi.mock('../../../../../posthog.config', () => ({
DEFAULT_POSTHOG_CONFIG: {},
POSTHOG_ENABLED: true,
POSTHOG_PROJECT_KEY: 'test-project-key',
}))
describe('stage PostHog initialization', () => {
// ROOT CAUSE:
//
// `surface` was registered as the runtime platform but individual events
// also used `surface` for entry points such as `settings_flux`. Event
// properties overwrite super properties, so platform breakdowns drifted.
it('registers the runtime under the dedicated app_surface property', () => {
expect(ensurePosthogInitialized(true)).toBe(true)
expect(posthogMocks.register).toHaveBeenCalledWith({ app_surface: 'web' })
})
})
packages/stage-ui/src/stores/analytics/posthog.ts
+2 -2
View File
@@ -13,7 +13,7 @@ import {
let posthogInitialized = false
// All AIRI surfaces (web, desktop, mobile) capture into a single PostHog
// project. The platform is carried on every event via the `surface` super
// project. The platform is carried on every event via the `app_surface` super
// property (registered at init), so cross-platform funnels live in one
// project instead of being split across per-platform projects.
function currentSurface(): 'web' | 'mobile' | 'electron' {
@@ -43,7 +43,7 @@ export function ensurePosthogInitialized(enabled: boolean): boolean {
})
// Tag every event (including autocapture / pageview) with the platform so
// the single project can still be broken down by web / desktop / mobile.
posthog.register({ surface: currentSurface() })
posthog.register({ app_surface: currentSurface() })
posthogInitialized = true
return true
}
packages/stage-ui/src/stores/chat.contract.test.ts
+128 -15
View File
@@ -39,7 +39,26 @@ const ioTracerMocks = vi.hoisted(() => {
const llmStreamMock = vi.fn()
const trackFirstMessageMock = vi.fn()
const trackSecondTurnStartedMock = vi.fn()
const chatAnalyticsMocks = vi.hoisted(() => ({
trackAssistantResponseRendered: vi.fn(),
trackChatActivationFailed: vi.fn(),
trackChatActivationStarted: vi.fn(),
trackChatActivationSucceeded: vi.fn(),
trackLlmFirstToken: vi.fn(),
trackLlmRequestStarted: vi.fn(),
trackMessageRound: vi.fn(),
trackMessageRoundFailed: vi.fn(),
trackMessageSendStarted: vi.fn(),
trackMessageSent: vi.fn(),
trackSecondTurnStarted: vi.fn(),
}))
const trackSecondTurnStartedMock = chatAnalyticsMocks.trackSecondTurnStarted
const redundantChatAnalyticsMocks = vi.hoisted(() => ({
trackAssistantResponseCompleted: vi.fn(),
trackChatFailed: vi.fn(),
trackChatStarted: vi.fn(),
trackFeatureUsed: vi.fn(),
}))
const ingestContextMessageMock = vi.fn()
const getContextsSnapshotMock = vi.fn()
const createMinecraftContextMock = vi.fn()
@@ -65,19 +84,20 @@ vi.mock('pinia', async () => {
vi.mock('../composables', () => ({
useAnalytics: () => ({
trackFirstMessage: trackFirstMessageMock,
trackChatFailed: vi.fn(),
trackChatStarted: vi.fn(),
trackMessageSendStarted: vi.fn(),
trackMessageSent: vi.fn(),
trackLlmRequestStarted: vi.fn(),
trackLlmFirstToken: vi.fn(),
trackAssistantResponseRendered: vi.fn(),
trackAssistantResponseCompleted: vi.fn(),
trackMessageRound: vi.fn(),
trackFeatureUsed: vi.fn(),
trackChatActivationStarted: vi.fn(),
trackChatActivationSucceeded: vi.fn(),
trackChatActivationFailed: vi.fn(),
trackChatFailed: redundantChatAnalyticsMocks.trackChatFailed,
trackChatStarted: redundantChatAnalyticsMocks.trackChatStarted,
trackMessageSendStarted: chatAnalyticsMocks.trackMessageSendStarted,
trackMessageSent: chatAnalyticsMocks.trackMessageSent,
trackLlmRequestStarted: chatAnalyticsMocks.trackLlmRequestStarted,
trackLlmFirstToken: chatAnalyticsMocks.trackLlmFirstToken,
trackAssistantResponseRendered: chatAnalyticsMocks.trackAssistantResponseRendered,
trackAssistantResponseCompleted: redundantChatAnalyticsMocks.trackAssistantResponseCompleted,
trackMessageRound: chatAnalyticsMocks.trackMessageRound,
trackMessageRoundFailed: chatAnalyticsMocks.trackMessageRoundFailed,
trackFeatureUsed: redundantChatAnalyticsMocks.trackFeatureUsed,
trackChatActivationStarted: chatAnalyticsMocks.trackChatActivationStarted,
trackChatActivationSucceeded: chatAnalyticsMocks.trackChatActivationSucceeded,
trackChatActivationFailed: chatAnalyticsMocks.trackChatActivationFailed,
trackSecondTurnStarted: trackSecondTurnStartedMock,
}),
}))
@@ -166,7 +186,12 @@ describe('chat orchestrator contract', () => {
setActivePinia(createPinia())
llmStreamMock.mockReset()
trackFirstMessageMock.mockReset()
trackSecondTurnStartedMock.mockReset()
for (const analyticsMock of Object.values(chatAnalyticsMocks))
analyticsMock.mockReset()
redundantChatAnalyticsMocks.trackAssistantResponseCompleted.mockReset()
redundantChatAnalyticsMocks.trackChatFailed.mockReset()
redundantChatAnalyticsMocks.trackChatStarted.mockReset()
redundantChatAnalyticsMocks.trackFeatureUsed.mockReset()
ingestContextMessageMock.mockReset()
getContextsSnapshotMock.mockReset()
getContextsSnapshotMock.mockReturnValue({})
@@ -190,6 +215,39 @@ describe('chat orchestrator contract', () => {
sessionMessages['session-1'] = [{ role: 'system', content: 'system prompt', createdAt: 1, id: 'system' }]
})
it('forwards one correlation identity across every PostHog chat milestone', async () => {
llmStreamMock.mockImplementation(async (_model: string, _chatProvider: ChatProvider, _messages: Message[], options: any) => {
await options.onStreamEvent({ type: 'text-delta', text: 'ok' })
await options.onStreamEvent({ type: 'finish', finishReason: 'stop' })
})
const store = useChatOrchestratorStore()
await store.ingest('hello', {
model: 'gpt-test',
chatProvider: provider,
})
const messageProperties = chatAnalyticsMocks.trackMessageSent.mock.calls[0]?.[0]
expect(messageProperties).toMatchObject({
conversation_id: 'session-1',
round_id: messageProperties.message_id,
turn_index: 1,
})
const correlation = {
conversation_id: 'session-1',
round_id: messageProperties.round_id,
turn_index: 1,
}
expect(chatAnalyticsMocks.trackMessageSendStarted).toHaveBeenCalledWith(expect.objectContaining(correlation))
expect(chatAnalyticsMocks.trackLlmRequestStarted).toHaveBeenCalledWith(expect.objectContaining(correlation))
expect(chatAnalyticsMocks.trackLlmFirstToken).toHaveBeenCalledWith(expect.objectContaining(correlation))
expect(chatAnalyticsMocks.trackAssistantResponseRendered).toHaveBeenCalledWith(expect.objectContaining(correlation))
expect(chatAnalyticsMocks.trackMessageRound).toHaveBeenCalledWith(expect.objectContaining(correlation))
expect(chatAnalyticsMocks.trackChatActivationStarted).toHaveBeenCalledWith(expect.objectContaining(correlation))
expect(chatAnalyticsMocks.trackChatActivationSucceeded).toHaveBeenCalledWith(expect.objectContaining(correlation))
})
it('emits second turn analytics from chat sends', async () => {
activeProviderRef.value = 'official-provider'
llmStreamMock.mockImplementation(async (_model: string, _chatProvider: ChatProvider, _messages: Message[], options: any) => {
@@ -210,9 +268,64 @@ describe('chat orchestrator contract', () => {
expect(trackSecondTurnStartedMock).toHaveBeenCalledTimes(1)
expect(trackSecondTurnStartedMock).toHaveBeenCalledWith({
conversation_id: 'session-1',
provider_id: 'official-provider',
provider_mode: 'official',
model_id: 'chat-auto',
round_id: expect.any(String),
source: 'text',
turn_index: 2,
})
})
// ROOT CAUSE:
//
// One successful send emitted both the canonical message/latency events
// and four generic aliases, multiplying PostHog volume without adding a
// distinct product decision.
it('does not emit redundant generic chat aliases for a successful send', async () => {
llmStreamMock.mockImplementation(async (_model: string, _chatProvider: ChatProvider, _messages: Message[], options: any) => {
await options.onStreamEvent({ type: 'text-delta', text: 'ok' })
await options.onStreamEvent({ type: 'finish', finishReason: 'stop' })
})
const store = useChatOrchestratorStore()
await store.ingest('hello', {
model: 'gpt-test',
chatProvider: provider,
})
expect(redundantChatAnalyticsMocks.trackChatStarted).not.toHaveBeenCalled()
expect(redundantChatAnalyticsMocks.trackAssistantResponseCompleted).not.toHaveBeenCalled()
expect(redundantChatAnalyticsMocks.trackChatFailed).not.toHaveBeenCalled()
expect(redundantChatAnalyticsMocks.trackFeatureUsed).not.toHaveBeenCalled()
})
it('forwards later-turn failures to the canonical round failure event', async () => {
llmStreamMock.mockImplementationOnce(async (_model: string, _chatProvider: ChatProvider, _messages: Message[], options: any) => {
await options.onStreamEvent({ type: 'text-delta', text: 'ok' })
await options.onStreamEvent({ type: 'finish', finishReason: 'stop' })
})
llmStreamMock.mockRejectedValueOnce(new Error('later turn rejected'))
const store = useChatOrchestratorStore()
await store.ingest('first turn', {
model: 'gpt-test',
chatProvider: provider,
})
await expect(store.ingest('second turn', {
model: 'gpt-test',
chatProvider: provider,
})).rejects.toThrow('later turn rejected')
expect(chatAnalyticsMocks.trackChatActivationFailed).not.toHaveBeenCalled()
expect(chatAnalyticsMocks.trackMessageRoundFailed).toHaveBeenCalledWith({
conversation_id: 'session-1',
error_code: 'llm_response_failed',
failure_stage: 'llm_response',
model_id: 'gpt-test',
provider_id: 'mock-provider',
round_id: expect.any(String),
source: 'text',
turn_index: 2,
})
packages/stage-ui/src/stores/chat.ts
+48 -41
View File
@@ -51,16 +51,13 @@ export const useChatOrchestratorStore = defineStore('chat-orchestrator', () => {
const { activeModel, activeProvider } = storeToRefs(consciousnessStore)
const {
trackFirstMessage,
trackChatFailed,
trackChatStarted,
trackMessageSendStarted,
trackMessageSent,
trackLlmRequestStarted,
trackLlmFirstToken,
trackAssistantResponseRendered,
trackAssistantResponseCompleted,
trackMessageRound,
trackFeatureUsed,
trackMessageRoundFailed,
trackChatActivationStarted,
trackChatActivationSucceeded,
trackChatActivationFailed,
@@ -185,109 +182,119 @@ export const useChatOrchestratorStore = defineStore('chat-orchestrator', () => {
onStateChange: syncRuntimeState,
onSendSettled: settleOwnedActiveTurnSpan,
onTrackFirstMessage: trackFirstMessage,
onMessageSendStarted: ({ source, model }) => {
onMessageSendStarted: ({ conversationId, roundId, turnIndex, source, model }) => {
lastSendSource = source
trackMessageSendStarted({
conversation_id: conversationId,
round_id: roundId,
turn_index: turnIndex,
source,
model,
})
trackChatStarted({
conversation_id: activeSessionId.value || 'unknown',
provider_type: providerMode(activeProvider.value),
provider_name: activeProvider.value || 'unknown',
model: model || 'unknown',
entry: 'chat',
})
},
onLlmRequestStarted: ({ model, provider, hasVoice }) => trackLlmRequestStarted({
onLlmRequestStarted: ({ conversationId, roundId, turnIndex, model, provider, hasVoice }) => trackLlmRequestStarted({
conversation_id: conversationId,
round_id: roundId,
turn_index: turnIndex,
model,
provider,
has_voice: hasVoice,
}),
onLlmFirstToken: ({ model, ttfbMs }) => trackLlmFirstToken({
onLlmFirstToken: ({ conversationId, roundId, turnIndex, model, ttfbMs }) => trackLlmFirstToken({
conversation_id: conversationId,
round_id: roundId,
turn_index: turnIndex,
model,
ttfb_ms: ttfbMs,
}),
onAssistantResponseRendered: ({ model, latencyMs }) => {
onAssistantResponseRendered: ({ conversationId, roundId, turnIndex, model, latencyMs }) => {
trackAssistantResponseRendered({
conversation_id: conversationId,
round_id: roundId,
turn_index: turnIndex,
model,
latency_ms: latencyMs,
})
trackAssistantResponseCompleted({
conversation_id: activeSessionId.value || 'unknown',
provider_type: providerMode(activeProvider.value),
provider_name: activeProvider.value || 'unknown',
model: model || 'unknown',
latency_ms: latencyMs,
})
},
onMessageRound: ({ durationMs, hasVoice, model }) => trackMessageRound({
onMessageRound: ({ conversationId, roundId, turnIndex, durationMs, hasVoice, model }) => trackMessageRound({
conversation_id: conversationId,
round_id: roundId,
turn_index: turnIndex,
duration_ms: durationMs,
has_voice: hasVoice,
model,
}),
onChatActivationStarted: ({ model, provider, source }) => {
onMessageRoundFailed: ({ conversationId, roundId, turnIndex, model, provider, errorCode, failureStage, source }) => trackMessageRoundFailed({
conversation_id: conversationId,
round_id: roundId,
turn_index: turnIndex,
provider_id: provider || 'unknown',
model_id: model || 'unknown',
source,
error_code: errorCode,
failure_stage: failureStage,
}),
onChatActivationStarted: ({ conversationId, roundId, turnIndex, model, provider, source }) => {
const mode = providerMode(provider)
const providerId = provider || 'unknown'
const modelId = model || 'unknown'
trackChatActivationStarted({
conversation_id: conversationId,
provider_mode: mode,
provider_id: providerId,
model_id: modelId,
round_id: roundId,
source,
turn_index: turnIndex,
})
},
onChatActivationSucceeded: ({ model, provider, durationMs, source }) => trackChatActivationSucceeded({
onChatActivationSucceeded: ({ conversationId, roundId, turnIndex, model, provider, durationMs, source }) => trackChatActivationSucceeded({
conversation_id: conversationId,
provider_mode: providerMode(provider),
provider_id: provider || 'unknown',
model_id: model || 'unknown',
round_id: roundId,
time_to_first_message_ms: durationMs,
source,
turn_index: turnIndex,
}),
onChatActivationFailed: ({ model, provider, errorCode, failureStage, source }) => {
onChatActivationFailed: ({ conversationId, roundId, turnIndex, model, provider, errorCode, failureStage, source }) => {
trackChatActivationFailed({
conversation_id: conversationId,
provider_mode: providerMode(provider),
provider_id: provider || 'unknown',
model_id: model || 'unknown',
round_id: roundId,
error_code: errorCode,
failure_stage: failureStage,
source,
})
trackChatFailed({
conversation_id: activeSessionId.value || 'unknown',
provider_type: providerMode(provider),
provider_name: provider || 'unknown',
model: model || 'unknown',
failure_stage: failureStage,
error_code: errorCode,
turn_index: turnIndex,
})
},
onLifecycle: record => contextObservability.recordLifecycle(record),
onPromptProjection: payload => contextObservability.capturePromptProjection(payload),
onUserMessageAppended: ({ sessionId, message, messageText, source, model, provider, turnIndex }) => {
onUserMessageAppended: ({ sessionId, message, messageText, source, model, provider, roundId, turnIndex }) => {
trackMessageSent({
conversation_id: sessionId,
provider_type: providerMode(activeProvider.value),
provider_name: activeProvider.value || 'unknown',
model: activeModel.value || 'unknown',
message_id: message.id,
round_id: roundId,
turn_index: turnIndex,
message_index: chatSession.getSessionMessages(sessionId).length,
message_length: messageText.length,
has_attachment: false,
mode: lastSendSource,
})
trackFeatureUsed({
feature_name: 'chat',
business_domain: 'conversation',
entry: 'chat',
success: true,
})
if (turnIndex === 2) {
trackSecondTurnStarted({
conversation_id: sessionId,
provider_mode: providerMode(provider),
provider_id: provider || 'unknown',
model_id: model || 'unknown',
round_id: roundId,
source,
turn_index: turnIndex,
})
packages/stage-ui/src/stores/providers.ts
+2 -2
View File
@@ -132,7 +132,7 @@ function trackModelListLoaded(properties: {
capturePosthogEvent('model_list_loaded', {
...properties,
surface: analyticsSurface(),
app_surface: analyticsSurface(),
})
}
@@ -150,7 +150,7 @@ function trackModelListFailed(properties: {
capturePosthogEvent('model_list_failed', {
...properties,
surface: analyticsSurface(),
app_surface: analyticsSurface(),
})
}