feat(server): track blocked TTS preflight
This commit is contained in:
+18
-1
@@ -209,7 +209,10 @@ export async function buildApp(deps: AppDeps) {
|
|||||||
if (!session?.user)
|
if (!session?.user)
|
||||||
return createUnauthorizedWsEvents()
|
return createUnauthorizedWsEvents()
|
||||||
|
|
||||||
return audioSpeechWsSetup(session.user.id)
|
return audioSpeechWsSetup(session.user.id, {
|
||||||
|
trigger: c.req.query('tts_trigger') === 'auto' ? 'auto' : 'manual',
|
||||||
|
source: parseTtsSource(c.req.query('tts_source'), 'audio.speech.ws'),
|
||||||
|
})
|
||||||
}))
|
}))
|
||||||
|
|
||||||
// Cross-instance config invalidation. The subscriber owns its own
|
// Cross-instance config invalidation. The subscriber owns its own
|
||||||
@@ -429,6 +432,20 @@ export async function buildApp(deps: AppDeps) {
|
|||||||
return { app: builtApp, injectWebSocket }
|
return { app: builtApp, injectWebSocket }
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function parseTtsSource(
|
||||||
|
value: string | undefined,
|
||||||
|
fallback: 'audio.speech.ws',
|
||||||
|
): 'audio.speech.ws' | 'chat_auto_tts' | 'manual_preview' | 'settings_test' {
|
||||||
|
switch (value) {
|
||||||
|
case 'chat_auto_tts':
|
||||||
|
case 'manual_preview':
|
||||||
|
case 'settings_test':
|
||||||
|
return value
|
||||||
|
default:
|
||||||
|
return fallback
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
export type AppType = Awaited<ReturnType<typeof buildApp>>['app']
|
export type AppType = Awaited<ReturnType<typeof buildApp>>['app']
|
||||||
|
|
||||||
export async function createApp() {
|
export async function createApp() {
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
import type { WSEvents } from 'hono/ws'
|
import type { WSEvents } from 'hono/ws'
|
||||||
|
|
||||||
|
import type { AudioSpeechSessionAnalytics } from './session'
|
||||||
import type { AudioSpeechWsHandlersOptions } from './types'
|
import type { AudioSpeechWsHandlersOptions } from './types'
|
||||||
|
|
||||||
import { useLogger } from '@guiiai/logg'
|
import { useLogger } from '@guiiai/logg'
|
||||||
@@ -31,8 +32,8 @@ export type { AudioSpeechWsHandlersOptions } from './types'
|
|||||||
* peer registry because streaming TTS is single-session per connection.
|
* peer registry because streaming TTS is single-session per connection.
|
||||||
*/
|
*/
|
||||||
export function createAudioSpeechWsHandlers(opts: AudioSpeechWsHandlersOptions) {
|
export function createAudioSpeechWsHandlers(opts: AudioSpeechWsHandlersOptions) {
|
||||||
return function setupPeer(userId: string): WSEvents {
|
return function setupPeer(userId: string, analytics?: AudioSpeechSessionAnalytics): WSEvents {
|
||||||
const sessionState = createSessionState(userId, opts)
|
const sessionState = createSessionState(userId, opts, analytics)
|
||||||
|
|
||||||
return {
|
return {
|
||||||
onOpen(_event, ws) {
|
onOpen(_event, ws) {
|
||||||
|
|||||||
@@ -152,7 +152,10 @@ function makeFakeDeps(overrides: {
|
|||||||
decryptedKey?: string
|
decryptedKey?: string
|
||||||
}) {
|
}) {
|
||||||
const ttsMeter = {
|
const ttsMeter = {
|
||||||
assertCanAfford: vi.fn(async () => undefined),
|
assertCanAfford: vi.fn(async (_userId: string, _newUnits: number, currentBalance: number) => {
|
||||||
|
if (currentBalance <= 0)
|
||||||
|
throw Object.assign(new Error('Insufficient flux'), { statusCode: 402 })
|
||||||
|
}),
|
||||||
accumulate: vi.fn(async () => ({
|
accumulate: vi.fn(async () => ({
|
||||||
fluxDebited: 1,
|
fluxDebited: 1,
|
||||||
debtAfter: 0,
|
debtAfter: 0,
|
||||||
@@ -282,7 +285,7 @@ describe('audio-speech-ws route', () => {
|
|||||||
upstream = await startMockUpstream([])
|
upstream = await startMockUpstream([])
|
||||||
const deps = makeFakeDeps({ upstreamURL: upstream.url, fluxBalance: 0 })
|
const deps = makeFakeDeps({ upstreamURL: upstream.url, fluxBalance: 0 })
|
||||||
const handlers = createAudioSpeechWsHandlers(deps as any)
|
const handlers = createAudioSpeechWsHandlers(deps as any)
|
||||||
const events = handlers('user-broke')
|
const events = handlers('user-broke', { trigger: 'auto', source: 'chat_auto_tts' })
|
||||||
const client = makeMockClientWs()
|
const client = makeMockClientWs()
|
||||||
|
|
||||||
await driveClientSession(events, client, [])
|
await driveClientSession(events, client, [])
|
||||||
@@ -299,6 +302,18 @@ describe('audio-speech-ws route', () => {
|
|||||||
})
|
})
|
||||||
expect(client.closed).toBe(true)
|
expect(client.closed).toBe(true)
|
||||||
expect(client.closeCode).toBe(1008)
|
expect(client.closeCode).toBe(1008)
|
||||||
|
expect(deps.productEventService.track).toHaveBeenCalledWith(expect.objectContaining({
|
||||||
|
userId: 'user-broke',
|
||||||
|
feature: 'tts',
|
||||||
|
action: 'speech_blocked',
|
||||||
|
status: 'blocked',
|
||||||
|
source: 'chat_auto_tts',
|
||||||
|
reason: 'insufficient_balance',
|
||||||
|
metadata: expect.objectContaining({
|
||||||
|
trigger: 'auto',
|
||||||
|
balance_state: 'insufficient',
|
||||||
|
}),
|
||||||
|
}))
|
||||||
})
|
})
|
||||||
|
|
||||||
it('refuses with streaming_tts_not_configured when UNSPEECH_UPSTREAM.streaming is empty', async () => {
|
it('refuses with streaming_tts_not_configured when UNSPEECH_UPSTREAM.streaming is empty', async () => {
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ import WebSocket from 'ws'
|
|||||||
import { useLogger } from '@guiiai/logg'
|
import { useLogger } from '@guiiai/logg'
|
||||||
import { context as otelContext, SpanStatusCode, trace } from '@opentelemetry/api'
|
import { context as otelContext, SpanStatusCode, trace } from '@opentelemetry/api'
|
||||||
|
|
||||||
|
import { ApiError } from '../../utils/error'
|
||||||
import { nanoid } from '../../utils/id'
|
import { nanoid } from '../../utils/id'
|
||||||
import {
|
import {
|
||||||
AIRI_ATTR_BILLING_FLUX_CONSUMED,
|
AIRI_ATTR_BILLING_FLUX_CONSUMED,
|
||||||
@@ -49,6 +50,14 @@ export interface AudioSpeechSessionState {
|
|||||||
handleClientClose: () => void
|
handleClientClose: () => void
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export type StreamingTtsTrigger = 'auto' | 'manual'
|
||||||
|
export type StreamingTtsSource = 'audio.speech.ws' | 'chat_auto_tts' | 'manual_preview' | 'settings_test'
|
||||||
|
|
||||||
|
export interface AudioSpeechSessionAnalytics {
|
||||||
|
trigger?: StreamingTtsTrigger
|
||||||
|
source?: StreamingTtsSource
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Creates the per-connection streaming speech state machine.
|
* Creates the per-connection streaming speech state machine.
|
||||||
*
|
*
|
||||||
@@ -63,9 +72,14 @@ export interface AudioSpeechSessionState {
|
|||||||
* Returns:
|
* Returns:
|
||||||
* - A connection-scoped state object with no global peer registry.
|
* - A connection-scoped state object with no global peer registry.
|
||||||
*/
|
*/
|
||||||
export function createSessionState(userId: string, opts: AudioSpeechWsHandlersOptions): AudioSpeechSessionState {
|
export function createSessionState(
|
||||||
|
userId: string,
|
||||||
|
opts: AudioSpeechWsHandlersOptions,
|
||||||
|
analyticsInput: AudioSpeechSessionAnalytics = {},
|
||||||
|
): AudioSpeechSessionState {
|
||||||
const requestId = nanoid()
|
const requestId = nanoid()
|
||||||
const startedAt = Date.now()
|
const startedAt = Date.now()
|
||||||
|
const analytics = normalizeAnalytics(analyticsInput)
|
||||||
const span = tracer.startSpan('llm.gateway.tts.stream', {
|
const span = tracer.startSpan('llm.gateway.tts.stream', {
|
||||||
attributes: {
|
attributes: {
|
||||||
[AIRI_ATTR_GEN_AI_OPERATION_KIND]: 'text_to_speech_stream',
|
[AIRI_ATTR_GEN_AI_OPERATION_KIND]: 'text_to_speech_stream',
|
||||||
@@ -96,8 +110,11 @@ export function createSessionState(userId: string, opts: AudioSpeechWsHandlersOp
|
|||||||
feature: 'tts',
|
feature: 'tts',
|
||||||
action: 'speech_requested',
|
action: 'speech_requested',
|
||||||
status: 'started',
|
status: 'started',
|
||||||
source: 'audio.speech.ws',
|
source: analytics.source,
|
||||||
model: modelLabel,
|
model: modelLabel,
|
||||||
|
metadata: {
|
||||||
|
trigger: analytics.trigger,
|
||||||
|
},
|
||||||
})
|
})
|
||||||
|
|
||||||
let unspeech: Awaited<ReturnType<AudioSpeechWsHandlersOptions['configKV']['getOptional']>>
|
let unspeech: Awaited<ReturnType<AudioSpeechWsHandlersOptions['configKV']['getOptional']>>
|
||||||
@@ -120,10 +137,6 @@ export function createSessionState(userId: string, opts: AudioSpeechWsHandlersOp
|
|||||||
// afford the worst-case session.
|
// afford the worst-case session.
|
||||||
try {
|
try {
|
||||||
const flux = await opts.fluxService.getFlux(userId)
|
const flux = await opts.fluxService.getFlux(userId)
|
||||||
if (flux.flux <= 0) {
|
|
||||||
closeWithError(1008, 'insufficient_flux')
|
|
||||||
return
|
|
||||||
}
|
|
||||||
await opts.ttsMeter.assertCanAfford(userId, STREAMING_PREFLIGHT_CHARS_ESTIMATE, flux.flux)
|
await opts.ttsMeter.assertCanAfford(userId, STREAMING_PREFLIGHT_CHARS_ESTIMATE, flux.flux)
|
||||||
}
|
}
|
||||||
catch (err) {
|
catch (err) {
|
||||||
@@ -131,7 +144,10 @@ export function createSessionState(userId: string, opts: AudioSpeechWsHandlersOp
|
|||||||
// assertCanAfford throws PaymentRequiredError (402) — translate to ws
|
// assertCanAfford throws PaymentRequiredError (402) — translate to ws
|
||||||
// policy-violation close. The client can read the close code/reason to
|
// policy-violation close. The client can read the close code/reason to
|
||||||
// surface a 'top up' prompt.
|
// surface a 'top up' prompt.
|
||||||
closeWithError(1008, 'insufficient_flux')
|
if (isPaymentRequiredError(err))
|
||||||
|
closeWithBlockedPreflight(1008, 'insufficient_flux')
|
||||||
|
else
|
||||||
|
closeWithError(1011, 'flux_preflight_failed')
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -205,11 +221,12 @@ export function createSessionState(userId: string, opts: AudioSpeechWsHandlersOp
|
|||||||
feature: 'tts',
|
feature: 'tts',
|
||||||
action: 'speech_failed',
|
action: 'speech_failed',
|
||||||
status: 'failed',
|
status: 'failed',
|
||||||
source: 'audio.speech.ws',
|
source: analytics.source,
|
||||||
model: modelLabel,
|
model: modelLabel,
|
||||||
reason: 'upstream_error',
|
reason: 'upstream_error',
|
||||||
metadata: {
|
metadata: {
|
||||||
duration_ms: Date.now() - startedAt,
|
duration_ms: Date.now() - startedAt,
|
||||||
|
trigger: analytics.trigger,
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
try {
|
try {
|
||||||
@@ -409,12 +426,13 @@ export function createSessionState(userId: string, opts: AudioSpeechWsHandlersOp
|
|||||||
feature: 'tts',
|
feature: 'tts',
|
||||||
action: 'speech_succeeded',
|
action: 'speech_succeeded',
|
||||||
status: 'succeeded',
|
status: 'succeeded',
|
||||||
source: 'audio.speech.ws',
|
source: analytics.source,
|
||||||
model: modelLabel,
|
model: modelLabel,
|
||||||
metadata: {
|
metadata: {
|
||||||
input_chars: units,
|
input_chars: units,
|
||||||
duration_ms: durationMs,
|
duration_ms: durationMs,
|
||||||
flux_consumed: fluxConsumed,
|
flux_consumed: fluxConsumed,
|
||||||
|
trigger: analytics.trigger,
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
|
|
||||||
@@ -445,12 +463,46 @@ export function createSessionState(userId: string, opts: AudioSpeechWsHandlersOp
|
|||||||
feature: 'tts',
|
feature: 'tts',
|
||||||
action: 'speech_failed',
|
action: 'speech_failed',
|
||||||
status: 'failed',
|
status: 'failed',
|
||||||
source: 'audio.speech.ws',
|
source: analytics.source,
|
||||||
model: modelLabel,
|
model: modelLabel,
|
||||||
reason,
|
reason,
|
||||||
metadata: {
|
metadata: {
|
||||||
close_code: code,
|
close_code: code,
|
||||||
duration_ms: Date.now() - startedAt,
|
duration_ms: Date.now() - startedAt,
|
||||||
|
trigger: analytics.trigger,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if (clientWs) {
|
||||||
|
try {
|
||||||
|
clientWs.send(JSON.stringify({ event: 'error', code: reason, message: reason }))
|
||||||
|
}
|
||||||
|
catch {}
|
||||||
|
try {
|
||||||
|
clientWs.close(code, reason)
|
||||||
|
}
|
||||||
|
catch {}
|
||||||
|
}
|
||||||
|
closed = true
|
||||||
|
span.end()
|
||||||
|
}
|
||||||
|
|
||||||
|
function closeWithBlockedPreflight(code: number, reason: string) {
|
||||||
|
if (closed)
|
||||||
|
return
|
||||||
|
void opts.productEventService.track({
|
||||||
|
userId,
|
||||||
|
feature: 'tts',
|
||||||
|
action: 'speech_blocked',
|
||||||
|
status: 'blocked',
|
||||||
|
source: analytics.source,
|
||||||
|
model: modelLabel,
|
||||||
|
reason: 'insufficient_balance',
|
||||||
|
metadata: {
|
||||||
|
balance_state: 'insufficient',
|
||||||
|
billing_units: STREAMING_PREFLIGHT_CHARS_ESTIMATE,
|
||||||
|
close_code: code,
|
||||||
|
duration_ms: Date.now() - startedAt,
|
||||||
|
trigger: analytics.trigger,
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
if (clientWs) {
|
if (clientWs) {
|
||||||
@@ -474,3 +526,35 @@ export function createSessionState(userId: string, opts: AudioSpeechWsHandlersOp
|
|||||||
handleClientClose,
|
handleClientClose,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function normalizeAnalytics(input: AudioSpeechSessionAnalytics): Required<AudioSpeechSessionAnalytics> {
|
||||||
|
return {
|
||||||
|
trigger: normalizeTrigger(input.trigger),
|
||||||
|
source: normalizeSource(input.source),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizeTrigger(trigger: AudioSpeechSessionAnalytics['trigger']): StreamingTtsTrigger {
|
||||||
|
return trigger === 'auto' ? 'auto' : 'manual'
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizeSource(source: AudioSpeechSessionAnalytics['source']): StreamingTtsSource {
|
||||||
|
switch (source) {
|
||||||
|
case 'audio.speech.ws':
|
||||||
|
case 'chat_auto_tts':
|
||||||
|
case 'manual_preview':
|
||||||
|
case 'settings_test':
|
||||||
|
return source
|
||||||
|
default:
|
||||||
|
return 'audio.speech.ws'
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function isPaymentRequiredError(err: unknown): boolean {
|
||||||
|
if (err instanceof ApiError)
|
||||||
|
return err.statusCode === 402
|
||||||
|
return typeof err === 'object'
|
||||||
|
&& err != null
|
||||||
|
&& 'statusCode' in err
|
||||||
|
&& (err as { statusCode?: unknown }).statusCode === 402
|
||||||
|
}
|
||||||
|
|||||||
@@ -75,7 +75,12 @@ function createMockRequestLogService(): RequestLogService {
|
|||||||
function createMockTtsMeter(unitsPerFlux = 1000) {
|
function createMockTtsMeter(unitsPerFlux = 1000) {
|
||||||
let debt = 0
|
let debt = 0
|
||||||
return {
|
return {
|
||||||
assertCanAfford: vi.fn(async () => undefined),
|
assertCanAfford: vi.fn(async (_userId: string, newUnits: number, currentBalance: number) => {
|
||||||
|
const projectedFlux = Math.floor((debt + newUnits) / unitsPerFlux)
|
||||||
|
const required = Math.max(projectedFlux, currentBalance <= 0 ? 1 : 0)
|
||||||
|
if (currentBalance < required)
|
||||||
|
throw new ApiError(402, 'PAYMENT_REQUIRED', 'Insufficient flux')
|
||||||
|
}),
|
||||||
accumulate: vi.fn(async ({ units, currentBalance }: { units: number, currentBalance: number }) => {
|
accumulate: vi.fn(async ({ units, currentBalance }: { units: number, currentBalance: number }) => {
|
||||||
debt += units
|
debt += units
|
||||||
const fluxDebited = Math.floor(debt / unitsPerFlux)
|
const fluxDebited = Math.floor(debt / unitsPerFlux)
|
||||||
@@ -910,10 +915,18 @@ describe('v1CompletionsRoutes', () => {
|
|||||||
)
|
)
|
||||||
})
|
})
|
||||||
|
|
||||||
it('should return 402 when flux is insufficient', async () => {
|
it('returns 402 and records blocked event for manual TTS when flux is insufficient', async () => {
|
||||||
|
const productEventService = createMockProductEventService()
|
||||||
|
const llmRouter = createMockLlmRouter()
|
||||||
const app = createTestApp(
|
const app = createTestApp(
|
||||||
createMockFluxService(0),
|
createMockFluxService(0),
|
||||||
createMockConfigKV(),
|
createMockConfigKV(),
|
||||||
|
undefined,
|
||||||
|
undefined,
|
||||||
|
undefined,
|
||||||
|
llmRouter,
|
||||||
|
createMockLlmTracing(),
|
||||||
|
productEventService,
|
||||||
)
|
)
|
||||||
|
|
||||||
const res = await app.fetch(
|
const res = await app.fetch(
|
||||||
@@ -925,6 +938,63 @@ describe('v1CompletionsRoutes', () => {
|
|||||||
{ user: testUser } as any,
|
{ user: testUser } as any,
|
||||||
)
|
)
|
||||||
expect(res.status).toBe(402)
|
expect(res.status).toBe(402)
|
||||||
|
expect(llmRouter.routeTts).not.toHaveBeenCalled()
|
||||||
|
expect(productEventService.track).toHaveBeenCalledWith(expect.objectContaining({
|
||||||
|
action: 'speech_blocked',
|
||||||
|
status: 'blocked',
|
||||||
|
source: 'audio.speech',
|
||||||
|
reason: 'insufficient_balance',
|
||||||
|
metadata: expect.objectContaining({
|
||||||
|
trigger: 'manual',
|
||||||
|
balance_state: 'insufficient',
|
||||||
|
}),
|
||||||
|
}))
|
||||||
|
})
|
||||||
|
|
||||||
|
it('returns 204 and records blocked event for auto TTS when flux is insufficient', async () => {
|
||||||
|
const productEventService = createMockProductEventService()
|
||||||
|
const llmRouter = createMockLlmRouter()
|
||||||
|
const app = createTestApp(
|
||||||
|
createMockFluxService(0),
|
||||||
|
createMockConfigKV(),
|
||||||
|
undefined,
|
||||||
|
undefined,
|
||||||
|
undefined,
|
||||||
|
llmRouter,
|
||||||
|
createMockLlmTracing(),
|
||||||
|
productEventService,
|
||||||
|
)
|
||||||
|
|
||||||
|
const res = await app.fetch(
|
||||||
|
new Request('http://localhost/api/v1/audio/speech', {
|
||||||
|
method: 'POST',
|
||||||
|
headers: { 'Content-Type': 'application/json' },
|
||||||
|
body: JSON.stringify({
|
||||||
|
model: 'auto',
|
||||||
|
input: 'hello',
|
||||||
|
voice: 'alloy',
|
||||||
|
extra_body: {
|
||||||
|
airi_analytics: {
|
||||||
|
trigger: 'auto',
|
||||||
|
source: 'chat_auto_tts',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
}),
|
||||||
|
{ user: testUser } as any,
|
||||||
|
)
|
||||||
|
expect(res.status).toBe(204)
|
||||||
|
expect(llmRouter.routeTts).not.toHaveBeenCalled()
|
||||||
|
expect(productEventService.track).toHaveBeenCalledWith(expect.objectContaining({
|
||||||
|
action: 'speech_blocked',
|
||||||
|
status: 'blocked',
|
||||||
|
source: 'chat_auto_tts',
|
||||||
|
reason: 'insufficient_balance',
|
||||||
|
metadata: expect.objectContaining({
|
||||||
|
trigger: 'auto',
|
||||||
|
balance_state: 'insufficient',
|
||||||
|
}),
|
||||||
|
}))
|
||||||
})
|
})
|
||||||
|
|
||||||
it('should not charge when input is empty', async () => {
|
it('should not charge when input is empty', async () => {
|
||||||
|
|||||||
@@ -62,6 +62,13 @@ export interface OpenAiSpeechRequest {
|
|||||||
abortSignal?: AbortSignal
|
abortSignal?: AbortSignal
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type TtsTrigger = 'auto' | 'manual'
|
||||||
|
|
||||||
|
interface TtsAnalyticsContext {
|
||||||
|
trigger: TtsTrigger
|
||||||
|
source: 'audio.speech' | 'chat_auto_tts' | 'manual_preview' | 'settings_test'
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Runs the OpenAI-shaped text-to-speech gateway flow.
|
* Runs the OpenAI-shaped text-to-speech gateway flow.
|
||||||
*
|
*
|
||||||
@@ -83,6 +90,7 @@ export function createOpenAiSpeechService(deps: OpenAiSpeechServiceDeps) {
|
|||||||
const requestId = nanoid()
|
const requestId = nanoid()
|
||||||
let requestModel = typeof input.body.model === 'string' ? input.body.model : 'auto'
|
let requestModel = typeof input.body.model === 'string' ? input.body.model : 'auto'
|
||||||
const inputText = typeof input.body.input === 'string' ? input.body.input : ''
|
const inputText = typeof input.body.input === 'string' ? input.body.input : ''
|
||||||
|
const analytics = ttsAnalyticsContext(input.body)
|
||||||
|
|
||||||
if (requestModel === 'auto')
|
if (requestModel === 'auto')
|
||||||
requestModel = await deps.configKV.getOrThrow('DEFAULT_TTS_MODEL')
|
requestModel = await deps.configKV.getOrThrow('DEFAULT_TTS_MODEL')
|
||||||
@@ -107,17 +115,50 @@ export function createOpenAiSpeechService(deps: OpenAiSpeechServiceDeps) {
|
|||||||
feature: 'tts',
|
feature: 'tts',
|
||||||
action: 'speech_requested',
|
action: 'speech_requested',
|
||||||
status: 'started',
|
status: 'started',
|
||||||
source: 'audio.speech',
|
source: analytics.source,
|
||||||
model: requestModel,
|
model: requestModel,
|
||||||
metadata: {
|
metadata: {
|
||||||
input_chars: inputText.length,
|
input_chars: inputText.length,
|
||||||
|
trigger: analytics.trigger,
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
|
|
||||||
const flux = await deps.fluxService.getFlux(input.userId)
|
const flux = await deps.fluxService.getFlux(input.userId)
|
||||||
if (flux.flux <= 0)
|
try {
|
||||||
|
await deps.ttsMeter.assertCanAfford(input.userId, billingUnits, flux.flux)
|
||||||
|
}
|
||||||
|
catch (err) {
|
||||||
|
if (!(err instanceof ApiError) || err.statusCode !== 402)
|
||||||
|
throw err
|
||||||
|
|
||||||
|
void deps.productEventService.track({
|
||||||
|
userId: input.userId,
|
||||||
|
feature: 'tts',
|
||||||
|
action: 'speech_blocked',
|
||||||
|
status: 'blocked',
|
||||||
|
source: analytics.source,
|
||||||
|
model: requestModel,
|
||||||
|
reason: 'insufficient_balance',
|
||||||
|
metadata: {
|
||||||
|
input_chars: inputText.length,
|
||||||
|
billing_units: billingUnits,
|
||||||
|
balance_state: 'insufficient',
|
||||||
|
trigger: analytics.trigger,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
logger.withError(err).withFields({
|
||||||
|
requestId,
|
||||||
|
userId: input.userId,
|
||||||
|
model: requestModel,
|
||||||
|
trigger: analytics.trigger,
|
||||||
|
source: analytics.source,
|
||||||
|
}).warn('tts speech blocked by pre-flight balance check')
|
||||||
|
|
||||||
|
if (analytics.trigger === 'auto')
|
||||||
|
return new Response(null, { status: 204 })
|
||||||
|
|
||||||
throw createPaymentRequiredError('Insufficient flux')
|
throw createPaymentRequiredError('Insufficient flux')
|
||||||
await deps.ttsMeter.assertCanAfford(input.userId, billingUnits, flux.flux)
|
}
|
||||||
|
|
||||||
const ttsInput = {
|
const ttsInput = {
|
||||||
text: inputText,
|
text: inputText,
|
||||||
@@ -170,13 +211,14 @@ export function createOpenAiSpeechService(deps: OpenAiSpeechServiceDeps) {
|
|||||||
feature: 'tts',
|
feature: 'tts',
|
||||||
action: 'speech_failed',
|
action: 'speech_failed',
|
||||||
status: 'failed',
|
status: 'failed',
|
||||||
source: 'audio.speech',
|
source: analytics.source,
|
||||||
model: requestModel,
|
model: requestModel,
|
||||||
provider: routeCtx.provider,
|
provider: routeCtx.provider,
|
||||||
reason: failure.reason,
|
reason: failure.reason,
|
||||||
metadata: {
|
metadata: {
|
||||||
http_status: failure.status,
|
http_status: failure.status,
|
||||||
duration_ms: Date.now() - startedAt,
|
duration_ms: Date.now() - startedAt,
|
||||||
|
trigger: analytics.trigger,
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
throw err
|
throw err
|
||||||
@@ -195,13 +237,14 @@ export function createOpenAiSpeechService(deps: OpenAiSpeechServiceDeps) {
|
|||||||
feature: 'tts',
|
feature: 'tts',
|
||||||
action: 'speech_failed',
|
action: 'speech_failed',
|
||||||
status: 'failed',
|
status: 'failed',
|
||||||
source: 'audio.speech',
|
source: analytics.source,
|
||||||
model: requestModel,
|
model: requestModel,
|
||||||
provider: routeCtx.provider,
|
provider: routeCtx.provider,
|
||||||
reason: 'upstream_error',
|
reason: 'upstream_error',
|
||||||
metadata: {
|
metadata: {
|
||||||
http_status: response.status,
|
http_status: response.status,
|
||||||
duration_ms: durationMs,
|
duration_ms: durationMs,
|
||||||
|
trigger: analytics.trigger,
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
logger.withFields({ requestId, userId: input.userId, model: requestModel, status: response.status, durationMs })
|
logger.withFields({ requestId, userId: input.userId, model: requestModel, status: response.status, durationMs })
|
||||||
@@ -243,7 +286,7 @@ export function createOpenAiSpeechService(deps: OpenAiSpeechServiceDeps) {
|
|||||||
feature: 'tts',
|
feature: 'tts',
|
||||||
action: 'speech_succeeded',
|
action: 'speech_succeeded',
|
||||||
status: 'succeeded',
|
status: 'succeeded',
|
||||||
source: 'audio.speech',
|
source: analytics.source,
|
||||||
model: requestModel,
|
model: requestModel,
|
||||||
provider: routeCtx.provider,
|
provider: routeCtx.provider,
|
||||||
metadata: {
|
metadata: {
|
||||||
@@ -253,6 +296,7 @@ export function createOpenAiSpeechService(deps: OpenAiSpeechServiceDeps) {
|
|||||||
cost_multiplier: voicePackRequest.costMultiplier,
|
cost_multiplier: voicePackRequest.costMultiplier,
|
||||||
duration_ms: durationMs,
|
duration_ms: durationMs,
|
||||||
flux_consumed: fluxConsumed,
|
flux_consumed: fluxConsumed,
|
||||||
|
trigger: analytics.trigger,
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
deps.requestLogService.logRequest({
|
deps.requestLogService.logRequest({
|
||||||
@@ -300,6 +344,20 @@ export function createOpenAiSpeechService(deps: OpenAiSpeechServiceDeps) {
|
|||||||
return { handleSpeechRequest }
|
return { handleSpeechRequest }
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function ttsAnalyticsContext(body: Record<string, unknown>): TtsAnalyticsContext {
|
||||||
|
const extraBody = asRecord(body.extra_body)
|
||||||
|
const analytics = asRecord(extraBody?.airi_analytics)
|
||||||
|
const trigger = analytics?.trigger === 'auto' ? 'auto' : 'manual'
|
||||||
|
const rawSource = analytics?.source
|
||||||
|
const source = rawSource === 'chat_auto_tts'
|
||||||
|
|| rawSource === 'manual_preview'
|
||||||
|
|| rawSource === 'settings_test'
|
||||||
|
? rawSource
|
||||||
|
: 'audio.speech'
|
||||||
|
|
||||||
|
return { trigger, source }
|
||||||
|
}
|
||||||
|
|
||||||
async function voicePackRequestOptions(
|
async function voicePackRequestOptions(
|
||||||
body: Record<string, unknown>,
|
body: Record<string, unknown>,
|
||||||
context: {
|
context: {
|
||||||
|
|||||||
@@ -99,4 +99,38 @@ describe('productEventService', () => {
|
|||||||
distinctUsers: 2,
|
distinctUsers: 2,
|
||||||
}])
|
}])
|
||||||
})
|
})
|
||||||
|
|
||||||
|
it('writes blocked TTS events for server-side preflight decisions', async () => {
|
||||||
|
const events = { add: vi.fn() }
|
||||||
|
const service = createProductEventService(db, { events } as unknown as ProductMetrics)
|
||||||
|
|
||||||
|
await service.track({
|
||||||
|
userId: 'user-1',
|
||||||
|
feature: 'tts',
|
||||||
|
action: 'speech_blocked',
|
||||||
|
status: 'blocked',
|
||||||
|
source: 'chat_auto_tts',
|
||||||
|
reason: 'insufficient_balance',
|
||||||
|
metadata: {
|
||||||
|
trigger: 'auto',
|
||||||
|
balance_state: 'insufficient',
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
const rows = await db.select().from(schema.productEvents)
|
||||||
|
expect(rows).toHaveLength(1)
|
||||||
|
expect(rows[0]).toMatchObject({
|
||||||
|
feature: 'tts',
|
||||||
|
action: 'speech_blocked',
|
||||||
|
status: 'blocked',
|
||||||
|
source: 'chat_auto_tts',
|
||||||
|
reason: 'insufficient_balance',
|
||||||
|
})
|
||||||
|
expect(events.add).toHaveBeenCalledWith(1, {
|
||||||
|
feature: 'tts',
|
||||||
|
action: 'speech_blocked',
|
||||||
|
status: 'blocked',
|
||||||
|
source: 'chat_auto_tts',
|
||||||
|
})
|
||||||
|
})
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ const logger = useLogger('product-events')
|
|||||||
|
|
||||||
export type ProductFeature = 'auth' | 'chat' | 'gen_ai_chat' | 'tts' | 'billing' | 'voice_pack'
|
export type ProductFeature = 'auth' | 'chat' | 'gen_ai_chat' | 'tts' | 'billing' | 'voice_pack'
|
||||||
|
|
||||||
export type ProductEventStatus = 'started' | 'succeeded' | 'failed'
|
export type ProductEventStatus = 'started' | 'succeeded' | 'failed' | 'blocked'
|
||||||
|
|
||||||
export type ProductAction
|
export type ProductAction
|
||||||
= | 'user_signed_up'
|
= | 'user_signed_up'
|
||||||
@@ -23,6 +23,7 @@ export type ProductAction
|
|||||||
| 'speech_requested'
|
| 'speech_requested'
|
||||||
| 'speech_succeeded'
|
| 'speech_succeeded'
|
||||||
| 'speech_failed'
|
| 'speech_failed'
|
||||||
|
| 'speech_blocked'
|
||||||
| 'voice_pack_created'
|
| 'voice_pack_created'
|
||||||
| 'voice_pack_updated'
|
| 'voice_pack_updated'
|
||||||
| 'voice_pack_disabled'
|
| 'voice_pack_disabled'
|
||||||
|
|||||||
@@ -430,8 +430,20 @@ const speechPipeline = createSpeechPipeline<AudioBuffer>({
|
|||||||
// Non-streaming providers only: synth via REST. Streaming provider
|
// Non-streaming providers only: synth via REST. Streaming provider
|
||||||
// was already early-returned above; it owns its own ws path opened
|
// was already early-returned above; it owns its own ws path opened
|
||||||
// in `onBeforeMessageComposed`.
|
// in `onBeforeMessageComposed`.
|
||||||
|
const providerConfigWithAnalytics = activeSpeechProvider.value === OFFICIAL_SPEECH_PROVIDER_ID
|
||||||
|
? {
|
||||||
|
...speechRequest.providerConfig,
|
||||||
|
extraBody: {
|
||||||
|
...(speechRequest.providerConfig.extraBody as Record<string, unknown> | undefined),
|
||||||
|
airi_analytics: {
|
||||||
|
trigger: 'auto',
|
||||||
|
source: 'chat_auto_tts',
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
: speechRequest.providerConfig
|
||||||
const res = await generateSpeech({
|
const res = await generateSpeech({
|
||||||
...provider.speech(model, speechRequest.providerConfig),
|
...provider.speech(model, providerConfigWithAnalytics),
|
||||||
input: speechRequest.input,
|
input: speechRequest.input,
|
||||||
voice: voice.id,
|
voice: voice.id,
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -48,6 +48,10 @@ export interface StreamingTtsPipelineOptions extends StreamingTtsPipelineEvents
|
|||||||
responseFormat?: 'mp3' | 'opus' | 'aac' | 'flac' | 'pcm'
|
responseFormat?: 'mp3' | 'opus' | 'aac' | 'flac' | 'pcm'
|
||||||
/** Backend-specific knobs forwarded as the `extra_body` of the `start` frame. */
|
/** Backend-specific knobs forwarded as the `extra_body` of the `start` frame. */
|
||||||
extraBody?: Record<string, unknown>
|
extraBody?: Record<string, unknown>
|
||||||
|
/** Business trigger hint sent to server-side product analytics. */
|
||||||
|
ttsTrigger?: 'auto' | 'manual'
|
||||||
|
/** Low-cardinality source hint sent to server-side product analytics. */
|
||||||
|
ttsSource?: 'chat_auto_tts' | 'manual_preview' | 'settings_test'
|
||||||
/**
|
/**
|
||||||
* Decoder context. The pipeline calls `decodeAudioData` on it for each
|
* Decoder context. The pipeline calls `decodeAudioData` on it for each
|
||||||
* sentence (or once at session end in buffered mode). Reusing the page's
|
* sentence (or once at session end in buffered mode). Reusing the page's
|
||||||
@@ -115,7 +119,10 @@ export function createStreamingTtsPipeline(options: StreamingTtsPipelineOptions)
|
|||||||
return noopHandle()
|
return noopHandle()
|
||||||
}
|
}
|
||||||
|
|
||||||
const wsUrl = toWebSocketUrl(options.serverUrl ?? SERVER_URL, '/api/v1/audio/speech/ws', token)
|
const wsUrl = toWebSocketUrl(options.serverUrl ?? SERVER_URL, '/api/v1/audio/speech/ws', token, {
|
||||||
|
ttsTrigger: options.ttsTrigger ?? 'auto',
|
||||||
|
ttsSource: options.ttsSource ?? 'chat_auto_tts',
|
||||||
|
})
|
||||||
const ws = new WebSocket(wsUrl)
|
const ws = new WebSocket(wsUrl)
|
||||||
ws.binaryType = 'arraybuffer'
|
ws.binaryType = 'arraybuffer'
|
||||||
|
|
||||||
@@ -400,10 +407,17 @@ function noopHandle(): StreamingTtsPipelineHandle {
|
|||||||
return { appendText: () => {}, finish: () => {}, cancel: () => {} }
|
return { appendText: () => {}, finish: () => {}, cancel: () => {} }
|
||||||
}
|
}
|
||||||
|
|
||||||
function toWebSocketUrl(httpBase: string, path: string, token: string): string {
|
function toWebSocketUrl(
|
||||||
|
httpBase: string,
|
||||||
|
path: string,
|
||||||
|
token: string,
|
||||||
|
analytics: { ttsTrigger: 'auto' | 'manual', ttsSource: 'chat_auto_tts' | 'manual_preview' | 'settings_test' },
|
||||||
|
): string {
|
||||||
const u = new URL(path, httpBase)
|
const u = new URL(path, httpBase)
|
||||||
u.protocol = u.protocol === 'https:' ? 'wss:' : 'ws:'
|
u.protocol = u.protocol === 'https:' ? 'wss:' : 'ws:'
|
||||||
u.searchParams.set('token', token)
|
u.searchParams.set('token', token)
|
||||||
|
u.searchParams.set('tts_trigger', analytics.ttsTrigger)
|
||||||
|
u.searchParams.set('tts_source', analytics.ttsSource)
|
||||||
return u.toString()
|
return u.toString()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -54,6 +54,10 @@ export interface StreamingTtsSessionOptions {
|
|||||||
* `section_id`, `context_texts`, etc.
|
* `section_id`, `context_texts`, etc.
|
||||||
*/
|
*/
|
||||||
extraBody?: Record<string, unknown>
|
extraBody?: Record<string, unknown>
|
||||||
|
/** Business trigger hint sent to server-side product analytics. */
|
||||||
|
ttsTrigger?: 'auto' | 'manual'
|
||||||
|
/** Low-cardinality source hint sent to server-side product analytics. */
|
||||||
|
ttsSource?: 'chat_auto_tts' | 'manual_preview' | 'settings_test'
|
||||||
/** Caller-side abort signal. Closes the ws and rejects with `AbortError`. */
|
/** Caller-side abort signal. Closes the ws and rejects with `AbortError`. */
|
||||||
signal?: AbortSignal
|
signal?: AbortSignal
|
||||||
}
|
}
|
||||||
@@ -85,7 +89,10 @@ export async function streamingSynthesize(options: StreamingTtsSessionOptions):
|
|||||||
throw new Error('streaming-tts: not authenticated')
|
throw new Error('streaming-tts: not authenticated')
|
||||||
|
|
||||||
const baseUrl = options.serverUrl ?? SERVER_URL
|
const baseUrl = options.serverUrl ?? SERVER_URL
|
||||||
const wsUrl = toWebSocketUrl(baseUrl, '/api/v1/audio/speech/ws', token)
|
const wsUrl = toWebSocketUrl(baseUrl, '/api/v1/audio/speech/ws', token, {
|
||||||
|
ttsTrigger: options.ttsTrigger ?? 'manual',
|
||||||
|
ttsSource: options.ttsSource ?? 'manual_preview',
|
||||||
|
})
|
||||||
|
|
||||||
const audioChunks: ArrayBuffer[] = []
|
const audioChunks: ArrayBuffer[] = []
|
||||||
const sentences: StreamingTtsSessionResult['sentences'] = []
|
const sentences: StreamingTtsSessionResult['sentences'] = []
|
||||||
@@ -225,10 +232,17 @@ export async function streamingSynthesize(options: StreamingTtsSessionOptions):
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
function toWebSocketUrl(httpBase: string, path: string, token: string): string {
|
function toWebSocketUrl(
|
||||||
|
httpBase: string,
|
||||||
|
path: string,
|
||||||
|
token: string,
|
||||||
|
analytics: { ttsTrigger: 'auto' | 'manual', ttsSource: 'chat_auto_tts' | 'manual_preview' | 'settings_test' },
|
||||||
|
): string {
|
||||||
const u = new URL(path, httpBase)
|
const u = new URL(path, httpBase)
|
||||||
u.protocol = u.protocol === 'https:' ? 'wss:' : 'ws:'
|
u.protocol = u.protocol === 'https:' ? 'wss:' : 'ws:'
|
||||||
u.searchParams.set('token', token)
|
u.searchParams.set('token', token)
|
||||||
|
u.searchParams.set('tts_trigger', analytics.ttsTrigger)
|
||||||
|
u.searchParams.set('tts_source', analytics.ttsSource)
|
||||||
return u.toString()
|
return u.toString()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -393,10 +393,15 @@ export const useSpeechStore = defineStore('speech', () => {
|
|||||||
voice: string,
|
voice: string,
|
||||||
providerConfig: Record<string, any> = {},
|
providerConfig: Record<string, any> = {},
|
||||||
): Promise<ArrayBuffer> {
|
): Promise<ArrayBuffer> {
|
||||||
|
const requestProviderConfig = activeSpeechProvider.value === OFFICIAL_SPEECH_PROVIDER_ID
|
||||||
|
|| activeSpeechProvider.value === OFFICIAL_SPEECH_STREAMING_PROVIDER_ID
|
||||||
|
? withAiriTtsAnalytics(providerConfig, {
|
||||||
|
trigger: 'manual',
|
||||||
|
source: 'manual_preview',
|
||||||
|
})
|
||||||
|
: providerConfig
|
||||||
const response = await generateSpeech({
|
const response = await generateSpeech({
|
||||||
...provider.speech(model, {
|
...provider.speech(model, requestProviderConfig),
|
||||||
...providerConfig,
|
|
||||||
}),
|
|
||||||
input,
|
input,
|
||||||
voice,
|
voice,
|
||||||
})
|
})
|
||||||
@@ -404,6 +409,19 @@ export const useSpeechStore = defineStore('speech', () => {
|
|||||||
return response
|
return response
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function withAiriTtsAnalytics(
|
||||||
|
providerConfig: Record<string, any>,
|
||||||
|
analytics: { trigger: 'auto' | 'manual', source: 'chat_auto_tts' | 'manual_preview' | 'settings_test' },
|
||||||
|
): Record<string, any> {
|
||||||
|
return {
|
||||||
|
...providerConfig,
|
||||||
|
extraBody: {
|
||||||
|
...(providerConfig.extraBody as Record<string, unknown> | undefined),
|
||||||
|
airi_analytics: analytics,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
function generateSSML(
|
function generateSSML(
|
||||||
text: string,
|
text: string,
|
||||||
voice: VoiceInfo,
|
voice: VoiceInfo,
|
||||||
|
|||||||
Reference in New Issue
Block a user