nanakura/moeka-project
1
0
Fork 0
Code Issues Pull Requests 1 Actions 2 Packages Projects Releases Wiki Activity

refactor: streamline speech module by removing voice pack dependencies

Browse Source 
- Removed voice pack related imports and functionality from speech.vue and Stage.vue.
- Simplified speech input handling by eliminating voice pack parameters in speech store.
- Updated tests to reflect changes in voice pack handling and speech input resolution.
- Adjusted Airi card store to remove voice pack binding logic, focusing on speech configuration updates.
- Enhanced voice pack list item structure for clarity and maintainability.
This commit is contained in:
RainbowBird
2026-07-01 19:23:33 +08:00
parent c2aee68513
commit 27255001bd
17 changed files with 233 additions and 848 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/server/src/routes/admin/voice-packs/route.test.ts
+3 -2
View File
@@ -42,6 +42,7 @@ function createService() {
disable: vi.fn(async (id: string): Promise<VoicePack | null> => makePack({ id, enabled: false })),
listEnabled: vi.fn(),
findById: vi.fn(),
findEnabledByVoiceId: vi.fn(),
} satisfies VoicePackService
}
@@ -111,7 +112,7 @@ describe('admin voice packs — CRUD', () => {
})
it('creates a pack with validated fields', async () => {
// @example valid body -> route forwards normalized params and enabled default.
// @example valid body -> route forwards canonical numeric params and enabled default.
const service = createService()
const productEventService = createProductEventService()
const app = createTestApp(service, ADMIN, productEventService)
@@ -122,7 +123,7 @@ describe('admin voice packs — CRUD', () => {
voiceId: 'voice-neuro',
upstreamVoiceId: 'voice-neuro-upstream',
ttsModelId: 'volcengine/neuro-pool',
params: { pitch: '+20%' },
params: { pitch: 20 },
costMultiplier: 1.5,
}
const res = await jsonRequest(app, 'POST', '/api/admin/voice-packs', body)
apps/server/src/routes/openai/v1/operations/speech-catalog/index.ts
+16 -2
View File
@@ -1,3 +1,4 @@
import type { VoicePack } from '../../../../../schemas/voice-packs'
import type { V1RouteDeps } from '../../types'
import { useLogger } from '@guiiai/logg'
@@ -5,6 +6,18 @@ import { ofetch } from 'ofetch'
import { createBadGatewayError, createBadRequestError, createServiceUnavailableError } from '../../../../../utils/error'
function voicePackCatalogVoice(pack: VoicePack) {
const cost = `Flux cost: ${pack.costMultiplier}x`
return {
id: pack.voiceId,
name: pack.name,
description: pack.description ? `${pack.description} · ${cost}` : cost,
labels: { type: 'voice_pack' },
tags: ['voice_pack'],
languages: [{ code: 'en', title: 'English' }],
}
}
export interface SpeechCatalogOperation {
listSpeechModels: () => Promise<Response>
listStreamingSpeechModels: () => Promise<Response>
@@ -43,12 +56,13 @@ export function createSpeechCatalogOperation(deps: V1RouteDeps): SpeechCatalogOp
: requested
const voices = await deps.llmRouter.listTtsVoices(model)
const voicePacks = await deps.voicePackService.listEnabled()
const recommended = (await deps.configKV.getOptional('DEFAULT_TTS_VOICES'))?.[model] ?? {}
// Debug level: high-frequency catalog poll from UI selectors, no
// billing / user-facing side effect — useful only when debugging
// voice-picker drift, never as a permanent audit trail line.
logger.withFields({ model, voiceCount: voices.length }).debug('list tts voices')
return Response.json({ voices, recommended })
logger.withFields({ model, voiceCount: voices.length, voicePackCount: voicePacks.length }).debug('list tts voices')
return Response.json({ voices: [...voicePacks.map(voicePackCatalogVoice), ...voices], recommended })
}
/**
apps/server/src/routes/openai/v1/route.test.ts
+63 -26
View File
@@ -150,6 +150,7 @@ function createMockVoicePackService(impl?: Partial<VoicePackService>): VoicePack
update: vi.fn(),
disable: vi.fn(),
findById: vi.fn(async () => null),
findEnabledByVoiceId: vi.fn(async () => null),
...impl,
} as unknown as VoicePackService
}
@@ -686,9 +687,9 @@ describe('v1CompletionsRoutes', () => {
/**
* @example
* POST /api/v1/audio/speech { "speed": 1.2, "extra_body": { "voice_pack": { "pitch": 20 } } }
* POST /api/v1/audio/speech { "model": "auto", "voice": "friendly-azure" }
*/
it('forwards TTS speed and Voice Pack prosody options to the router input', async () => {
it('resolves Voice Pack aliases to server-owned model, voice, and params', async () => {
const routeTts = vi.fn(async () => new Response(new Uint8Array([1]), {
status: 200,
headers: { 'Content-Type': 'audio/mpeg' },
@@ -704,7 +705,7 @@ describe('v1CompletionsRoutes', () => {
createMockLlmTracing(),
createMockProductEventService(),
createMockVoicePackService({
findById: vi.fn(async () => ({
findEnabledByVoiceId: vi.fn(async () => ({
id: 'vp-azure',
name: 'Azure',
description: null,
@@ -713,7 +714,7 @@ describe('v1CompletionsRoutes', () => {
voiceId: 'friendly-azure',
upstreamVoiceId: 'en-US-AvaMultilingualNeural',
ttsModelId: 'microsoft/v1',
params: {},
params: { pitch: 20, volume: 5, rate: 1.2 },
costMultiplier: 1.5,
enabled: true,
createdAt: new Date(),
@@ -730,14 +731,6 @@ describe('v1CompletionsRoutes', () => {
model: 'auto',
input: 'test',
voice: 'friendly-azure',
speed: 1.2,
extra_body: {
voice_pack: {
pack_id: 'vp-azure',
pitch: 20,
volume: 5,
},
},
}),
}),
{ user: testUser } as any,
@@ -784,7 +777,7 @@ describe('v1CompletionsRoutes', () => {
/**
* @example
* POST /api/v1/audio/speech { "input": "hello", "extra_body": { "voice_pack": { "pack_id": "vp-premium" } } }
* POST /api/v1/audio/speech { "input": "hello", "voice": "alloy" }
*/
it('uses Voice Pack cost multiplier for affordability and billing units', async () => {
globalThis.fetch = vi.fn(async () => new Response(new Uint8Array([1]), {
@@ -794,7 +787,7 @@ describe('v1CompletionsRoutes', () => {
const ttsMeter = createMockTtsMeter()
const voicePackService = createMockVoicePackService({
findById: vi.fn(async () => ({
findEnabledByVoiceId: vi.fn(async () => ({
id: 'vp-premium',
name: 'Premium',
description: null,
@@ -805,7 +798,7 @@ describe('v1CompletionsRoutes', () => {
ttsModelId: 'tts-1',
params: {},
costMultiplier: 2,
enabled: false,
enabled: true,
createdAt: new Date(),
updatedAt: new Date(),
})),
@@ -830,11 +823,6 @@ describe('v1CompletionsRoutes', () => {
model: 'auto',
input: 'hello',
voice: 'alloy',
extra_body: {
voice_pack: {
pack_id: 'vp-premium',
},
},
}),
}),
{ user: testUser } as any,
@@ -851,7 +839,7 @@ describe('v1CompletionsRoutes', () => {
/**
* @example
* POST /api/v1/audio/speech { "voice": "alloy", "extra_body": { "voice_pack": { "pack_id": "vp-premium" } } }
* POST /api/v1/audio/speech { "voice": "alloy" }
*/
it('records TTS voice and Voice Pack metadata in product events', async () => {
globalThis.fetch = vi.fn(async () => new Response(new Uint8Array([1]), {
@@ -861,7 +849,7 @@ describe('v1CompletionsRoutes', () => {
const productEventService = createMockProductEventService()
const voicePackService = createMockVoicePackService({
findById: vi.fn(async () => ({
findEnabledByVoiceId: vi.fn(async () => ({
id: 'vp-premium',
name: 'Premium',
description: null,
@@ -898,12 +886,9 @@ describe('v1CompletionsRoutes', () => {
input: 'hello',
voice: 'alloy',
extra_body: {
voice_pack: {
pack_id: 'vp-premium',
},
airi_analytics: {
source: 'manual_preview',
voice_type: 'voice_pack',
voice_type: 'official_selected',
},
},
}),
@@ -1404,6 +1389,58 @@ describe('v1CompletionsRoutes', () => {
expect(llmRouter.listTtsVoices).toHaveBeenCalledWith('microsoft/v1')
})
it('includes enabled Voice Packs as official catalog voices without upstream details', async () => {
const llmRouter = createMockLlmRouter({
listTtsVoices: vi.fn(async () => [
{ id: 'en-US-AvaMultilingualNeural', name: 'Ava', languages: [{ code: 'en-US', title: 'English' }] },
]) as any,
})
const voicePackService = createMockVoicePackService({
listEnabled: vi.fn(async () => [{
id: 'vp-1',
name: 'Narrator',
description: 'Warm voice',
provider: 'azure',
model: 'microsoft/v1',
voiceId: 'narrator-alias',
upstreamVoiceId: 'en-US-AvaMultilingualNeural',
ttsModelId: 'microsoft/v1',
params: {},
costMultiplier: 2,
enabled: true,
createdAt: new Date(),
updatedAt: new Date(),
}]),
})
const app = createTestApp(
createMockFluxService(),
createMockConfigKV({ DEFAULT_TTS_VOICES: { 'microsoft/v1': { 'en-US': 'en-US-AvaMultilingualNeural' } } }),
undefined,
undefined,
undefined,
llmRouter,
createMockLlmTracing(),
createMockProductEventService(),
voicePackService,
)
const res = await app.fetch(
new Request('http://localhost/api/v1/audio/voices?model=microsoft/v1', { method: 'GET' }),
{ user: testUser } as any,
)
expect(res.status).toBe(200)
const data = await res.json() as { voices: Array<Record<string, unknown>> }
expect(data.voices[0]).toMatchObject({
id: 'narrator-alias',
name: 'Narrator',
description: 'Warm voice · Flux cost: 2x',
})
expect(data.voices[0]).not.toHaveProperty('upstreamVoiceId')
expect(data.voices[0]).not.toHaveProperty('ttsModelId')
expect(data.voices[1]).toMatchObject({ id: 'en-US-AvaMultilingualNeural' })
})
it('returns an empty recommended map when the resolved model has no bucket', async () => {
const llmRouter = createMockLlmRouter({
listTtsVoices: vi.fn(async () => []) as any,
apps/server/src/routes/voice-packs/route.test.ts
+3 -2
View File
@@ -32,7 +32,7 @@ function createService() {
voiceId: 'friendly-voice',
upstreamVoiceId: 'en-US-AvaMultilingualNeural',
ttsModelId: 'microsoft/v1',
params: { pitch: '+10%' },
params: { pitch: 10 },
costMultiplier: 2,
enabled: true,
createdAt: new Date('2026-01-01T00:00:00.000Z'),
@@ -43,6 +43,7 @@ function createService() {
update: vi.fn(),
disable: vi.fn(),
findById: vi.fn(),
findEnabledByVoiceId: vi.fn(),
} as unknown as VoicePackService
}
@@ -69,7 +70,7 @@ describe('voice packs routes', () => {
name: 'Enabled',
description: 'Public description',
voiceId: 'friendly-voice',
params: { pitch: '+10%' },
params: { pitch: 10 },
costMultiplier: 2,
enabled: true,
createdAt: '2026-01-01T00:00:00.000Z',
apps/server/src/schemas/voice-packs.ts
+5 -1
View File
@@ -4,7 +4,11 @@ import { boolean, jsonb, pgTable, real, text, timestamp } from 'drizzle-orm/pg-c
import { nanoid } from '../utils/id'
export type VoicePackParams = Record<string, string | number | boolean | null>
export interface VoicePackParams {
pitch?: number
volume?: number
rate?: number
}
export const voicePacks = pgTable(
'voice_packs',
apps/server/src/services/domain/openai-speech/index.ts
+21 -21
View File
@@ -175,7 +175,7 @@ export function createOpenAiSpeechService(deps: OpenAiSpeechServiceDeps) {
const ttsInput = {
text: inputText,
voice: routedVoice,
speed: typeof input.body.speed === 'number' ? input.body.speed : undefined,
speed: voicePackRequest.speed ?? (typeof input.body.speed === 'number' ? input.body.speed : undefined),
responseFormat: typeof input.body.response_format === 'string' ? input.body.response_format : undefined,
extraOptions: voicePackRequest.extraOptions,
}
@@ -418,6 +418,7 @@ async function voicePackRequestOptions(
voicePackId?: string
model?: string
voice?: string
speed?: number
}> {
const extraBody = asRecord(body.extra_body)
const voicePackOptions = asRecord(extraBody?.voice_pack)
@@ -425,10 +426,12 @@ async function voicePackRequestOptions(
const volume = readOptionalNumber(voicePackOptions, 'volume')
const voicePack = await resolveVoicePackRequest(voicePackOptions, context)
const extraOptions: Record<string, unknown> = {}
if (pitch != null)
extraOptions.pitch = pitch
if (volume != null)
extraOptions.volume = volume
const resolvedPitch = voicePack?.params.pitch ?? pitch
const resolvedVolume = voicePack?.params.volume ?? volume
if (resolvedPitch != null)
extraOptions.pitch = resolvedPitch
if (resolvedVolume != null)
extraOptions.volume = resolvedVolume
return {
extraOptions: Object.keys(extraOptions).length > 0 ? extraOptions : undefined,
@@ -436,6 +439,7 @@ async function voicePackRequestOptions(
voicePackId: voicePack?.id,
model: voicePack?.ttsModelId,
voice: voicePack?.upstreamVoiceId,
speed: voicePack?.params.rate,
}
}
@@ -448,31 +452,27 @@ async function resolveVoicePackRequest(
},
): Promise<Awaited<ReturnType<VoicePackService['findById']>> | null> {
const packId = voicePackOptions?.pack_id
const requestedVoice = context.voice?.trim()
if (voicePackOptions?.cost_multiplier != null) {
throw createBadRequestError('voice_pack.cost_multiplier is server-managed', 'INVALID_VOICE_PACK', {
field: 'voice_pack.cost_multiplier',
})
}
if (packId == null)
return null
if (typeof packId !== 'string' || !packId.trim())
if (packId != null && (typeof packId !== 'string' || !packId.trim()))
throw createBadRequestError('voice_pack.pack_id is required when Voice Pack billing metadata is provided', 'INVALID_VOICE_PACK')
const pack = await context.voicePackService.findById(packId)
const pack = typeof packId === 'string'
? await context.voicePackService.findById(packId)
: requestedVoice
? await context.voicePackService.findEnabledByVoiceId(requestedVoice)
: null
if (!pack && packId == null)
return null
if (!pack)
throw createBadRequestError('Voice Pack not found', 'INVALID_VOICE_PACK', { packId })
if (context.requestedModel !== 'auto' && pack.ttsModelId !== context.requestedModel) {
throw createBadRequestError('Voice Pack does not match requested model and voice', 'INVALID_VOICE_PACK', {
packId,
actualModel: context.requestedModel,
})
}
if (pack.voiceId !== context.voice) {
throw createBadRequestError('Voice Pack does not match requested model and voice', 'INVALID_VOICE_PACK', {
packId,
actualVoice: context.voice,
})
}
if (!pack.enabled)
throw createBadRequestError('Voice Pack not found', 'INVALID_VOICE_PACK', { packId })
return pack
}
apps/server/src/services/domain/voice-packs/index.test.ts
+36 -5
View File
@@ -29,7 +29,7 @@ describe('voicePackService', () => {
voiceId: 'voice-neuro',
upstreamVoiceId: 'voice-neuro-upstream',
ttsModelId: 'volcengine/neuro-pool',
params: { pitch: '+20%', volume: '+5%' },
params: { pitch: 20, volume: 5 },
costMultiplier: 1.5,
enabled: true,
})
@@ -40,7 +40,7 @@ describe('voicePackService', () => {
expect(pack.voiceId).toBe('voice-neuro')
expect(pack.upstreamVoiceId).toBe('voice-neuro-upstream')
expect(pack.ttsModelId).toBe('volcengine/neuro-pool')
expect(pack.params).toEqual({ pitch: '+20%', volume: '+5%' })
expect(pack.params).toEqual({ pitch: 20, volume: 5 })
expect(pack.costMultiplier).toBe(1.5)
expect(pack.enabled).toBe(true)
})
@@ -65,7 +65,7 @@ describe('voicePackService', () => {
voiceId: 'voice-a',
upstreamVoiceId: 'voice-a-upstream',
ttsModelId: 'volcengine/pool',
params: { pitch: '+20%' },
params: { pitch: 20 },
costMultiplier: 1,
enabled: true,
})
@@ -91,13 +91,13 @@ describe('voicePackService', () => {
const updated = await service.update(pack.id, {
name: 'New',
params: { rate: '+10%' },
params: { rate: 1.1 },
costMultiplier: 2,
})
expect(updated?.id).toBe(pack.id)
expect(updated?.name).toBe('New')
expect(updated?.params).toEqual({ rate: '+10%' })
expect(updated?.params).toEqual({ rate: 1.1 })
expect(updated?.costMultiplier).toBe(2)
})
@@ -124,6 +124,37 @@ describe('voicePackService', () => {
expect(enabled).toEqual([])
})
it('finds only enabled packs by product-facing voice alias', async () => {
// @example TTS request voice="narrator" -> enabled Voice Pack row resolves server-side.
await service.create({
name: 'Disabled narrator',
provider: 'azure',
model: 'v1',
voiceId: 'narrator',
upstreamVoiceId: 'disabled-upstream',
ttsModelId: 'microsoft/v1',
params: {},
costMultiplier: 1,
enabled: false,
})
const enabled = await service.create({
name: 'Enabled narrator',
provider: 'azure',
model: 'v1',
voiceId: 'narrator',
upstreamVoiceId: 'enabled-upstream',
ttsModelId: 'microsoft/v1',
params: {},
costMultiplier: 1,
enabled: true,
})
expect(await service.findEnabledByVoiceId('narrator')).toMatchObject({
id: enabled.id,
upstreamVoiceId: 'enabled-upstream',
})
})
it('returns null when updating or disabling a missing pack', async () => {
// @example unknown id -> null so routes can map to 404.
expect(await service.update('missing', { name: 'Nope' })).toBeNull()
apps/server/src/services/domain/voice-packs/index.ts
+15 -5
View File
@@ -4,14 +4,15 @@ import type { Database } from '../../../libs/db'
import type { VoicePack } from '../../../schemas/voice-packs'
import { and, eq } from 'drizzle-orm'
import { boolean, maxLength, minValue, nonEmpty, null_, number, object, optional, pipe, record, string, union } from 'valibot'
import { boolean, maxLength, minValue, nonEmpty, number, object, optional, pipe, string } from 'valibot'
import * as schema from '../../../schemas/voice-packs'
export const VoicePackParamsSchema = record(
pipe(string(), nonEmpty('params keys must not be empty'), maxLength(100)),
union([string(), number(), boolean(), null_()]),
)
export const VoicePackParamsSchema = object({
pitch: optional(number()),
volume: optional(number()),
rate: optional(pipe(number(), minValue(0.01, 'rate must be positive'))),
})
export const VoicePackCostMultiplierSchema = pipe(
number(),
@@ -105,6 +106,15 @@ export function createVoicePackService(db: Database) {
})
},
async findEnabledByVoiceId(voiceId: string) {
return await db.query.voicePacks.findFirst({
where: and(
eq(schema.voicePacks.voiceId, voiceId),
eq(schema.voicePacks.enabled, true),
),
})
},
async update(id: string, input: UpdateVoicePackInput): Promise<VoicePack | null> {
const [updated] = await db.update(schema.voicePacks)
.set({ ...input, updatedAt: new Date() })
apps/ui-admin/src/modules/api.ts
+3 -1
View File
@@ -170,7 +170,9 @@ export interface AdminRouterConfigCurrent {
}
export interface VoicePackParams {
[key: string]: string | number | boolean | null
pitch?: number
volume?: number
rate?: number
}
export interface VoicePack {
apps/ui-admin/src/pages/VoicePackFormPage.vue
+11 -57
View File
@@ -272,9 +272,10 @@ function parseParams(): VoicePackParams {
throw new Error('Params keys must not be empty')
if (!supportedParams.has(key))
throw new Error(`Unsupported Voice Pack parameter "${key}"`)
const valid = typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean' || value == null
if (!valid)
throw new Error(`Unsupported params value for "${key}"`)
if (typeof value !== 'number' || !Number.isFinite(value))
throw new Error(`Voice Pack parameter "${key}" must be a finite number`)
if (key === 'rate' && value <= 0)
throw new Error('Voice Pack parameter "rate" must be positive')
}
return parsed as VoicePackParams
@@ -350,7 +351,7 @@ async function testVoicePack() {
model: form.ttsModelId.trim(),
input: text,
voice: form.upstreamVoiceId.trim(),
speed: normalizeRateOption(params.rate),
speed: params.rate,
extra_body: voicePackExtraBody(params),
}
const blob = await adminApi.testSpeech(body)
@@ -366,13 +367,11 @@ async function testVoicePack() {
}
function voicePackExtraBody(params: VoicePackParams) {
const pitch = normalizePercentOption(params.pitch, 'pitch')
const volume = normalizePercentOption(params.volume, 'volume')
const voicePack: Record<string, unknown> = {}
if (pitch != null)
voicePack.pitch = pitch
if (volume != null)
voicePack.volume = volume
if (params.pitch != null)
voicePack.pitch = params.pitch
if (params.volume != null)
voicePack.volume = params.volume
return Object.keys(voicePack).length > 0 ? { voice_pack: voicePack } : undefined
}
@@ -411,51 +410,6 @@ function voiceOptionDescription(voice: SpeechVoice): string | undefined {
function firstRecommendedVoiceId(recommended: Record<string, string>): string | undefined {
return recommended['zh-CN'] ?? recommended['en-US'] ?? Object.values(recommended)[0]
}
function normalizePercentOption(value: string | number | boolean | null | undefined, name: string): number | undefined {
if (value == null)
return undefined
if (typeof value === 'number') {
if (Number.isFinite(value))
return value
throw new Error(`Voice Pack parameter "${name}" must be a finite number.`)
}
if (typeof value !== 'string')
throw new Error(`Voice Pack parameter "${name}" must be a number or percent string.`)
const trimmed = value.trim()
const normalized = trimmed.endsWith('%') ? trimmed.slice(0, -1) : trimmed
const parsed = Number(normalized)
if (!Number.isFinite(parsed))
throw new Error(`Voice Pack parameter "${name}" must be a number or percent string.`)
return parsed
}
function normalizeRateOption(value: string | number | boolean | null | undefined): number | undefined {
if (value == null)
return undefined
if (typeof value === 'number') {
if (Number.isFinite(value) && value > 0)
return value
throw new Error('Voice Pack parameter "rate" must be a positive finite number or percent string.')
}
if (typeof value !== 'string')
throw new Error('Voice Pack parameter "rate" must be a positive finite number or percent string.')
const trimmed = value.trim()
if (trimmed.endsWith('%')) {
const percent = normalizePercentOption(trimmed, 'rate')
const speed = 1 + (percent ?? 0) / 100
if (speed > 0)
return speed
throw new Error('Voice Pack parameter "rate" percent must resolve to a positive speed.')
}
const parsed = Number(trimmed)
if (Number.isFinite(parsed) && parsed > 0)
return parsed
throw new Error('Voice Pack parameter "rate" must be a positive finite number or percent string.')
}
</script>
<template>
@@ -557,9 +511,9 @@ function normalizeRateOption(value: string | number | boolean | null | undefined
<FieldTextArea
v-model="form.paramsJson"
description="Supported keys: rate, pitch, volume. Example: { &quot;rate&quot;: &quot;+8%&quot;, &quot;pitch&quot;: 3 }"
description="Supported numeric keys: rate, pitch, volume. Example: { &quot;rate&quot;: 1.08, &quot;pitch&quot;: 3 }"
label="Params JSON"
placeholder="{&#10; &quot;rate&quot;: &quot;+8%&quot;&#10;}"
placeholder="{&#10; &quot;rate&quot;: 1.08&#10;}"
:required="false"
:rows="9"
textarea-class="font-mono text-xs leading-5"
packages/stage-pages/src/pages/settings/modules/speech.vue
+13 -240
View File
@@ -1,7 +1,5 @@
<script setup lang="ts">
import type { VoiceType } from '@proj-airi/stage-ui/composables'
import type { VoicePackSnapshot } from '@proj-airi/stage-ui/stores/modules/airi-card'
import type { VoiceInfo } from '@proj-airi/stage-ui/stores/providers'
import type { SpeechProviderWithExtraOptions } from '@xsai-ext/providers/utils'
import { errorMessageFrom } from '@moeru/std'
@@ -15,8 +13,8 @@ import {
} from '@proj-airi/stage-ui/components'
import { useAnalytics } from '@proj-airi/stage-ui/composables'
import { OFFICIAL_SPEECH_PROVIDER_ID, OFFICIAL_SPEECH_STREAMING_PROVIDER_ID } from '@proj-airi/stage-ui/libs/providers/providers/official'
import { useAiriCardStore, useVoicePacksStore } from '@proj-airi/stage-ui/stores'
import { useSpeechStore, voicePackForSpeechProvider } from '@proj-airi/stage-ui/stores/modules/speech'
import { useAiriCardStore } from '@proj-airi/stage-ui/stores'
import { useSpeechStore } from '@proj-airi/stage-ui/stores/modules/speech'
import { useProvidersStore } from '@proj-airi/stage-ui/stores/providers'
import {
FieldCheckbox,
@@ -35,10 +33,7 @@ const { t } = useI18n()
const providersStore = useProvidersStore()
const speechStore = useSpeechStore()
const airiCardStore = useAiriCardStore()
const voicePacksStore = useVoicePacksStore()
const { allAudioSpeechProvidersMetadata, configuredSpeechProvidersMetadata } = storeToRefs(providersStore)
const { activeCard } = storeToRefs(airiCardStore)
const { packs: voicePacks, loading: isLoadingVoicePacks, error: voicePacksError } = storeToRefs(voicePacksStore)
const {
activeSpeechProvider,
activeSpeechModel,
@@ -59,7 +54,6 @@ const {
const {
trackProviderClick,
trackTtsProviderSelected,
trackVoicePackBound,
trackVoicePreviewPlayed,
trackVoiceSelected,
} = useAnalytics()
@@ -72,18 +66,9 @@ const isGenerating = ref(false)
const audioUrl = ref('')
const audioPlayer = ref<HTMLAudioElement | null>(null)
const errorMessage = ref('')
const selectedSpeechSource = ref<string | null>(null)
const VOICE_PACK_SOURCE_ID = 'voice-pack'
const VOICE_PACK_REQUEST_MODEL_ID = 'auto'
const VOICE_PACK_ANALYTICS_MODEL_ID = 'voice_pack'
const STREAMING_MODEL_OPTION_PREFIX = 'streaming:'
const isOfficialSpeechProvider = computed(() => activeSpeechProvider.value === OFFICIAL_SPEECH_PROVIDER_ID)
const boundVoicePack = computed(() =>
voicePackForSpeechProvider(activeSpeechProvider.value, activeCard.value?.extensions.airi.modules.speech.voicePack),
)
const selectableSpeechSources = computed(() => {
const configuredSources = configuredSpeechProvidersMetadata.value
.filter(metadata =>
@@ -98,12 +83,6 @@ const selectableSpeechSources = computed(() => {
}))
return [
{
id: VOICE_PACK_SOURCE_ID,
providerId: undefined,
title: 'Voice Pack',
description: 'Server-curated voices',
},
...configuredSources,
...allAudioSpeechProvidersMetadata.value
.filter(metadata => metadata.id === 'speech-noop')
@@ -118,10 +97,6 @@ const selectableSpeechSources = computed(() => {
const displayedSpeechSource = computed({
get: () => {
if (selectedSpeechSource.value === VOICE_PACK_SOURCE_ID)
return VOICE_PACK_SOURCE_ID
if (boundVoicePack.value && isOfficialSpeechProvider.value)
return VOICE_PACK_SOURCE_ID
if (activeSpeechProvider.value === OFFICIAL_SPEECH_STREAMING_PROVIDER_ID)
return OFFICIAL_SPEECH_PROVIDER_ID
return activeSpeechProvider.value
@@ -131,7 +106,6 @@ const displayedSpeechSource = computed({
},
})
const isVoicePackSourceSelected = computed(() => displayedSpeechSource.value === VOICE_PACK_SOURCE_ID)
const isOfficialSpeechSourceSelected = computed(() => displayedSpeechSource.value === OFFICIAL_SPEECH_PROVIDER_ID)
function streamingModelOptionId(modelId: string) {
@@ -188,58 +162,7 @@ const displayedModelError = computed(() => {
|| null
})
function createVoicePackVoice(voicePack: VoicePackSnapshot): VoiceInfo {
return {
id: voicePack.voiceId,
name: voicePack.name,
description: voicePack.name,
previewURL: '',
languages: [{ code: 'en', title: 'English' }],
provider: activeSpeechProvider.value,
gender: 'neutral',
}
}
function formatVoicePackCostMultiplier(costMultiplier: number) {
return `Flux cost: ${costMultiplier}x`
}
function voicePackDescription(description: string | null | undefined, costMultiplier: number) {
const cost = formatVoicePackCostMultiplier(costMultiplier)
return description ? `${description} · ${cost}` : cost
}
function voicePackVoiceId(packId: string) {
return `voice-pack:${packId}`
}
function packIdFromVoicePackVoiceId(voiceId: string) {
return voiceId.startsWith('voice-pack:') ? voiceId.slice('voice-pack:'.length) : null
}
const displayedVoiceOptions = computed(() => {
if (isVoicePackSourceSelected.value) {
const options = voicePacks.value.map(pack => ({
id: voicePackVoiceId(pack.id),
name: pack.name,
description: voicePackDescription(pack.description, pack.costMultiplier),
previewURL: '',
customizable: false,
}))
const voicePack = boundVoicePack.value
const frozenVoiceId = voicePack ? voicePackVoiceId(voicePack.packId) : null
if (voicePack && frozenVoiceId && !options.some(option => option.id === frozenVoiceId)) {
options.unshift({
id: frozenVoiceId,
name: voicePack.name,
description: voicePackDescription(voicePack.name, voicePack.costMultiplier),
previewURL: '',
customizable: false,
})
}
return options
}
return (availableVoices.value[activeSpeechProvider.value] ?? [])
.filter((voice) => {
if (!activeSpeechModel.value)
@@ -256,64 +179,25 @@ const displayedVoiceOptions = computed(() => {
})
const displayedSpeechVoiceId = computed({
get: () => {
if (isVoicePackSourceSelected.value && boundVoicePack.value)
return voicePackVoiceId(boundVoicePack.value.packId)
return activeSpeechVoiceId.value
},
get: () => activeSpeechVoiceId.value,
set: (value: string) => {
if (isVoicePackSourceSelected.value) {
void selectSpeechVoice(value)
return
}
activeSpeechVoiceId.value = value
},
})
const currentSpeechVoiceId = computed(() => {
if (isVoicePackSourceSelected.value && boundVoicePack.value)
return boundVoicePack.value.voiceId
return activeSpeechVoiceId.value || ''
})
const currentVoicePackCostMultiplier = computed(() => {
if (isVoicePackSourceSelected.value && boundVoicePack.value)
return formatVoicePackCostMultiplier(boundVoicePack.value.costMultiplier)
return ''
})
function syncBoundVoicePackSelection() {
const voicePack = boundVoicePack.value
if (!voicePack)
return false
selectedSpeechSource.value = VOICE_PACK_SOURCE_ID
activeSpeechProvider.value = OFFICIAL_SPEECH_PROVIDER_ID
activeSpeechModel.value = VOICE_PACK_REQUEST_MODEL_ID
activeSpeechVoiceId.value = voicePack.voiceId
activeSpeechVoice.value = createVoicePackVoice(voicePack)
return true
}
const currentSpeechVoiceId = computed(() => activeSpeechVoiceId.value || '')
/**
* Resolves the current TTS model id for low-cardinality analytics payloads.
*/
function currentTtsModelId() {
if (isVoicePackSourceSelected.value && boundVoicePack.value)
return VOICE_PACK_ANALYTICS_MODEL_ID
return activeSpeechModel.value || 'unknown'
}
/**
* Classifies the selected voice without sending free-form provider config as a dimension.
*/
function currentVoiceType(voiceId: string, providerId = activeSpeechProvider.value, voicePack = boundVoicePack.value): VoiceType {
if (packIdFromVoicePackVoiceId(voiceId) != null)
return 'voice_pack'
if (voicePack?.voiceId === voiceId)
return 'voice_pack'
function currentVoiceType(voiceId: string, providerId = activeSpeechProvider.value): VoiceType {
const catalogVoice = availableVoices.value[providerId]?.some(voice => voice.id === voiceId)
if (catalogVoice)
return providerId === OFFICIAL_SPEECH_PROVIDER_ID || providerId === OFFICIAL_SPEECH_STREAMING_PROVIDER_ID ? 'official_selected' : 'custom_configured'
@@ -326,21 +210,18 @@ function currentVoiceType(voiceId: string, providerId = activeSpeechProvider.val
*/
function voiceAnalyticsPayload(
voiceId: string,
voicePack: VoicePackSnapshot | undefined = boundVoicePack.value,
providerId = activeSpeechProvider.value,
): {
voice_id: string
voice_type: VoiceType
voice_pack_id?: string
} {
const voiceType = voicePack?.voiceId === voiceId ? 'voice_pack' : currentVoiceType(voiceId, providerId, voicePack)
const voiceType = currentVoiceType(voiceId, providerId)
const isCatalogVoice = availableVoices.value[providerId]?.some(voice => voice.id === voiceId) ?? false
const shouldBucketVoiceId = voiceType === 'custom_configured' && !isCatalogVoice
return {
voice_id: shouldBucketVoiceId ? 'custom' : voiceId,
voice_type: voiceType,
...(voiceType === 'voice_pack' && voicePack ? { voice_pack_id: voicePack.packId } : {}),
}
}
@@ -388,16 +269,6 @@ async function selectSpeechVoice(voiceId: string | undefined) {
if (!voiceId)
return
const voicePackId = packIdFromVoicePackVoiceId(voiceId)
if (isVoicePackSourceSelected.value && voicePackId) {
const pack = voicePacks.value.find(item => item.id === voicePackId)
if (!pack)
return
await bindVoicePack(pack)
return
}
trackVoiceSelected({
tts_provider_id: activeSpeechProvider.value || 'unknown',
tts_model_id: currentTtsModelId(),
@@ -407,31 +278,6 @@ async function selectSpeechVoice(voiceId: string | undefined) {
}
function selectSpeechSource(sourceId: string) {
selectedSpeechSource.value = sourceId === VOICE_PACK_SOURCE_ID ? VOICE_PACK_SOURCE_ID : null
if (sourceId === VOICE_PACK_SOURCE_ID) {
activeSpeechProvider.value = OFFICIAL_SPEECH_PROVIDER_ID
const voicePack = boundVoicePack.value
if (voicePack) {
activeSpeechModel.value = VOICE_PACK_REQUEST_MODEL_ID
activeSpeechVoiceId.value = voicePack.voiceId
activeSpeechVoice.value = createVoicePackVoice(voicePack)
return
}
activeSpeechModel.value = ''
activeSpeechVoiceId.value = ''
activeSpeechVoice.value = undefined
return
}
if (sourceId === OFFICIAL_SPEECH_PROVIDER_ID && boundVoicePack.value) {
airiCardStore.updateActiveCardSpeech({
provider: OFFICIAL_SPEECH_PROVIDER_ID,
model: '',
voice_id: '',
})
}
activeSpeechProvider.value = sourceId
}
@@ -482,51 +328,11 @@ function syncOpenAICompatibleSettings() {
onMounted(async () => {
await providersStore.loadModelsForConfiguredProviders()
await voicePacksStore.load()
const syncedVoicePack = syncBoundVoicePackSelection()
if (!syncedVoicePack) {
speechStore.ensureActiveSpeechModel()
await speechStore.loadVoicesForProvider(activeSpeechProvider.value, activeSpeechModel.value || undefined)
}
speechStore.ensureActiveSpeechModel()
await speechStore.loadVoicesForProvider(activeSpeechProvider.value, activeSpeechModel.value || undefined)
syncOpenAICompatibleSettings()
})
async function bindVoicePack(pack: (typeof voicePacks.value)[number]) {
const bound = airiCardStore.bindVoicePackToActiveCard(pack)
if (!bound)
return
selectedSpeechSource.value = VOICE_PACK_SOURCE_ID
activeSpeechProvider.value = OFFICIAL_SPEECH_PROVIDER_ID
activeSpeechModel.value = VOICE_PACK_REQUEST_MODEL_ID
activeSpeechVoiceId.value = pack.voiceId
activeSpeechVoice.value = {
id: pack.voiceId,
name: pack.name,
description: voicePackDescription(pack.description ?? pack.name, pack.costMultiplier),
previewURL: '',
languages: [{ code: 'en', title: 'English' }],
provider: activeSpeechProvider.value,
gender: 'neutral',
}
trackVoicePackBound({
tts_provider_id: activeSpeechProvider.value || 'unknown',
tts_model_id: VOICE_PACK_ANALYTICS_MODEL_ID,
voice_id: pack.voiceId,
voice_pack_id: pack.id,
source: 'settings',
})
trackVoiceSelected({
tts_provider_id: activeSpeechProvider.value || 'unknown',
tts_model_id: VOICE_PACK_ANALYTICS_MODEL_ID,
voice_id: pack.voiceId,
voice_type: 'voice_pack',
voice_pack_id: pack.id,
source: 'settings',
})
}
watch(activeSpeechProvider, async (newProvider, oldProvider) => {
await providersStore.loadModelsForConfiguredProviders()
@@ -544,9 +350,6 @@ watch(activeSpeechProvider, async (newProvider, oldProvider) => {
activeSpeechVoice.value = undefined
}
if (isVoicePackSourceSelected.value)
return
// Re-seed the streaming default model after the reset above so its voices
// load model-scoped (the server only returns recommended voices for an
// explicit ?model=). No-op for other providers / when a model is selected.
@@ -556,21 +359,10 @@ watch(activeSpeechProvider, async (newProvider, oldProvider) => {
syncOpenAICompatibleSettings()
})
watch(boundVoicePack, () => {
syncBoundVoicePackSelection()
})
watch(voicePacks, () => {
syncBoundVoicePackSelection()
})
watch(activeSpeechModel, async (model) => {
if (!activeSpeechProvider.value)
return
if (isVoicePackSourceSelected.value)
return
activeSpeechVoiceId.value = ''
activeSpeechVoice.value = undefined
@@ -618,13 +410,6 @@ async function generateTestSpeech() {
}
}
const voicePack = boundVoicePack.value
if (voicePack) {
model = VOICE_PACK_REQUEST_MODEL_ID
if (!voice || voice.id !== voicePack.voiceId)
voice = createVoicePackVoice(voicePack)
}
if (!model) {
console.error('No model selected')
return
@@ -635,11 +420,10 @@ async function generateTestSpeech() {
return
}
const previewVoicePack = voicePack
const previewVoice = voice
const previewModel = model
const previewProvider = activeSpeechProvider.value || 'unknown'
const previewAnalytics = voiceAnalyticsPayload(previewVoice.id, previewVoicePack, previewProvider)
const previewAnalytics = voiceAnalyticsPayload(previewVoice.id, previewProvider)
isGenerating.value = true
errorMessage.value = ''
@@ -655,18 +439,15 @@ async function generateTestSpeech() {
input: ssmlText.value,
providerConfig,
}
: speechStore.resolveVoicePackSpeechInput({
: speechStore.resolveSpeechInput({
text: testText.value,
voice,
providerConfig: {
...providerConfig,
pitch: ssmlEnabled.value ? pitch.value : undefined,
},
params: voicePack?.params,
voicePack,
forceSSML: ssmlEnabled.value,
supportsSSML: speechStore.supportsSSML,
supportsAdapterProsody: activeSpeechProvider.value === OFFICIAL_SPEECH_PROVIDER_ID,
})
const response = await generateSpeech({
@@ -845,7 +626,7 @@ function handleDeleteProvider(providerId: string) {
</div>
<!-- Model selection section -->
<div v-if="activeSpeechProvider && activeSpeechProvider !== 'speech-noop' && !isVoicePackSourceSelected">
<div v-if="activeSpeechProvider && activeSpeechProvider !== 'speech-noop'">
<div flex="~ col gap-4">
<div>
<h2 class="text-lg md:text-2xl">
@@ -947,13 +728,12 @@ function handleDeleteProvider(providerId: string) {
<span>Customize how your AI assistant speaks</span>
<span v-if="currentSpeechVoiceId" class="text-sm text-neutral-400 font-medium dark:text-neutral-400">
Current voice: {{ currentSpeechVoiceId }}
<span v-if="currentVoicePackCostMultiplier">· {{ currentVoicePackCostMultiplier }}</span>
</span>
</div>
</div>
<!-- Loading state -->
<div v-if="isLoadingSpeechProviderVoices || (isVoicePackSourceSelected && isLoadingVoicePacks)">
<div v-if="isLoadingSpeechProviderVoices">
<div class="flex flex-col gap-4">
<Skeleton class="w-full rounded-lg p-2.5 text-sm">
<div class="h-1lh" />
@@ -1002,13 +782,6 @@ function handleDeleteProvider(providerId: string) {
/>
</div>
<ErrorContainer
v-else-if="isVoicePackSourceSelected && voicePacksError"
class="mb-2"
:title="t('settings.pages.modules.speech.sections.section.voice-pack.error')"
:error="voicePacksError"
/>
<ErrorContainer
v-else-if="speechProviderError"
class="mb-2"
@@ -1051,7 +824,7 @@ function handleDeleteProvider(providerId: string) {
<!-- Manual voice input when no voices are available or for OpenAI Compatible -->
<div
v-if="!isVoicePackSourceSelected && (activeSpeechProvider === 'openai-compatible-audio-speech' || !availableVoices[activeSpeechProvider] || availableVoices[activeSpeechProvider].length === 0)"
v-if="activeSpeechProvider === 'openai-compatible-audio-speech' || !availableVoices[activeSpeechProvider] || availableVoices[activeSpeechProvider].length === 0"
class="mt-2 space-y-6"
>
<FieldInput
packages/stage-ui/src/components/scenes/Stage.vue
+5 -31
View File
@@ -6,8 +6,6 @@ import type { UnElevenLabsOptions } from 'unspeech'
import type { EmotionPayload } from '../../constants/emotions'
import type { SpeechTransport, StageTtsSession, StreamingSessionSnapshot } from '../../libs/speech/tts-session'
import type { VoicePackSnapshot } from '../../stores/modules/airi-card'
import type { VoiceInfo } from '../../stores/providers'
import { sleep } from '@moeru/std'
import { createLive2DLipSync } from '@proj-airi/model-driver-lipsync'
@@ -43,7 +41,7 @@ import { useBackgroundStore } from '../../stores/background'
import { useChatOrchestratorStore } from '../../stores/chat'
import { useLlmStreamingControlStore } from '../../stores/llm-streaming-control'
import { useAiriCardStore } from '../../stores/modules'
import { useSpeechStore, voicePackForSpeechProvider } from '../../stores/modules/speech'
import { useSpeechStore } from '../../stores/modules/speech'
import { useProvidersStore } from '../../stores/providers'
import { useSettings } from '../../stores/settings'
import { useSpeechOutputControlStore } from '../../stores/speech-output-control'
@@ -332,24 +330,10 @@ const playbackManager = createPlaybackManager<AudioBuffer>({
ownerOverflowPolicy: 'steal-oldest',
})
function createVoicePackVoice(voicePack: VoicePackSnapshot): VoiceInfo {
return {
id: voicePack.voiceId,
name: voicePack.name,
description: voicePack.name,
previewURL: '',
languages: [{ code: 'en', title: 'English' }],
provider: activeSpeechProvider.value,
gender: 'neutral',
}
}
/**
* Classifies chat auto-TTS voice usage before forwarding analytics to the server.
*/
function resolveStageVoiceType(voicePack: VoicePackSnapshot | undefined): 'official_selected' | 'custom_configured' | 'voice_pack' {
if (voicePack)
return 'voice_pack'
function resolveStageVoiceType(): 'official_selected' | 'custom_configured' {
return activeSpeechProvider.value === OFFICIAL_SPEECH_PROVIDER_ID || activeSpeechProvider.value === OFFICIAL_SPEECH_STREAMING_PROVIDER_ID ? 'official_selected' : 'custom_configured'
}
@@ -434,29 +418,19 @@ const speechPipeline = createSpeechPipeline<AudioBuffer>({
}
}
const voicePack = voicePackForSpeechProvider(activeSpeechProvider.value, activeCard.value?.extensions.airi.modules.speech.voicePack)
if (voicePack) {
model = 'auto'
if (!voice || voice.id !== voicePack.voiceId)
voice = createVoicePackVoice(voicePack)
}
if (!model || !voice)
return null
try {
const speechRequest = speechStore.resolveVoicePackSpeechInput({
const speechRequest = speechStore.resolveSpeechInput({
text: request.text,
voice,
providerConfig: {
...providerConfig,
pitch: ssmlEnabled.value ? pitch.value : undefined,
},
params: voicePack?.params,
voicePack,
forceSSML: ssmlEnabled.value,
supportsSSML: speechStore.supportsSSML,
supportsAdapterProsody: activeSpeechProvider.value === OFFICIAL_SPEECH_PROVIDER_ID,
})
// Non-streaming providers only: synth via REST. Streaming provider
@@ -470,7 +444,7 @@ const speechPipeline = createSpeechPipeline<AudioBuffer>({
airi_analytics: {
trigger: 'auto',
source: 'chat_auto_tts',
voice_type: resolveStageVoiceType(voicePack),
voice_type: resolveStageVoiceType(),
},
},
}
@@ -685,7 +659,7 @@ function buildStreamingSnapshot(): StreamingSessionSnapshot | null {
return {
model: sessionModel,
voice: voiceId,
voiceType: resolveStageVoiceType(undefined),
voiceType: resolveStageVoiceType(),
bufferEntireSession,
extraBody: {
api_resource_id: apiResourceId,
packages/stage-ui/src/stores/modules/airi-card.test.ts
+6 -47
View File
@@ -1,7 +1,6 @@
import { createPinia, setActivePinia } from 'pinia'
import { beforeEach, describe, expect, it, vi } from 'vitest'
import { OFFICIAL_SPEECH_PROVIDER_ID } from '../../libs/providers/providers/official'
import { useSettingsStageModel } from '../settings/stage-model'
import { useAiriCardStore } from './airi-card'
@@ -108,57 +107,17 @@ describe('airi-card store', () => {
/**
* @example
* it('freezes a Voice Pack snapshot on the active card', () => {})
* it('updates speech config on the active card', () => {})
*/
it('freezes a Voice Pack snapshot on the active card', () => {
it('updates speech config on the active card', () => {
const cardStore = useAiriCardStore()
cardStore.initialize()
const pack = {
id: 'vp-1',
name: 'Neuro Sama',
voiceId: 'voice-neuro',
params: { pitch: '+20%', volume: '+5%' },
costMultiplier: 1.5,
}
const bound = cardStore.bindVoicePackToActiveCard(pack)
expect(bound).toBe(true)
expect(cardStore.updateActiveCardSpeech({ provider: 'elevenlabs', model: 'eleven_multilingual_v2', voice_id: 'aria' })).toBe(true)
expect(cardStore.activeCard?.extensions.airi.modules.speech).toMatchObject({
provider: OFFICIAL_SPEECH_PROVIDER_ID,
model: 'auto',
voice_id: 'voice-neuro',
voicePack: {
packId: 'vp-1',
name: 'Neuro Sama',
voiceId: 'voice-neuro',
params: { pitch: '+20%', volume: '+5%' },
costMultiplier: 1.5,
},
provider: 'elevenlabs',
model: 'eleven_multilingual_v2',
voice_id: 'aria',
})
})
/**
* @example
* it('keeps the frozen Voice Pack independent from later library edits', () => {})
*/
it('keeps the frozen Voice Pack independent from later library edits', () => {
const cardStore = useAiriCardStore()
cardStore.initialize()
const params = { pitch: '+20%' }
cardStore.bindVoicePackToActiveCard({
id: 'vp-1',
name: 'Frozen',
voiceId: 'voice-a',
params,
costMultiplier: 2,
})
params.pitch = '-10%'
expect(cardStore.activeCard?.extensions.airi.modules.speech.voicePack?.params).toEqual({ pitch: '+20%' })
expect(cardStore.activeCard?.extensions.airi.modules.speech.voicePack?.voiceId).toBe('voice-a')
})
})
packages/stage-ui/src/stores/modules/airi-card.ts
+6 -80
View File
@@ -10,7 +10,6 @@ import { useI18n } from 'vue-i18n'
import SystemPromptV2 from '../../constants/prompts/system-v2'
import { DEFAULT_ARTISTRY_WIDGET_SPAWNING_PROMPT } from '../../constants/prompts/character-defaults'
import { OFFICIAL_SPEECH_PROVIDER_ID } from '../../libs/providers/providers/official'
import { capturePosthogEvent } from '../analytics/posthog'
import { useSettingsStageModel } from '../settings/stage-model'
import { useArtistryStore } from './artistry'
@@ -18,24 +17,6 @@ import { useConsciousnessStore } from './consciousness'
import { useSpeechStore } from './speech'
import { useVisionStore } from './vision'
export type VoicePackParams = Record<string, string | number | boolean | null>
export interface VoicePackBindingInput {
id: string
name: string
voiceId: string
params: VoicePackParams
costMultiplier: number
}
export interface VoicePackSnapshot {
packId: string
name: string
voiceId: string
params: VoicePackParams
costMultiplier: number
}
export interface AiriExtension {
modules: {
consciousness: {
@@ -57,7 +38,6 @@ export interface AiriExtension {
rate?: number
ssml?: boolean
language?: string
voicePack?: VoicePackSnapshot
}
vrm?: {
@@ -201,19 +181,12 @@ export const useAiriCardStore = defineStore('airi-card', () => {
}
function updateActiveCardSpeech(speech: Pick<AiriExtension['modules']['speech'], 'provider' | 'model' | 'voice_id'>) {
return updateActiveCardModules(({ modules }) => {
const existingVoicePack = modules.speech.voicePack
const shouldKeepVoicePack = speech.provider === OFFICIAL_SPEECH_PROVIDER_ID
&& existingVoicePack?.voiceId === speech.voice_id
return {
speech: {
...modules.speech,
...speech,
voicePack: shouldKeepVoicePack ? existingVoicePack : undefined,
},
}
})
return updateActiveCardModules(({ modules }) => ({
speech: {
...modules.speech,
...speech,
},
}))
}
function resolveAiriExtension(card: Card | ccv3.CharacterCardV3): AiriExtension {
@@ -279,7 +252,6 @@ export const useAiriCardStore = defineStore('airi-card', () => {
rate: existingExtension.modules?.speech?.rate,
ssml: existingExtension.modules?.speech?.ssml,
language: existingExtension.modules?.speech?.language,
voicePack: existingExtension.modules?.speech?.voicePack,
},
vrm: existingExtension.modules?.vrm,
live2d: existingExtension.modules?.live2d,
@@ -351,50 +323,6 @@ export const useAiriCardStore = defineStore('airi-card', () => {
}
}
function bindVoicePackToActiveCard(pack: VoicePackBindingInput) {
const cardId = activeCardId.value
const card = cards.value.get(cardId)
if (!card)
return false
const extension = resolveAiriExtension(card)
const voicePack: VoicePackSnapshot = {
packId: pack.id,
name: pack.name,
voiceId: pack.voiceId,
params: { ...pack.params },
costMultiplier: pack.costMultiplier,
}
const speech: AiriExtension['modules']['speech'] = {
...extension.modules.speech,
provider: OFFICIAL_SPEECH_PROVIDER_ID,
model: 'auto',
voice_id: pack.voiceId,
voicePack,
}
cards.value.set(cardId, {
...card,
extensions: {
...card.extensions,
airi: {
...extension,
modules: {
...extension.modules,
speech,
},
},
},
})
activeSpeechProvider.value = OFFICIAL_SPEECH_PROVIDER_ID
activeSpeechModel.value = 'auto'
activeSpeechVoiceId.value = pack.voiceId
return true
}
function initialize() {
if (cards.value.has('default'))
return
@@ -462,7 +390,6 @@ export const useAiriCardStore = defineStore('airi-card', () => {
addCard,
removeCard,
updateCard,
bindVoicePackToActiveCard,
updateActiveCardConsciousness,
updateActiveCardDisplayModel,
updateActiveCardSpeech,
@@ -485,7 +412,6 @@ export const useAiriCardStore = defineStore('airi-card', () => {
provider: activeSpeechProvider.value,
model: activeSpeechModel.value,
voice_id: activeSpeechVoiceId.value,
voicePack: activeCard.value?.extensions?.airi?.modules?.speech?.voicePack,
},
displayModelId: stageModelStore.stageModelSelected,
activeBackgroundId: activeCard.value?.extensions?.airi?.modules?.activeBackgroundId,
packages/stage-ui/src/stores/modules/speech.test.ts
+10 -162
View File
@@ -3,7 +3,7 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'
import { OFFICIAL_SPEECH_PROVIDER_ID, OFFICIAL_SPEECH_STREAMING_PROVIDER_ID, providerOfficialSpeech } from '../../libs/providers/providers/official'
import { useProvidersStore } from '../providers'
import { toSignedPercent, useSpeechStore, voicePackForSpeechProvider } from './speech'
import { toSignedPercent, useSpeechStore } from './speech'
const i18nState = vi.hoisted(() => ({
locale: { value: 'en-US' },
@@ -37,9 +37,9 @@ describe('speech store helpers', () => {
/**
* @example
* speechStore.resolveVoicePackSpeechInput({ text, voice, providerConfig: { voice: 'plain' } })
* speechStore.resolveSpeechInput({ text, voice, providerConfig: { voice: 'plain' } })
*/
it('leaves speech input unchanged when no Voice Pack is configured', () => {
it('leaves speech input unchanged by default', () => {
const speechStore = useSpeechStore()
const voice = {
id: 'plain-voice',
@@ -48,7 +48,7 @@ describe('speech store helpers', () => {
languages: [{ code: 'en-US', title: 'English' }],
}
const request = speechStore.resolveVoicePackSpeechInput({
const request = speechStore.resolveSpeechInput({
text: 'hello',
voice,
providerConfig: { voice: 'plain-voice' },
@@ -58,52 +58,7 @@ describe('speech store helpers', () => {
expect(request.providerConfig).toEqual({ voice: 'plain-voice' })
})
/**
* @example
* voicePackForSpeechProvider('openai-compatible-audio-speech', voicePack)
*/
it('ignores Voice Pack snapshots for non-official speech providers', () => {
const voicePack = {
packId: 'vp-1',
name: 'Frozen',
voiceId: 'voice-a',
params: {},
costMultiplier: 1,
}
expect(voicePackForSpeechProvider('openai-compatible-audio-speech', voicePack)).toBeUndefined()
expect(voicePackForSpeechProvider(OFFICIAL_SPEECH_PROVIDER_ID, voicePack)).toBe(voicePack)
expect(voicePackForSpeechProvider(OFFICIAL_SPEECH_STREAMING_PROVIDER_ID, voicePack)).toBeUndefined()
})
/**
* @example
* speechStore.resolveVoicePackSpeechInput({ text, voice, params: { rate: '+20%' } })
*/
it('maps Voice Pack rate params to provider speed', () => {
const speechStore = useSpeechStore()
const voice = {
id: 'voice-1',
name: 'Voice 1',
provider: OFFICIAL_SPEECH_PROVIDER_ID,
languages: [{ code: 'en-US', title: 'English' }],
}
const request = speechStore.resolveVoicePackSpeechInput({
text: 'hello',
voice,
params: { rate: '+20%' },
})
expect(request.input).toBe('hello')
expect(request.providerConfig.speed).toBe(1.2)
})
/**
* @example
* speechStore.resolveVoicePackSpeechInput({ text, voice, params: { pitch: '+20%' }, supportsSSML: true })
*/
it('applies Voice Pack prosody params through SSML when supported', () => {
it('applies configured pitch through SSML when supported', () => {
const speechStore = useSpeechStore()
const voice = {
id: 'voice-1',
@@ -113,24 +68,21 @@ describe('speech store helpers', () => {
gender: 'neutral',
}
const request = speechStore.resolveVoicePackSpeechInput({
const request = speechStore.resolveSpeechInput({
text: 'hello',
voice,
params: {
pitch: '+20%',
volume: '-5%',
},
providerConfig: { pitch: 20 },
forceSSML: true,
supportsSSML: true,
})
expect(request.input).toContain('<prosody')
expect(request.input).toContain('pitch="+20%"')
expect(request.input).toContain('volume="-5%"')
})
/**
* @example
* speechStore.resolveVoicePackSpeechInput({ text, voice, forceSSML: true, supportsSSML: false, supportsAdapterProsody: true })
* speechStore.resolveSpeechInput({ text, voice, forceSSML: true, supportsSSML: false })
*/
it('keeps official adapter-backed speech input as plain text when global SSML is enabled', () => {
const speechStore = useSpeechStore()
@@ -148,122 +100,18 @@ describe('speech store helpers', () => {
// speech provider to DashScope CosyVoice. DashScope rejects `<speak>...`
// payloads with `SSML text is not supported at the moment!`, so providers
// that apply prosody through adapter options must keep the text field plain.
const request = speechStore.resolveVoicePackSpeechInput({
const request = speechStore.resolveSpeechInput({
text: 'hello',
voice,
providerConfig: { pitch: 0 },
forceSSML: true,
supportsSSML: false,
supportsAdapterProsody: true,
})
expect(request.input).toBe('hello')
expect(request.input).not.toContain('<speak')
})
/**
* @example
* speechStore.resolveVoicePackSpeechInput({ text, voice, params: { pitch: '+20%' }, supportsAdapterProsody: true })
*/
it('passes Voice Pack prosody params through adapter options when supported', () => {
const speechStore = useSpeechStore()
const voice = {
id: 'voice-1',
name: 'Voice 1',
provider: OFFICIAL_SPEECH_PROVIDER_ID,
languages: [{ code: 'en-US', title: 'English' }],
}
const request = speechStore.resolveVoicePackSpeechInput({
text: 'hello',
voice,
params: {
pitch: '+20%',
volume: '+5%',
},
supportsAdapterProsody: true,
})
expect(request.input).toBe('hello')
expect(request.providerConfig.extraBody).toEqual({
voice_pack: {
pitch: 20,
volume: 5,
},
})
})
/**
* @example
* speechStore.resolveVoicePackSpeechInput({ text, voice, voicePack: { packId: 'vp-1' } })
*/
it('passes only Voice Pack identity through adapter options', () => {
const speechStore = useSpeechStore()
const voice = {
id: 'voice-1',
name: 'Voice 1',
provider: OFFICIAL_SPEECH_PROVIDER_ID,
languages: [{ code: 'en-US', title: 'English' }],
}
const request = speechStore.resolveVoicePackSpeechInput({
text: 'hello',
voice,
params: {},
voicePack: {
packId: 'vp-1',
},
supportsAdapterProsody: true,
})
expect(request.providerConfig.extraBody).toEqual({
voice_pack: {
pack_id: 'vp-1',
},
})
})
/**
* @example
* speechStore.resolveVoicePackSpeechInput({ text, voice, params: { pitch: '+20%' }, supportsSSML: false })
*/
it('fails fast when Voice Pack prosody params cannot be applied', () => {
const speechStore = useSpeechStore()
const voice = {
id: 'voice-1',
name: 'Voice 1',
provider: OFFICIAL_SPEECH_PROVIDER_ID,
languages: [{ code: 'en-US', title: 'English' }],
}
expect(() => speechStore.resolveVoicePackSpeechInput({
text: 'hello',
voice,
params: { pitch: '+20%' },
supportsSSML: false,
})).toThrow('SSML-capable speech provider')
})
/**
* @example
* speechStore.resolveVoicePackSpeechInput({ text, voice, params: { emotion: 'happy' } })
*/
it('fails fast on unsupported Voice Pack params', () => {
const speechStore = useSpeechStore()
const voice = {
id: 'voice-1',
name: 'Voice 1',
provider: OFFICIAL_SPEECH_PROVIDER_ID,
languages: [{ code: 'en-US', title: 'English' }],
}
expect(() => speechStore.resolveVoicePackSpeechInput({
text: 'hello',
voice,
params: { emotion: 'happy' },
})).toThrow('Unsupported Voice Pack parameter "emotion"')
})
/**
* @example
* await speechStore.loadVoicesForProvider(OFFICIAL_SPEECH_STREAMING_PROVIDER_ID, 'volcengine/seed-tts-2.0')
packages/stage-ui/src/stores/modules/speech.ts
+7 -163
View File
@@ -1,7 +1,6 @@
import type { SpeechProviderWithExtraOptions } from '@xsai-ext/providers/utils'
import type { VoiceInfo } from '../providers'
import type { VoicePackParams, VoicePackSnapshot } from './airi-card'
import { errorMessageFrom } from '@moeru/std'
import { useLocalStorageManualReset } from '@proj-airi/stage-shared/composables'
@@ -24,121 +23,19 @@ export function toSignedPercent(value: number): string {
return '0%'
}
interface VoicePackSpeechInputOptions {
interface SpeechInputOptions {
text: string
voice: VoiceInfo
providerConfig?: Record<string, unknown>
params?: VoicePackParams
voicePack?: Pick<VoicePackSnapshot, 'packId'>
forceSSML?: boolean
supportsSSML?: boolean
supportsAdapterProsody?: boolean
}
interface VoicePackSpeechInput {
interface SpeechInput {
input: string
providerConfig: Record<string, unknown>
}
const voicePackSupportedParams = new Set(['pitch', 'rate', 'volume'])
export function voicePackForSpeechProvider(
providerId: string | undefined,
voicePack: VoicePackSnapshot | undefined,
): VoicePackSnapshot | undefined {
return providerId === OFFICIAL_SPEECH_PROVIDER_ID ? voicePack : undefined
}
/**
* Normalizes a Voice Pack percent-style option.
*
* Before:
* - "+20%"
* - "-10%"
* - 15
*
* After:
* - 20
* - -10
* - 15
*/
function normalizePercentOption(value: string | number | boolean | null | undefined, name: string): number | undefined {
if (value == null)
return undefined
if (typeof value === 'number') {
if (Number.isFinite(value))
return value
throw new Error(`Voice Pack parameter "${name}" must be a finite number.`)
}
if (typeof value !== 'string')
throw new Error(`Voice Pack parameter "${name}" must be a number or percent string.`)
const trimmed = value.trim()
const normalized = trimmed.endsWith('%') ? trimmed.slice(0, -1) : trimmed
const parsed = Number(normalized)
if (!Number.isFinite(parsed))
throw new Error(`Voice Pack parameter "${name}" must be a number or percent string.`)
return parsed
}
/**
* Normalizes a Voice Pack rate option into provider speed.
*
* Before:
* - "+20%"
* - "-10%"
* - 1.2
*
* After:
* - 1.2
* - 0.9
* - 1.2
*/
function normalizeRateOption(value: string | number | boolean | null | undefined): number | undefined {
if (value == null)
return undefined
if (typeof value === 'number') {
if (Number.isFinite(value) && value > 0)
return value
throw new Error('Voice Pack parameter "rate" must be a positive finite number or percent string.')
}
if (typeof value !== 'string')
throw new Error('Voice Pack parameter "rate" must be a positive finite number or percent string.')
const trimmed = value.trim()
if (trimmed.endsWith('%')) {
const percent = normalizePercentOption(trimmed, 'rate')
const speed = 1 + (percent ?? 0) / 100
if (speed > 0)
return speed
throw new Error('Voice Pack parameter "rate" percent must resolve to a positive speed.')
}
const parsed = Number(trimmed)
if (Number.isFinite(parsed) && parsed > 0)
return parsed
throw new Error('Voice Pack parameter "rate" must be a positive finite number or percent string.')
}
function assertSupportedVoicePackParams(params: VoicePackParams | undefined) {
if (!params)
return
for (const [key, value] of Object.entries(params)) {
if (value == null)
continue
if (!voicePackSupportedParams.has(key))
throw new Error(`Unsupported Voice Pack parameter "${key}".`)
}
}
export const useSpeechStore = defineStore('speech', () => {
const providersStore = useProvidersStore()
const { allAudioSpeechProvidersMetadata } = storeToRefs(providersStore)
@@ -487,67 +384,14 @@ export const useSpeechStore = defineStore('speech', () => {
return toXml(ssmlXast)
}
function resolveVoicePackSpeechInput(options: VoicePackSpeechInputOptions): VoicePackSpeechInput {
function resolveSpeechInput(options: SpeechInputOptions): SpeechInput {
const providerConfig = { ...options.providerConfig }
const canUseSSML = options.supportsSSML === true
if (!options.params) {
return {
input: options.forceSSML === true && canUseSSML
? generateSSML(options.text, options.voice, providerConfig)
: options.text,
providerConfig,
}
}
assertSupportedVoicePackParams(options.params)
const pitch = normalizePercentOption(options.params.pitch, 'pitch')
const volume = normalizePercentOption(options.params.volume, 'volume')
const speed = normalizeRateOption(options.params.rate)
const needsProsody = pitch != null || volume != null
if (speed != null)
providerConfig.speed = speed
const shouldUseSSML = canUseSSML && (options.forceSSML === true || (needsProsody && !options.supportsAdapterProsody))
if (options.voicePack) {
providerConfig.extraBody = {
...(providerConfig.extraBody as Record<string, unknown> | undefined),
voice_pack: {
pack_id: options.voicePack.packId,
...(needsProsody && options.supportsAdapterProsody
? { pitch, volume }
: {}),
},
}
}
else if (needsProsody && options.supportsAdapterProsody) {
providerConfig.extraBody = {
...(providerConfig.extraBody as Record<string, unknown> | undefined),
voice_pack: { pitch, volume },
}
}
else if (needsProsody && !options.supportsAdapterProsody && !shouldUseSSML) {
throw new Error('Voice Pack pitch and volume parameters require an SSML-capable speech provider.')
}
if (!shouldUseSSML && (!needsProsody || options.supportsAdapterProsody)) {
return {
input: options.text,
providerConfig,
}
}
const ssmlConfig = { ...providerConfig }
if (pitch != null)
ssmlConfig.pitch = pitch
if (volume != null)
ssmlConfig.volume = volume
return {
input: generateSSML(options.text, options.voice, ssmlConfig),
input: options.forceSSML === true && canUseSSML
? generateSSML(options.text, options.voice, providerConfig)
: options.text,
providerConfig,
}
}
@@ -617,7 +461,7 @@ export const useSpeechStore = defineStore('speech', () => {
ensureStreamingDefaultModel,
ensureActiveSpeechModel,
generateSSML,
resolveVoicePackSpeechInput,
resolveSpeechInput,
resetState,
}
})
packages/stage-ui/src/stores/voice-packs.ts
+10 -3
View File
@@ -1,5 +1,3 @@
import type { VoicePackBindingInput } from './modules/airi-card'
import { errorMessageFrom } from '@moeru/std'
import { defineStore } from 'pinia'
import { ref } from 'vue'
@@ -7,8 +5,17 @@ import { ref } from 'vue'
import { authedFetch } from '../libs/auth-fetch'
import { SERVER_URL } from '../libs/server'
export type VoicePackListItem = VoicePackBindingInput & {
export interface VoicePackListItem {
id: string
name: string
description: string | null
voiceId: string
params: {
pitch?: number
volume?: number
rate?: number
}
costMultiplier: number
enabled: boolean
createdAt: string
updatedAt: string