feat: add upstreamVoiceId to voice packs and related services

- Introduced upstreamVoiceId field in voice pack schema and database.
- Updated voice pack service to handle upstreamVoiceId in CRUD operations.
- Modified API routes and tests to accommodate upstreamVoiceId.
- Enhanced UI components to include upstreamVoiceId in forms and displays.
- Adjusted speech processing logic to utilize upstreamVoiceId where applicable.
- Updated related tests to ensure proper functionality with new field.
This commit is contained in:
RainbowBird
2026-07-01 16:07:33 +08:00
parent 1f1ef49eed
commit c2aee68513
20 changed files with 3221 additions and 96 deletions
@@ -0,0 +1,3 @@
ALTER TABLE "voice_packs" ADD COLUMN "upstream_voice_id" text;
UPDATE "voice_packs" SET "upstream_voice_id" = "voice_id";
ALTER TABLE "voice_packs" ALTER COLUMN "upstream_voice_id" SET NOT NULL;
File diff suppressed because it is too large Load Diff
+8 -1
View File
@@ -113,6 +113,13 @@
"when": 1780498188308, "when": 1780498188308,
"tag": "0015_concerned_piledriver", "tag": "0015_concerned_piledriver",
"breakpoints": true "breakpoints": true
},
{
"idx": 16,
"version": "7",
"when": 1782847276369,
"tag": "0016_tired_dagger",
"breakpoints": true
} }
] ]
} }
@@ -25,6 +25,7 @@ function createService() {
provider: 'volcengine', provider: 'volcengine',
model: 'seed-tts-2.0', model: 'seed-tts-2.0',
voiceId: 'voice-neuro', voiceId: 'voice-neuro',
upstreamVoiceId: 'voice-neuro-upstream',
ttsModelId: 'volcengine/neuro-pool', ttsModelId: 'volcengine/neuro-pool',
params: {}, params: {},
costMultiplier: 1.5, costMultiplier: 1.5,
@@ -119,6 +120,7 @@ describe('admin voice packs — CRUD', () => {
provider: 'volcengine', provider: 'volcengine',
model: 'seed-tts-2.0', model: 'seed-tts-2.0',
voiceId: 'voice-neuro', voiceId: 'voice-neuro',
upstreamVoiceId: 'voice-neuro-upstream',
ttsModelId: 'volcengine/neuro-pool', ttsModelId: 'volcengine/neuro-pool',
params: { pitch: '+20%' }, params: { pitch: '+20%' },
costMultiplier: 1.5, costMultiplier: 1.5,
@@ -149,6 +151,7 @@ describe('admin voice packs — CRUD', () => {
provider: 'volcengine', provider: 'volcengine',
model: 'seed-tts-2.0', model: 'seed-tts-2.0',
voiceId: 'voice-neuro', voiceId: 'voice-neuro',
upstreamVoiceId: 'voice-neuro-upstream',
ttsModelId: 'volcengine/neuro-pool', ttsModelId: 'volcengine/neuro-pool',
params: {}, params: {},
costMultiplier: -1, costMultiplier: -1,
@@ -710,7 +710,8 @@ describe('v1CompletionsRoutes', () => {
description: null, description: null,
provider: 'azure', provider: 'azure',
model: 'microsoft/v1', model: 'microsoft/v1',
voiceId: 'en-US-AvaMultilingualNeural', voiceId: 'friendly-azure',
upstreamVoiceId: 'en-US-AvaMultilingualNeural',
ttsModelId: 'microsoft/v1', ttsModelId: 'microsoft/v1',
params: {}, params: {},
costMultiplier: 1.5, costMultiplier: 1.5,
@@ -728,12 +729,11 @@ describe('v1CompletionsRoutes', () => {
body: JSON.stringify({ body: JSON.stringify({
model: 'auto', model: 'auto',
input: 'test', input: 'test',
voice: 'en-US-AvaMultilingualNeural', voice: 'friendly-azure',
speed: 1.2, speed: 1.2,
extra_body: { extra_body: {
voice_pack: { voice_pack: {
pack_id: 'vp-azure', pack_id: 'vp-azure',
cost_multiplier: 1.5,
pitch: 20, pitch: 20,
volume: 5, volume: 5,
}, },
@@ -784,7 +784,7 @@ describe('v1CompletionsRoutes', () => {
/** /**
* @example * @example
* POST /api/v1/audio/speech { "input": "hello", "extra_body": { "voice_pack": { "cost_multiplier": 2 } } } * POST /api/v1/audio/speech { "input": "hello", "extra_body": { "voice_pack": { "pack_id": "vp-premium" } } }
*/ */
it('uses Voice Pack cost multiplier for affordability and billing units', async () => { it('uses Voice Pack cost multiplier for affordability and billing units', async () => {
globalThis.fetch = vi.fn(async () => new Response(new Uint8Array([1]), { globalThis.fetch = vi.fn(async () => new Response(new Uint8Array([1]), {
@@ -801,6 +801,7 @@ describe('v1CompletionsRoutes', () => {
provider: 'azure', provider: 'azure',
model: 'microsoft/v1', model: 'microsoft/v1',
voiceId: 'alloy', voiceId: 'alloy',
upstreamVoiceId: 'upstream-alloy',
ttsModelId: 'tts-1', ttsModelId: 'tts-1',
params: {}, params: {},
costMultiplier: 2, costMultiplier: 2,
@@ -832,7 +833,6 @@ describe('v1CompletionsRoutes', () => {
extra_body: { extra_body: {
voice_pack: { voice_pack: {
pack_id: 'vp-premium', pack_id: 'vp-premium',
cost_multiplier: 2,
}, },
}, },
}), }),
@@ -868,6 +868,7 @@ describe('v1CompletionsRoutes', () => {
provider: 'azure', provider: 'azure',
model: 'microsoft/v1', model: 'microsoft/v1',
voiceId: 'alloy', voiceId: 'alloy',
upstreamVoiceId: 'upstream-alloy',
ttsModelId: 'tts-1', ttsModelId: 'tts-1',
params: {}, params: {},
costMultiplier: 2, costMultiplier: 2,
@@ -899,7 +900,6 @@ describe('v1CompletionsRoutes', () => {
extra_body: { extra_body: {
voice_pack: { voice_pack: {
pack_id: 'vp-premium', pack_id: 'vp-premium',
cost_multiplier: 2,
}, },
airi_analytics: { airi_analytics: {
source: 'manual_preview', source: 'manual_preview',
+15 -1
View File
@@ -5,6 +5,20 @@ import { Hono } from 'hono'
import { authGuard } from '../../middlewares/auth' import { authGuard } from '../../middlewares/auth'
function publicVoicePack(pack: Awaited<ReturnType<VoicePackService['listEnabled']>>[number]) {
return {
id: pack.id,
name: pack.name,
description: pack.description,
voiceId: pack.voiceId,
params: pack.params,
costMultiplier: pack.costMultiplier,
enabled: pack.enabled,
createdAt: pack.createdAt,
updatedAt: pack.updatedAt,
}
}
/** /**
* User-facing Voice Pack routes. * User-facing Voice Pack routes.
* *
@@ -17,6 +31,6 @@ export function createVoicePackRoutes(service: VoicePackService) {
.use('*', authGuard) .use('*', authGuard)
.get('/', async (c) => { .get('/', async (c) => {
const packs = await service.listEnabled() const packs = await service.listEnabled()
return c.json(packs) return c.json(packs.map(publicVoicePack))
}) })
} }
@@ -23,7 +23,21 @@ function createTestApp(service: VoicePackService, user: { id: string } | null) {
function createService() { function createService() {
return { return {
listEnabled: vi.fn(async () => [{ id: 'vp-1', name: 'Enabled', enabled: true }]), listEnabled: vi.fn(async () => [{
id: 'vp-1',
name: 'Enabled',
description: 'Public description',
provider: 'azure',
model: 'microsoft/v1',
voiceId: 'friendly-voice',
upstreamVoiceId: 'en-US-AvaMultilingualNeural',
ttsModelId: 'microsoft/v1',
params: { pitch: '+10%' },
costMultiplier: 2,
enabled: true,
createdAt: new Date('2026-01-01T00:00:00.000Z'),
updatedAt: new Date('2026-01-02T00:00:00.000Z'),
}]),
list: vi.fn(), list: vi.fn(),
create: vi.fn(), create: vi.fn(),
update: vi.fn(), update: vi.fn(),
@@ -50,7 +64,17 @@ describe('voice packs routes', () => {
const res = await app.request('/api/v1/voice-packs') const res = await app.request('/api/v1/voice-packs')
expect(res.status).toBe(200) expect(res.status).toBe(200)
expect(await res.json()).toEqual([{ id: 'vp-1', name: 'Enabled', enabled: true }]) expect(await res.json()).toEqual([{
id: 'vp-1',
name: 'Enabled',
description: 'Public description',
voiceId: 'friendly-voice',
params: { pitch: '+10%' },
costMultiplier: 2,
enabled: true,
createdAt: '2026-01-01T00:00:00.000Z',
updatedAt: '2026-01-02T00:00:00.000Z',
}])
expect(service.listEnabled).toHaveBeenCalled() expect(service.listEnabled).toHaveBeenCalled()
}) })
}) })
+1
View File
@@ -16,6 +16,7 @@ export const voicePacks = pgTable(
provider: text('provider').notNull(), provider: text('provider').notNull(),
model: text('model').notNull(), model: text('model').notNull(),
voiceId: text('voice_id').notNull(), voiceId: text('voice_id').notNull(),
upstreamVoiceId: text('upstream_voice_id').notNull(),
ttsModelId: text('tts_model_id').notNull(), ttsModelId: text('tts_model_id').notNull(),
params: jsonb('params').notNull().$type<VoicePackParams>().default({}), params: jsonb('params').notNull().$type<VoicePackParams>().default({}),
costMultiplier: real('cost_multiplier').notNull().default(1), costMultiplier: real('cost_multiplier').notNull().default(1),
@@ -90,20 +90,23 @@ export function createOpenAiSpeechService(deps: OpenAiSpeechServiceDeps) {
async function handleSpeechRequest(input: OpenAiSpeechRequest): Promise<Response> { async function handleSpeechRequest(input: OpenAiSpeechRequest): Promise<Response> {
const requestId = nanoid() const requestId = nanoid()
let requestModel = typeof input.body.model === 'string' ? input.body.model : 'auto' const requestedModel = typeof input.body.model === 'string' ? input.body.model : 'auto'
let requestModel = requestedModel
const requestVoice = typeof input.body.voice === 'string' ? input.body.voice : undefined
const inputText = typeof input.body.input === 'string' ? input.body.input : '' const inputText = typeof input.body.input === 'string' ? input.body.input : ''
const analytics = ttsAnalyticsContext(input.body) const analytics = ttsAnalyticsContext(input.body)
if (requestModel === 'auto')
requestModel = await deps.configKV.getOrThrow('DEFAULT_TTS_MODEL')
const voicePackRequest = await voicePackRequestOptions(input.body, { const voicePackRequest = await voicePackRequestOptions(input.body, {
model: requestModel, requestedModel,
voice: typeof input.body.voice === 'string' ? input.body.voice : undefined, voice: requestVoice,
voicePackService: deps.voicePackService, voicePackService: deps.voicePackService,
}) })
requestModel = voicePackRequest.model ?? requestModel
if (requestModel === 'auto')
requestModel = await deps.configKV.getOrThrow('DEFAULT_TTS_MODEL')
const routedVoice = voicePackRequest.voice ?? requestVoice
const voiceMetadata = ttsVoiceMetadata({ const voiceMetadata = ttsVoiceMetadata({
voice: typeof input.body.voice === 'string' ? input.body.voice : undefined, voice: requestVoice,
voicePackId: voicePackRequest.voicePackId, voicePackId: voicePackRequest.voicePackId,
voiceType: analytics.voiceType, voiceType: analytics.voiceType,
}) })
@@ -114,7 +117,7 @@ export function createOpenAiSpeechService(deps: OpenAiSpeechServiceDeps) {
userId: input.userId, userId: input.userId,
model: requestModel, model: requestModel,
inputChars: inputText.length, inputChars: inputText.length,
voice: typeof input.body.voice === 'string' ? input.body.voice : undefined, voice: requestVoice,
}).log('tts speech request') }).log('tts speech request')
void deps.productEventService.track({ void deps.productEventService.track({
@@ -171,7 +174,7 @@ export function createOpenAiSpeechService(deps: OpenAiSpeechServiceDeps) {
const ttsInput = { const ttsInput = {
text: inputText, text: inputText,
voice: typeof input.body.voice === 'string' ? input.body.voice : undefined, voice: routedVoice,
speed: typeof input.body.speed === 'number' ? input.body.speed : undefined, speed: typeof input.body.speed === 'number' ? input.body.speed : undefined,
responseFormat: typeof input.body.response_format === 'string' ? input.body.response_format : undefined, responseFormat: typeof input.body.response_format === 'string' ? input.body.response_format : undefined,
extraOptions: voicePackRequest.extraOptions, extraOptions: voicePackRequest.extraOptions,
@@ -405,19 +408,22 @@ function ttsVoiceMetadata(input: {
async function voicePackRequestOptions( async function voicePackRequestOptions(
body: Record<string, unknown>, body: Record<string, unknown>,
context: { context: {
model: string requestedModel: string
voice?: string voice?: string
voicePackService: VoicePackService voicePackService: VoicePackService
}, },
): Promise<{ extraOptions: Record<string, unknown> | undefined, costMultiplier: number, voicePackId?: string }> { ): Promise<{
extraOptions: Record<string, unknown> | undefined
costMultiplier: number
voicePackId?: string
model?: string
voice?: string
}> {
const extraBody = asRecord(body.extra_body) const extraBody = asRecord(body.extra_body)
const voicePackOptions = asRecord(extraBody?.voice_pack) const voicePackOptions = asRecord(extraBody?.voice_pack)
const pitch = readOptionalNumber(voicePackOptions, 'pitch') const pitch = readOptionalNumber(voicePackOptions, 'pitch')
const volume = readOptionalNumber(voicePackOptions, 'volume') const volume = readOptionalNumber(voicePackOptions, 'volume')
const costMultiplier = await resolveVoicePackCostMultiplier(voicePackOptions, context) const voicePack = await resolveVoicePackRequest(voicePackOptions, context)
const voicePackId = typeof voicePackOptions?.pack_id === 'string' && voicePackOptions.pack_id.trim()
? voicePackOptions.pack_id
: undefined
const extraOptions: Record<string, unknown> = {} const extraOptions: Record<string, unknown> = {}
if (pitch != null) if (pitch != null)
extraOptions.pitch = pitch extraOptions.pitch = pitch
@@ -426,40 +432,49 @@ async function voicePackRequestOptions(
return { return {
extraOptions: Object.keys(extraOptions).length > 0 ? extraOptions : undefined, extraOptions: Object.keys(extraOptions).length > 0 ? extraOptions : undefined,
costMultiplier, costMultiplier: voicePack?.costMultiplier ?? 1,
voicePackId, voicePackId: voicePack?.id,
model: voicePack?.ttsModelId,
voice: voicePack?.upstreamVoiceId,
} }
} }
async function resolveVoicePackCostMultiplier( async function resolveVoicePackRequest(
voicePackOptions: Record<string, unknown> | undefined, voicePackOptions: Record<string, unknown> | undefined,
context: { context: {
model: string requestedModel: string
voice?: string voice?: string
voicePackService: VoicePackService voicePackService: VoicePackService
}, },
): Promise<number> { ): Promise<Awaited<ReturnType<VoicePackService['findById']>> | null> {
const packId = voicePackOptions?.pack_id const packId = voicePackOptions?.pack_id
const value = voicePackOptions?.cost_multiplier if (voicePackOptions?.cost_multiplier != null) {
if (packId == null && value == null) throw createBadRequestError('voice_pack.cost_multiplier is server-managed', 'INVALID_VOICE_PACK', {
return 1 field: 'voice_pack.cost_multiplier',
})
}
if (packId == null)
return null
if (typeof packId !== 'string' || !packId.trim()) if (typeof packId !== 'string' || !packId.trim())
throw createBadRequestError('voice_pack.pack_id is required when Voice Pack billing metadata is provided', 'INVALID_VOICE_PACK') throw createBadRequestError('voice_pack.pack_id is required when Voice Pack billing metadata is provided', 'INVALID_VOICE_PACK')
const pack = await context.voicePackService.findById(packId) const pack = await context.voicePackService.findById(packId)
if (!pack) if (!pack)
throw createBadRequestError('Voice Pack not found', 'INVALID_VOICE_PACK', { packId }) throw createBadRequestError('Voice Pack not found', 'INVALID_VOICE_PACK', { packId })
if (pack.ttsModelId !== context.model || pack.voiceId !== context.voice) { if (context.requestedModel !== 'auto' && pack.ttsModelId !== context.requestedModel) {
throw createBadRequestError('Voice Pack does not match requested model and voice', 'INVALID_VOICE_PACK', {
packId,
actualModel: context.requestedModel,
})
}
if (pack.voiceId !== context.voice) {
throw createBadRequestError('Voice Pack does not match requested model and voice', 'INVALID_VOICE_PACK', { throw createBadRequestError('Voice Pack does not match requested model and voice', 'INVALID_VOICE_PACK', {
packId, packId,
expectedModel: pack.ttsModelId,
actualModel: context.model,
expectedVoice: pack.voiceId,
actualVoice: context.voice, actualVoice: context.voice,
}) })
} }
return pack.costMultiplier return pack
} }
function routerFailure(error: unknown): { status: number, reason: string, message: string } { function routerFailure(error: unknown): { status: number, reason: string, message: string } {
@@ -27,6 +27,7 @@ describe('voicePackService', () => {
provider: 'volcengine', provider: 'volcengine',
model: 'seed-tts-2.0', model: 'seed-tts-2.0',
voiceId: 'voice-neuro', voiceId: 'voice-neuro',
upstreamVoiceId: 'voice-neuro-upstream',
ttsModelId: 'volcengine/neuro-pool', ttsModelId: 'volcengine/neuro-pool',
params: { pitch: '+20%', volume: '+5%' }, params: { pitch: '+20%', volume: '+5%' },
costMultiplier: 1.5, costMultiplier: 1.5,
@@ -37,6 +38,7 @@ describe('voicePackService', () => {
expect(pack.provider).toBe('volcengine') expect(pack.provider).toBe('volcengine')
expect(pack.model).toBe('seed-tts-2.0') expect(pack.model).toBe('seed-tts-2.0')
expect(pack.voiceId).toBe('voice-neuro') expect(pack.voiceId).toBe('voice-neuro')
expect(pack.upstreamVoiceId).toBe('voice-neuro-upstream')
expect(pack.ttsModelId).toBe('volcengine/neuro-pool') expect(pack.ttsModelId).toBe('volcengine/neuro-pool')
expect(pack.params).toEqual({ pitch: '+20%', volume: '+5%' }) expect(pack.params).toEqual({ pitch: '+20%', volume: '+5%' })
expect(pack.costMultiplier).toBe(1.5) expect(pack.costMultiplier).toBe(1.5)
@@ -50,6 +52,7 @@ describe('voicePackService', () => {
provider: 'volcengine', provider: 'volcengine',
model: 'seed-tts-2.0', model: 'seed-tts-2.0',
voiceId: 'voice-a', voiceId: 'voice-a',
upstreamVoiceId: 'voice-a-upstream',
ttsModelId: 'volcengine/pool', ttsModelId: 'volcengine/pool',
params: {}, params: {},
costMultiplier: 1, costMultiplier: 1,
@@ -60,6 +63,7 @@ describe('voicePackService', () => {
provider: 'volcengine', provider: 'volcengine',
model: 'seed-tts-2.0', model: 'seed-tts-2.0',
voiceId: 'voice-a', voiceId: 'voice-a',
upstreamVoiceId: 'voice-a-upstream',
ttsModelId: 'volcengine/pool', ttsModelId: 'volcengine/pool',
params: { pitch: '+20%' }, params: { pitch: '+20%' },
costMultiplier: 1, costMultiplier: 1,
@@ -78,6 +82,7 @@ describe('voicePackService', () => {
provider: 'azure', provider: 'azure',
model: 'v1', model: 'v1',
voiceId: 'en-US-AvaMultilingualNeural', voiceId: 'en-US-AvaMultilingualNeural',
upstreamVoiceId: 'en-US-AvaMultilingualNeural',
ttsModelId: 'microsoft/v1', ttsModelId: 'microsoft/v1',
params: {}, params: {},
costMultiplier: 1, costMultiplier: 1,
@@ -103,6 +108,7 @@ describe('voicePackService', () => {
provider: 'dashscope-cosyvoice', provider: 'dashscope-cosyvoice',
model: 'cosyvoice-v2', model: 'cosyvoice-v2',
voiceId: 'longxiaochun_v2', voiceId: 'longxiaochun_v2',
upstreamVoiceId: 'longxiaochun_v2',
ttsModelId: 'alibaba/cosyvoice-v2', ttsModelId: 'alibaba/cosyvoice-v2',
params: {}, params: {},
costMultiplier: 1, costMultiplier: 1,
@@ -24,6 +24,7 @@ export const CreateVoicePackInputSchema = object({
provider: pipe(string(), nonEmpty('provider is required'), maxLength(100)), provider: pipe(string(), nonEmpty('provider is required'), maxLength(100)),
model: pipe(string(), nonEmpty('model is required'), maxLength(200)), model: pipe(string(), nonEmpty('model is required'), maxLength(200)),
voiceId: pipe(string(), nonEmpty('voiceId is required'), maxLength(200)), voiceId: pipe(string(), nonEmpty('voiceId is required'), maxLength(200)),
upstreamVoiceId: pipe(string(), nonEmpty('upstreamVoiceId is required'), maxLength(200)),
ttsModelId: pipe(string(), nonEmpty('ttsModelId is required'), maxLength(200)), ttsModelId: pipe(string(), nonEmpty('ttsModelId is required'), maxLength(200)),
params: optional(VoicePackParamsSchema, {}), params: optional(VoicePackParamsSchema, {}),
costMultiplier: VoicePackCostMultiplierSchema, costMultiplier: VoicePackCostMultiplierSchema,
@@ -36,6 +37,7 @@ export const UpdateVoicePackInputSchema = object({
provider: optional(pipe(string(), nonEmpty('provider must not be empty'), maxLength(100))), provider: optional(pipe(string(), nonEmpty('provider must not be empty'), maxLength(100))),
model: optional(pipe(string(), nonEmpty('model must not be empty'), maxLength(200))), model: optional(pipe(string(), nonEmpty('model must not be empty'), maxLength(200))),
voiceId: optional(pipe(string(), nonEmpty('voiceId must not be empty'), maxLength(200))), voiceId: optional(pipe(string(), nonEmpty('voiceId must not be empty'), maxLength(200))),
upstreamVoiceId: optional(pipe(string(), nonEmpty('upstreamVoiceId must not be empty'), maxLength(200))),
ttsModelId: optional(pipe(string(), nonEmpty('ttsModelId must not be empty'), maxLength(200))), ttsModelId: optional(pipe(string(), nonEmpty('ttsModelId must not be empty'), maxLength(200))),
params: optional(VoicePackParamsSchema), params: optional(VoicePackParamsSchema),
costMultiplier: optional(VoicePackCostMultiplierSchema), costMultiplier: optional(VoicePackCostMultiplierSchema),
@@ -74,6 +76,7 @@ export function createVoicePackService(db: Database) {
provider: input.provider, provider: input.provider,
model: input.model, model: input.model,
voiceId: input.voiceId, voiceId: input.voiceId,
upstreamVoiceId: input.upstreamVoiceId,
ttsModelId: input.ttsModelId, ttsModelId: input.ttsModelId,
params: input.params, params: input.params,
costMultiplier: input.costMultiplier, costMultiplier: input.costMultiplier,
+2
View File
@@ -180,6 +180,7 @@ export interface VoicePack {
provider: string provider: string
model: string model: string
voiceId: string voiceId: string
upstreamVoiceId: string
ttsModelId: string ttsModelId: string
params: VoicePackParams params: VoicePackParams
costMultiplier: number costMultiplier: number
@@ -194,6 +195,7 @@ export interface VoicePackPayload {
provider: string provider: string
model: string model: string
voiceId: string voiceId: string
upstreamVoiceId: string
ttsModelId: string ttsModelId: string
params?: VoicePackParams params?: VoicePackParams
costMultiplier: number costMultiplier: number
+28 -11
View File
@@ -39,6 +39,7 @@ const form = reactive({
provider: '', provider: '',
model: '', model: '',
voiceId: '', voiceId: '',
upstreamVoiceId: '',
ttsModelId: '', ttsModelId: '',
paramsJson: DEFAULT_PARAMS, paramsJson: DEFAULT_PARAMS,
costMultiplier: 1, costMultiplier: 1,
@@ -113,7 +114,9 @@ const formError = computed(() => {
if (!form.ttsModelId.trim()) if (!form.ttsModelId.trim())
return 'TTS model ID is required' return 'TTS model ID is required'
if (!form.voiceId.trim()) if (!form.voiceId.trim())
return 'Voice ID is required' return 'Voice alias is required'
if (!form.upstreamVoiceId.trim())
return 'Upstream voice ID is required'
if (!Number.isFinite(Number(form.costMultiplier)) || Number(form.costMultiplier) < 0) if (!Number.isFinite(Number(form.costMultiplier)) || Number(form.costMultiplier) < 0)
return 'Cost multiplier must be a non-negative number' return 'Cost multiplier must be a non-negative number'
return null return null
@@ -125,7 +128,7 @@ onMounted(async () => {
fillSelectedPack() fillSelectedPack()
else else
resetForm() resetForm()
await loadVoices(form.ttsModelId, { autoPick: !form.voiceId.trim() }) await loadVoices(form.ttsModelId, { autoPick: !form.upstreamVoiceId.trim() })
modelChangeVoiceLoadingEnabled.value = true modelChangeVoiceLoadingEnabled.value = true
}) })
@@ -194,8 +197,11 @@ async function loadVoices(model: string, options: { autoPick: boolean }) {
const result = await adminApi.speechVoices(model.trim()) const result = await adminApi.speechVoices(model.trim())
voices.value = result.voices voices.value = result.voices
recommendedVoices.value = result.recommended recommendedVoices.value = result.recommended
if (options.autoPick && !form.voiceId.trim()) if (options.autoPick && !form.upstreamVoiceId.trim()) {
form.voiceId = firstRecommendedVoiceId(result.recommended) ?? result.voices[0]?.id ?? '' form.upstreamVoiceId = firstRecommendedVoiceId(result.recommended) ?? result.voices[0]?.id ?? ''
if (!form.voiceId.trim())
form.voiceId = form.upstreamVoiceId
}
} }
catch (error) { catch (error) {
voices.value = [] voices.value = []
@@ -223,6 +229,7 @@ function fillForm(pack: VoicePack) {
form.provider = pack.provider form.provider = pack.provider
form.model = pack.model form.model = pack.model
form.voiceId = pack.voiceId form.voiceId = pack.voiceId
form.upstreamVoiceId = pack.upstreamVoiceId
form.ttsModelId = pack.ttsModelId form.ttsModelId = pack.ttsModelId
form.paramsJson = JSON.stringify(pack.params ?? {}, null, 2) form.paramsJson = JSON.stringify(pack.params ?? {}, null, 2)
form.costMultiplier = pack.costMultiplier form.costMultiplier = pack.costMultiplier
@@ -238,6 +245,7 @@ function resetForm() {
form.provider = modelParts.provider form.provider = modelParts.provider
form.model = modelParts.model form.model = modelParts.model
form.voiceId = '' form.voiceId = ''
form.upstreamVoiceId = ''
form.ttsModelId = modelId form.ttsModelId = modelId
form.paramsJson = DEFAULT_PARAMS form.paramsJson = DEFAULT_PARAMS
form.costMultiplier = 1 form.costMultiplier = 1
@@ -279,6 +287,7 @@ function payload(): VoicePackPayload {
provider: form.provider.trim(), provider: form.provider.trim(),
model: form.model.trim(), model: form.model.trim(),
voiceId: form.voiceId.trim(), voiceId: form.voiceId.trim(),
upstreamVoiceId: form.upstreamVoiceId.trim(),
ttsModelId: form.ttsModelId.trim(), ttsModelId: form.ttsModelId.trim(),
params: parseParams(), params: parseParams(),
costMultiplier: Number(form.costMultiplier), costMultiplier: Number(form.costMultiplier),
@@ -340,7 +349,7 @@ async function testVoicePack() {
const body = { const body = {
model: form.ttsModelId.trim(), model: form.ttsModelId.trim(),
input: text, input: text,
voice: form.voiceId.trim(), voice: form.upstreamVoiceId.trim(),
speed: normalizeRateOption(params.rate), speed: normalizeRateOption(params.rate),
extra_body: voicePackExtraBody(params), extra_body: voicePackExtraBody(params),
} }
@@ -494,19 +503,27 @@ function normalizeRateOption(value: string | number | boolean | null | undefined
:placeholder="ttsModelPlaceholder" :placeholder="ttsModelPlaceholder"
required required
/> />
<DatalistField <FieldInput
v-model="form.voiceId" v-model="form.voiceId"
:description="loadingVoices ? 'Loading voices for the selected model...' : 'Voice catalog from /api/v1/audio/voices.'" description="Product-facing Voice Pack voice alias shown to clients and analytics."
input-class="font-mono text-xs" input-class="font-mono text-xs"
label="Voice ID" label="Voice alias"
list-id="voice-pack-voices" placeholder="narrator-cn"
:options="voiceOptions"
:placeholder="voicePlaceholder"
required required
/> />
</div> </div>
<div :class="['grid', 'gap-4', 'md:grid-cols-2']"> <div :class="['grid', 'gap-4', 'md:grid-cols-2']">
<DatalistField
v-model="form.upstreamVoiceId"
:description="loadingVoices ? 'Loading voices for the selected model...' : 'Voice catalog from /api/v1/audio/voices.'"
input-class="font-mono text-xs"
label="Upstream voice ID"
list-id="voice-pack-voices"
:options="voiceOptions"
:placeholder="voicePlaceholder"
required
/>
<DatalistField <DatalistField
v-model="form.provider" v-model="form.provider"
description="Derived from the model ID when possible; editable for custom routing metadata." description="Derived from the model ID when possible; editable for custom routing metadata."
+5 -1
View File
@@ -83,7 +83,8 @@ function editPack(pack: VoicePack) {
<tr> <tr>
<th>Name</th> <th>Name</th>
<th>Routing</th> <th>Routing</th>
<th>Voice</th> <th>Voice Alias</th>
<th>Upstream Voice</th>
<th>Cost</th> <th>Cost</th>
<th>Status</th> <th>Status</th>
<th>Updated</th> <th>Updated</th>
@@ -118,6 +119,9 @@ function editPack(pack: VoicePack) {
<td :class="['text-xs', 'font-mono']"> <td :class="['text-xs', 'font-mono']">
{{ pack.voiceId }} {{ pack.voiceId }}
</td> </td>
<td :class="['text-xs', 'font-mono']">
{{ pack.upstreamVoiceId }}
</td>
<td>{{ formatMultiplier(pack.costMultiplier) }}</td> <td>{{ formatMultiplier(pack.costMultiplier) }}</td>
<td> <td>
<span :class="['badge', pack.enabled ? 'badge-green' : 'badge-amber']"> <span :class="['badge', pack.enabled ? 'badge-green' : 'badge-amber']">
@@ -75,6 +75,8 @@ const errorMessage = ref('')
const selectedSpeechSource = ref<string | null>(null) const selectedSpeechSource = ref<string | null>(null)
const VOICE_PACK_SOURCE_ID = 'voice-pack' const VOICE_PACK_SOURCE_ID = 'voice-pack'
const VOICE_PACK_REQUEST_MODEL_ID = 'auto'
const VOICE_PACK_ANALYTICS_MODEL_ID = 'voice_pack'
const STREAMING_MODEL_OPTION_PREFIX = 'streaming:' const STREAMING_MODEL_OPTION_PREFIX = 'streaming:'
const isOfficialSpeechProvider = computed(() => activeSpeechProvider.value === OFFICIAL_SPEECH_PROVIDER_ID) const isOfficialSpeechProvider = computed(() => activeSpeechProvider.value === OFFICIAL_SPEECH_PROVIDER_ID)
@@ -198,6 +200,15 @@ function createVoicePackVoice(voicePack: VoicePackSnapshot): VoiceInfo {
} }
} }
function formatVoicePackCostMultiplier(costMultiplier: number) {
return `Flux cost: ${costMultiplier}x`
}
function voicePackDescription(description: string | null | undefined, costMultiplier: number) {
const cost = formatVoicePackCostMultiplier(costMultiplier)
return description ? `${description} · ${cost}` : cost
}
function voicePackVoiceId(packId: string) { function voicePackVoiceId(packId: string) {
return `voice-pack:${packId}` return `voice-pack:${packId}`
} }
@@ -211,7 +222,7 @@ const displayedVoiceOptions = computed(() => {
const options = voicePacks.value.map(pack => ({ const options = voicePacks.value.map(pack => ({
id: voicePackVoiceId(pack.id), id: voicePackVoiceId(pack.id),
name: pack.name, name: pack.name,
description: pack.description ?? undefined, description: voicePackDescription(pack.description, pack.costMultiplier),
previewURL: '', previewURL: '',
customizable: false, customizable: false,
})) }))
@@ -221,7 +232,7 @@ const displayedVoiceOptions = computed(() => {
options.unshift({ options.unshift({
id: frozenVoiceId, id: frozenVoiceId,
name: voicePack.name, name: voicePack.name,
description: voicePack.name, description: voicePackDescription(voicePack.name, voicePack.costMultiplier),
previewURL: '', previewURL: '',
customizable: false, customizable: false,
}) })
@@ -265,6 +276,12 @@ const currentSpeechVoiceId = computed(() => {
return activeSpeechVoiceId.value || '' return activeSpeechVoiceId.value || ''
}) })
const currentVoicePackCostMultiplier = computed(() => {
if (isVoicePackSourceSelected.value && boundVoicePack.value)
return formatVoicePackCostMultiplier(boundVoicePack.value.costMultiplier)
return ''
})
function syncBoundVoicePackSelection() { function syncBoundVoicePackSelection() {
const voicePack = boundVoicePack.value const voicePack = boundVoicePack.value
if (!voicePack) if (!voicePack)
@@ -272,7 +289,7 @@ function syncBoundVoicePackSelection() {
selectedSpeechSource.value = VOICE_PACK_SOURCE_ID selectedSpeechSource.value = VOICE_PACK_SOURCE_ID
activeSpeechProvider.value = OFFICIAL_SPEECH_PROVIDER_ID activeSpeechProvider.value = OFFICIAL_SPEECH_PROVIDER_ID
activeSpeechModel.value = voicePack.ttsModelId activeSpeechModel.value = VOICE_PACK_REQUEST_MODEL_ID
activeSpeechVoiceId.value = voicePack.voiceId activeSpeechVoiceId.value = voicePack.voiceId
activeSpeechVoice.value = createVoicePackVoice(voicePack) activeSpeechVoice.value = createVoicePackVoice(voicePack)
return true return true
@@ -283,7 +300,7 @@ function syncBoundVoicePackSelection() {
*/ */
function currentTtsModelId() { function currentTtsModelId() {
if (isVoicePackSourceSelected.value && boundVoicePack.value) if (isVoicePackSourceSelected.value && boundVoicePack.value)
return boundVoicePack.value.ttsModelId return VOICE_PACK_ANALYTICS_MODEL_ID
return activeSpeechModel.value || 'unknown' return activeSpeechModel.value || 'unknown'
} }
@@ -396,7 +413,7 @@ function selectSpeechSource(sourceId: string) {
activeSpeechProvider.value = OFFICIAL_SPEECH_PROVIDER_ID activeSpeechProvider.value = OFFICIAL_SPEECH_PROVIDER_ID
const voicePack = boundVoicePack.value const voicePack = boundVoicePack.value
if (voicePack) { if (voicePack) {
activeSpeechModel.value = voicePack.ttsModelId activeSpeechModel.value = VOICE_PACK_REQUEST_MODEL_ID
activeSpeechVoiceId.value = voicePack.voiceId activeSpeechVoiceId.value = voicePack.voiceId
activeSpeechVoice.value = createVoicePackVoice(voicePack) activeSpeechVoice.value = createVoicePackVoice(voicePack)
return return
@@ -481,12 +498,12 @@ async function bindVoicePack(pack: (typeof voicePacks.value)[number]) {
selectedSpeechSource.value = VOICE_PACK_SOURCE_ID selectedSpeechSource.value = VOICE_PACK_SOURCE_ID
activeSpeechProvider.value = OFFICIAL_SPEECH_PROVIDER_ID activeSpeechProvider.value = OFFICIAL_SPEECH_PROVIDER_ID
activeSpeechModel.value = pack.ttsModelId activeSpeechModel.value = VOICE_PACK_REQUEST_MODEL_ID
activeSpeechVoiceId.value = pack.voiceId activeSpeechVoiceId.value = pack.voiceId
activeSpeechVoice.value = { activeSpeechVoice.value = {
id: pack.voiceId, id: pack.voiceId,
name: pack.name, name: pack.name,
description: pack.description ?? pack.name, description: voicePackDescription(pack.description ?? pack.name, pack.costMultiplier),
previewURL: '', previewURL: '',
languages: [{ code: 'en', title: 'English' }], languages: [{ code: 'en', title: 'English' }],
provider: activeSpeechProvider.value, provider: activeSpeechProvider.value,
@@ -495,14 +512,14 @@ async function bindVoicePack(pack: (typeof voicePacks.value)[number]) {
trackVoicePackBound({ trackVoicePackBound({
tts_provider_id: activeSpeechProvider.value || 'unknown', tts_provider_id: activeSpeechProvider.value || 'unknown',
tts_model_id: pack.ttsModelId, tts_model_id: VOICE_PACK_ANALYTICS_MODEL_ID,
voice_id: pack.voiceId, voice_id: pack.voiceId,
voice_pack_id: pack.id, voice_pack_id: pack.id,
source: 'settings', source: 'settings',
}) })
trackVoiceSelected({ trackVoiceSelected({
tts_provider_id: activeSpeechProvider.value || 'unknown', tts_provider_id: activeSpeechProvider.value || 'unknown',
tts_model_id: pack.ttsModelId, tts_model_id: VOICE_PACK_ANALYTICS_MODEL_ID,
voice_id: pack.voiceId, voice_id: pack.voiceId,
voice_type: 'voice_pack', voice_type: 'voice_pack',
voice_pack_id: pack.id, voice_pack_id: pack.id,
@@ -603,7 +620,7 @@ async function generateTestSpeech() {
const voicePack = boundVoicePack.value const voicePack = boundVoicePack.value
if (voicePack) { if (voicePack) {
model = voicePack.ttsModelId model = VOICE_PACK_REQUEST_MODEL_ID
if (!voice || voice.id !== voicePack.voiceId) if (!voice || voice.id !== voicePack.voiceId)
voice = createVoicePackVoice(voicePack) voice = createVoicePackVoice(voicePack)
} }
@@ -928,7 +945,10 @@ function handleDeleteProvider(providerId: string) {
</h2> </h2>
<div class="flex flex-col items-start gap-1 text-neutral-400 md:flex-row md:items-center md:justify-between dark:text-neutral-500"> <div class="flex flex-col items-start gap-1 text-neutral-400 md:flex-row md:items-center md:justify-between dark:text-neutral-500">
<span>Customize how your AI assistant speaks</span> <span>Customize how your AI assistant speaks</span>
<span v-if="currentSpeechVoiceId" class="text-sm text-neutral-400 font-medium dark:text-neutral-400">Current voice: {{ currentSpeechVoiceId }}</span> <span v-if="currentSpeechVoiceId" class="text-sm text-neutral-400 font-medium dark:text-neutral-400">
Current voice: {{ currentSpeechVoiceId }}
<span v-if="currentVoicePackCostMultiplier">· {{ currentVoicePackCostMultiplier }}</span>
</span>
</div> </div>
</div> </div>
@@ -964,7 +984,6 @@ function handleDeleteProvider(providerId: string) {
<VoiceCardManySelect <VoiceCardManySelect
v-model:search-query="voiceSearchQuery" v-model:search-query="voiceSearchQuery"
v-model:voice-id="displayedSpeechVoiceId" v-model:voice-id="displayedSpeechVoiceId"
:show-visualizer="false"
:voices="displayedVoiceOptions" :voices="displayedVoiceOptions"
:searchable="true" :searchable="true"
:search-placeholder="t('settings.pages.modules.speech.sections.section.provider-voice-selection.search_voices_placeholder')" :search-placeholder="t('settings.pages.modules.speech.sections.section.provider-voice-selection.search_voices_placeholder')"
@@ -436,7 +436,7 @@ const speechPipeline = createSpeechPipeline<AudioBuffer>({
const voicePack = voicePackForSpeechProvider(activeSpeechProvider.value, activeCard.value?.extensions.airi.modules.speech.voicePack) const voicePack = voicePackForSpeechProvider(activeSpeechProvider.value, activeCard.value?.extensions.airi.modules.speech.voicePack)
if (voicePack) { if (voicePack) {
model = voicePack.ttsModelId model = 'auto'
if (!voice || voice.id !== voicePack.voiceId) if (!voice || voice.id !== voicePack.voiceId)
voice = createVoicePackVoice(voicePack) voice = createVoicePackVoice(voicePack)
} }
@@ -117,10 +117,7 @@ describe('airi-card store', () => {
const pack = { const pack = {
id: 'vp-1', id: 'vp-1',
name: 'Neuro Sama', name: 'Neuro Sama',
provider: 'volcengine',
model: 'seed-tts-2.0',
voiceId: 'voice-neuro', voiceId: 'voice-neuro',
ttsModelId: 'volcengine/neuro-pool',
params: { pitch: '+20%', volume: '+5%' }, params: { pitch: '+20%', volume: '+5%' },
costMultiplier: 1.5, costMultiplier: 1.5,
} }
@@ -130,15 +127,12 @@ describe('airi-card store', () => {
expect(bound).toBe(true) expect(bound).toBe(true)
expect(cardStore.activeCard?.extensions.airi.modules.speech).toMatchObject({ expect(cardStore.activeCard?.extensions.airi.modules.speech).toMatchObject({
provider: OFFICIAL_SPEECH_PROVIDER_ID, provider: OFFICIAL_SPEECH_PROVIDER_ID,
model: 'volcengine/neuro-pool', model: 'auto',
voice_id: 'voice-neuro', voice_id: 'voice-neuro',
voicePack: { voicePack: {
packId: 'vp-1', packId: 'vp-1',
name: 'Neuro Sama', name: 'Neuro Sama',
provider: 'volcengine',
model: 'seed-tts-2.0',
voiceId: 'voice-neuro', voiceId: 'voice-neuro',
ttsModelId: 'volcengine/neuro-pool',
params: { pitch: '+20%', volume: '+5%' }, params: { pitch: '+20%', volume: '+5%' },
costMultiplier: 1.5, costMultiplier: 1.5,
}, },
@@ -157,12 +151,9 @@ describe('airi-card store', () => {
cardStore.bindVoicePackToActiveCard({ cardStore.bindVoicePackToActiveCard({
id: 'vp-1', id: 'vp-1',
name: 'Frozen', name: 'Frozen',
provider: 'volcengine',
model: 'seed-tts-2.0',
voiceId: 'voice-a', voiceId: 'voice-a',
ttsModelId: 'volcengine/pool-a',
params, params,
costMultiplier: 1, costMultiplier: 2,
}) })
params.pitch = '-10%' params.pitch = '-10%'
@@ -23,10 +23,7 @@ export type VoicePackParams = Record<string, string | number | boolean | null>
export interface VoicePackBindingInput { export interface VoicePackBindingInput {
id: string id: string
name: string name: string
provider: string
model: string
voiceId: string voiceId: string
ttsModelId: string
params: VoicePackParams params: VoicePackParams
costMultiplier: number costMultiplier: number
} }
@@ -34,10 +31,7 @@ export interface VoicePackBindingInput {
export interface VoicePackSnapshot { export interface VoicePackSnapshot {
packId: string packId: string
name: string name: string
provider: string
model: string
voiceId: string voiceId: string
ttsModelId: string
params: VoicePackParams params: VoicePackParams
costMultiplier: number costMultiplier: number
} }
@@ -210,8 +204,7 @@ export const useAiriCardStore = defineStore('airi-card', () => {
return updateActiveCardModules(({ modules }) => { return updateActiveCardModules(({ modules }) => {
const existingVoicePack = modules.speech.voicePack const existingVoicePack = modules.speech.voicePack
const shouldKeepVoicePack = speech.provider === OFFICIAL_SPEECH_PROVIDER_ID const shouldKeepVoicePack = speech.provider === OFFICIAL_SPEECH_PROVIDER_ID
&& existingVoicePack?.ttsModelId === speech.model && existingVoicePack?.voiceId === speech.voice_id
&& existingVoicePack.voiceId === speech.voice_id
return { return {
speech: { speech: {
@@ -368,10 +361,7 @@ export const useAiriCardStore = defineStore('airi-card', () => {
const voicePack: VoicePackSnapshot = { const voicePack: VoicePackSnapshot = {
packId: pack.id, packId: pack.id,
name: pack.name, name: pack.name,
provider: pack.provider,
model: pack.model,
voiceId: pack.voiceId, voiceId: pack.voiceId,
ttsModelId: pack.ttsModelId,
params: { ...pack.params }, params: { ...pack.params },
costMultiplier: pack.costMultiplier, costMultiplier: pack.costMultiplier,
} }
@@ -379,7 +369,7 @@ export const useAiriCardStore = defineStore('airi-card', () => {
const speech: AiriExtension['modules']['speech'] = { const speech: AiriExtension['modules']['speech'] = {
...extension.modules.speech, ...extension.modules.speech,
provider: OFFICIAL_SPEECH_PROVIDER_ID, provider: OFFICIAL_SPEECH_PROVIDER_ID,
model: pack.ttsModelId, model: 'auto',
voice_id: pack.voiceId, voice_id: pack.voiceId,
voicePack, voicePack,
} }
@@ -399,7 +389,7 @@ export const useAiriCardStore = defineStore('airi-card', () => {
}) })
activeSpeechProvider.value = OFFICIAL_SPEECH_PROVIDER_ID activeSpeechProvider.value = OFFICIAL_SPEECH_PROVIDER_ID
activeSpeechModel.value = pack.ttsModelId activeSpeechModel.value = 'auto'
activeSpeechVoiceId.value = pack.voiceId activeSpeechVoiceId.value = pack.voiceId
return true return true
@@ -66,10 +66,7 @@ describe('speech store helpers', () => {
const voicePack = { const voicePack = {
packId: 'vp-1', packId: 'vp-1',
name: 'Frozen', name: 'Frozen',
provider: 'volcengine',
model: 'seed-tts-2.0',
voiceId: 'voice-a', voiceId: 'voice-a',
ttsModelId: 'volcengine/pool-a',
params: {}, params: {},
costMultiplier: 1, costMultiplier: 1,
} }
@@ -198,9 +195,9 @@ describe('speech store helpers', () => {
/** /**
* @example * @example
* speechStore.resolveVoicePackSpeechInput({ text, voice, voicePack: { packId: 'vp-1', costMultiplier: 1.5 } }) * speechStore.resolveVoicePackSpeechInput({ text, voice, voicePack: { packId: 'vp-1' } })
*/ */
it('passes Voice Pack snapshot billing metadata through adapter options', () => { it('passes only Voice Pack identity through adapter options', () => {
const speechStore = useSpeechStore() const speechStore = useSpeechStore()
const voice = { const voice = {
id: 'voice-1', id: 'voice-1',
@@ -215,7 +212,6 @@ describe('speech store helpers', () => {
params: {}, params: {},
voicePack: { voicePack: {
packId: 'vp-1', packId: 'vp-1',
costMultiplier: 1.5,
}, },
supportsAdapterProsody: true, supportsAdapterProsody: true,
}) })
@@ -223,7 +219,6 @@ describe('speech store helpers', () => {
expect(request.providerConfig.extraBody).toEqual({ expect(request.providerConfig.extraBody).toEqual({
voice_pack: { voice_pack: {
pack_id: 'vp-1', pack_id: 'vp-1',
cost_multiplier: 1.5,
}, },
}) })
}) })
@@ -29,7 +29,7 @@ interface VoicePackSpeechInputOptions {
voice: VoiceInfo voice: VoiceInfo
providerConfig?: Record<string, unknown> providerConfig?: Record<string, unknown>
params?: VoicePackParams params?: VoicePackParams
voicePack?: Pick<VoicePackSnapshot, 'packId' | 'costMultiplier'> voicePack?: Pick<VoicePackSnapshot, 'packId'>
forceSSML?: boolean forceSSML?: boolean
supportsSSML?: boolean supportsSSML?: boolean
supportsAdapterProsody?: boolean supportsAdapterProsody?: boolean
@@ -517,7 +517,6 @@ export const useSpeechStore = defineStore('speech', () => {
...(providerConfig.extraBody as Record<string, unknown> | undefined), ...(providerConfig.extraBody as Record<string, unknown> | undefined),
voice_pack: { voice_pack: {
pack_id: options.voicePack.packId, pack_id: options.voicePack.packId,
cost_multiplier: options.voicePack.costMultiplier,
...(needsProsody && options.supportsAdapterProsody ...(needsProsody && options.supportsAdapterProsody
? { pitch, volume } ? { pitch, volume }
: {}), : {}),