diff --git a/apps/server/src/services/adapters/config-kv.ts b/apps/server/src/services/adapters/config-kv.ts index f3539eeef..25d5e6b75 100644 --- a/apps/server/src/services/adapters/config-kv.ts +++ b/apps/server/src/services/adapters/config-kv.ts @@ -52,7 +52,7 @@ export const llmModelSchema = object({ fallbackTriggers: fallbackTriggersSchema, }) -const ttsProviderSchema = picklist(['azure', 'dashscope-cosyvoice', 'volcengine']) +const ttsProviderSchema = picklist(['azure', 'dashscope-cosyvoice', 'stepfun', 'volcengine']) export const ttsUpstreamSchema = object({ baseURL: pipe(string(), nonEmpty('tts.upstreams[].baseURL must not be empty')), diff --git a/apps/server/src/services/adapters/tts/audio-format.ts b/apps/server/src/services/adapters/tts/audio-format.ts new file mode 100644 index 000000000..e5b5bcde1 --- /dev/null +++ b/apps/server/src/services/adapters/tts/audio-format.ts @@ -0,0 +1,26 @@ +const AUDIO_MIME_TYPES: Record = { + flac: 'audio/flac', + mp3: 'audio/mpeg', + ogg_opus: 'audio/ogg', + opus: 'audio/opus', + pcm: 'audio/L16', + wav: 'audio/wav', +} + +/** + * Maps provider audio format keys to response MIME types. + * + * Use when: + * - A TTS adapter forwards OpenAI-shaped `response_format` / provider + * encoding keys through unspeech and needs a gateway fallback MIME type. + * + * Expects: + * - `format` is the exact provider/OpenAI format key. + * + * Returns: + * - A known audio MIME type, or `application/octet-stream` for unknown custom + * formats so operators can still experiment through config. + */ +export function audioMimeFromFormat(format: string): string { + return AUDIO_MIME_TYPES[format] ?? 'application/octet-stream' +} diff --git a/apps/server/src/services/adapters/tts/dashscope-cosyvoice.ts b/apps/server/src/services/adapters/tts/dashscope-cosyvoice.ts index 08eba0bd0..e4339620d 100644 --- a/apps/server/src/services/adapters/tts/dashscope-cosyvoice.ts +++ b/apps/server/src/services/adapters/tts/dashscope-cosyvoice.ts @@ -3,6 +3,7 @@ import type { Voice } from 'unspeech' import type { TtsAdapter, TtsAdapterContext, TtsInput, TtsResult, TtsVoiceCatalogContext } from './types' import { createBadRequestError } from '../../../utils/error' +import { audioMimeFromFormat } from './audio-format' import { listVoicesViaUnSpeech, sendSpeechViaUnSpeech } from './unspeech' /** @@ -69,7 +70,7 @@ export const dashscopeCosyvoiceAdapter: TtsAdapter = { input: input.text, voice, responseFormat: format, - fallbackContentType: formatToMime(format), + fallbackContentType: audioMimeFromFormat(format), providerLabel: 'dashscope-cosyvoice', }) }, @@ -89,16 +90,3 @@ export const dashscopeCosyvoiceAdapter: TtsAdapter = { }) }, } - -/** - * Maps cosyvoice's `format` (`mp3` / `wav` / `pcm`) to a MIME type for the - * client. Keeps the router contract symmetric with Azure / Volcengine. - */ -function formatToMime(format: string): string { - switch (format) { - case 'mp3': return 'audio/mpeg' - case 'wav': return 'audio/wav' - case 'pcm': return 'audio/L16' - default: return 'application/octet-stream' - } -} diff --git a/apps/server/src/services/adapters/tts/index.test.ts b/apps/server/src/services/adapters/tts/index.test.ts index 194ce7a59..092dbd02d 100644 --- a/apps/server/src/services/adapters/tts/index.test.ts +++ b/apps/server/src/services/adapters/tts/index.test.ts @@ -21,6 +21,11 @@ describe('getAdapter', () => { expect(adapter.id).toBe('volcengine') }) + it('returns the stepfun adapter by id', () => { + const adapter = getAdapter('stepfun') + expect(adapter.id).toBe('stepfun') + }) + it('throws BAD_REQUEST on unknown id with the available list in details', () => { expect(() => getAdapter('unknown-provider')).toThrow(ApiError) try { @@ -34,7 +39,7 @@ describe('getAdapter', () => { expect(apiErr.details).toEqual( expect.objectContaining({ id: 'unknown-provider', - available: expect.arrayContaining(['azure', 'dashscope-cosyvoice', 'volcengine']), + available: expect.arrayContaining(['azure', 'dashscope-cosyvoice', 'stepfun', 'volcengine']), }), ) } @@ -292,6 +297,133 @@ describe('azureAdapter.send', () => { }) }) +describe('stepfunAdapter', () => { + it('lists StepFun voices through unspeech provider=stepfun', async () => { + const adapter = getAdapter('stepfun') + const fetchImpl = vi.fn(async () => new Response(JSON.stringify({ + voices: [{ + id: 'cixingnansheng', + name: '磁性男声', + compatible_models: ['stepaudio-2.5-tts', 'step-tts-2', 'step-tts-mini'], + }], + }), { status: 200 })) as unknown as typeof fetch + + const voices = await adapter.getVoiceCatalog({ + adapterParams: {}, + unspeechBaseURL: 'http://unspeech.local', + fetchImpl, + }) + + expect(voices).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + id: 'cixingnansheng', + name: '磁性男声', + compatible_models: expect.arrayContaining(['stepaudio-2.5-tts', 'step-tts-2', 'step-tts-mini']), + }), + ]), + ) + const [calledUrl] = (fetchImpl as unknown as { mock: { calls: [string, RequestInit][] } }).mock.calls[0] + expect(calledUrl).toBe('http://unspeech.local/api/voices?provider=stepfun') + }) + + it('posts OpenAI-compatible speech JSON to unspeech with model=stepfun/', async () => { + const adapter = getAdapter('stepfun') + const fetchImpl = vi.fn(async () => new Response(new Uint8Array([1, 2, 3]), { + status: 200, + headers: { 'content-type': 'audio/mpeg' }, + })) as unknown as typeof fetch + + const result = await adapter.send( + { + text: '（轻声）你好', + voice: 'cixingnansheng', + responseFormat: 'mp3', + speed: 1.2, + extraOptions: { + instruction: '温柔、克制、有一点笑意', + volume: 1.1, + sampleRate: 24000, + }, + }, + { + keyPlaintext: Buffer.from('step-key', 'utf8'), + baseURL: 'https://api.stepfun.com/v1/audio/speech', + unspeechBaseURL: 'http://unspeech.local:5933', + adapterParams: { model: 'stepaudio-2.5-tts' }, + fetchImpl, + }, + ) + + const [calledURL, init] = (fetchImpl as unknown as { mock: { calls: [string, RequestInit][] } }).mock.calls[0] + expect(calledURL).toBe('http://unspeech.local:5933/v1/audio/speech') + expect(init.method).toBe('POST') + expect(init.headers).toMatchObject({ + 'Authorization': 'Bearer step-key', + 'Content-Type': 'application/json', + }) + const body = JSON.parse(init.body as string) as Record + expect(body).toEqual({ + model: 'stepfun/stepaudio-2.5-tts', + input: '（轻声）你好', + voice: 'cixingnansheng', + response_format: 'mp3', + speed: 1.2, + extra_body: { + volume: 1.1, + sample_rate: 24000, + instruction: '温柔、克制、有一点笑意', + }, + }) + expect(result.contentType).toBe('audio/mpeg') + expect(result.body).toBeInstanceOf(ArrayBuffer) + }) + + it('passes voice_label through to unspeech for provider-level validation', async () => { + const adapter = getAdapter('stepfun') + const fetchImpl = vi.fn(async () => new Response(new Uint8Array([1]), { + status: 200, + headers: { 'content-type': 'audio/mpeg' }, + })) as unknown as typeof fetch + + await adapter.send( + { + text: 'hi', + extraOptions: { + voice_label: { emotion: '高兴' }, + }, + }, + { + keyPlaintext: Buffer.from('step-key', 'utf8'), + baseURL: 'https://api.stepfun.com/v1/audio/speech', + unspeechBaseURL: 'http://unspeech.local', + adapterParams: { model: 'stepaudio-2.5-tts' }, + fetchImpl, + }, + ) + + const [, init] = (fetchImpl as unknown as { mock: { calls: [string, RequestInit][] } }).mock.calls[0] + const body = JSON.parse(init.body as string) as Record + expect(body.extra_body.voice_label).toEqual({ emotion: '高兴' }) + }) + + it('throws Error with .status when unspeech returns non-2xx', async () => { + const adapter = getAdapter('stepfun') + const fetchImpl = vi.fn(async () => new Response('bad key', { status: 401 })) as unknown as typeof fetch + + await expect(adapter.send( + { text: 'hi', voice: 'cixingnansheng' }, + { + keyPlaintext: Buffer.from('bad-key', 'utf8'), + baseURL: 'https://api.stepfun.com/v1/audio/speech', + unspeechBaseURL: 'http://unspeech.local', + adapterParams: { model: 'stepaudio-2.5-tts' }, + fetchImpl, + }, + )).rejects.toMatchObject({ status: 401 }) + }) +}) + describe('volcengineAdapter.send', () => { it('posts to unspeech with model=volcengine/ and app/cluster in extra_body', async () => { const adapter = getAdapter('volcengine') diff --git a/apps/server/src/services/adapters/tts/index.ts b/apps/server/src/services/adapters/tts/index.ts index b464b16a2..4672e4825 100644 --- a/apps/server/src/services/adapters/tts/index.ts +++ b/apps/server/src/services/adapters/tts/index.ts @@ -3,11 +3,13 @@ import type { TtsAdapter, TtsAdapterId } from './types' import { createBadRequestError } from '../../../utils/error' import { azureAdapter } from './azure' import { dashscopeCosyvoiceAdapter } from './dashscope-cosyvoice' +import { stepfunAdapter } from './stepfun' import { volcengineAdapter } from './volcengine' const ADAPTERS: Record = { 'azure': azureAdapter, 'dashscope-cosyvoice': dashscopeCosyvoiceAdapter, + 'stepfun': stepfunAdapter, 'volcengine': volcengineAdapter, } diff --git a/apps/server/src/services/adapters/tts/stepfun.ts b/apps/server/src/services/adapters/tts/stepfun.ts new file mode 100644 index 000000000..064151332 --- /dev/null +++ b/apps/server/src/services/adapters/tts/stepfun.ts @@ -0,0 +1,104 @@ +import type { Voice } from 'unspeech' + +import type { TtsAdapter, TtsAdapterContext, TtsInput, TtsResult, TtsVoiceCatalogContext } from './types' + +import { isPlainObject } from 'es-toolkit' + +import { audioMimeFromFormat } from './audio-format' +import { listVoicesViaUnSpeech, sendSpeechViaUnSpeech } from './unspeech' + +const STEPFUN_DEFAULT_MODEL = 'stepaudio-2.5-tts' +const STEPFUN_DEFAULT_FORMAT = 'mp3' +const STEPFUN_DEFAULT_VOICE = 'cixingnansheng' + +/** + * StepFun TTS adapter. + * + * Use when: + * - Routing hosted speech synthesis to StepFun through unspeech's + * OpenAI-compatible `stepfun/*` backend. + * + * Expects: + * - `ctx.unspeechBaseURL` points at an unspeech deployment that includes the + * StepFun backend. + * - `ctx.keyPlaintext` is the StepFun API key. + * - `ctx.adapterParams.model` optionally selects `stepaudio-2.5-tts`, + * `step-tts-2`, or `step-tts-mini`. + * + * Returns: + * - {@link TtsResult} with the upstream audio body and content type. + */ +export const stepfunAdapter: TtsAdapter = { + id: 'stepfun', + + async send(input: TtsInput, ctx: TtsAdapterContext): Promise { + const model = typeof ctx.adapterParams.model === 'string' && ctx.adapterParams.model + ? ctx.adapterParams.model + : STEPFUN_DEFAULT_MODEL + const voice = input.voice ?? (typeof ctx.adapterParams.defaultVoice === 'string' && ctx.adapterParams.defaultVoice + ? ctx.adapterParams.defaultVoice + : STEPFUN_DEFAULT_VOICE) + const responseFormat = input.responseFormat ?? (typeof ctx.adapterParams.responseFormat === 'string' && ctx.adapterParams.responseFormat + ? ctx.adapterParams.responseFormat + : STEPFUN_DEFAULT_FORMAT) + + return sendSpeechViaUnSpeech({ + ctx, + model: `stepfun/${model}`, + input: input.text, + voice, + speed: input.speed, + responseFormat, + extraBody: buildExtraBody(input, ctx), + fallbackContentType: audioMimeFromFormat(responseFormat), + providerLabel: 'stepfun', + }) + }, + + async getVoiceCatalog(ctx: TtsVoiceCatalogContext): Promise { + return listVoicesViaUnSpeech({ + ctx, + query: 'provider=stepfun', + providerLabel: 'stepfun', + }) + }, +} + +function buildExtraBody(input: TtsInput, ctx: TtsAdapterContext): Record { + const extraOptions = input.extraOptions ?? {} + const body: Record = {} + + if (typeof extraOptions.volume === 'number' && Number.isFinite(extraOptions.volume)) + body.volume = extraOptions.volume + else if (typeof ctx.adapterParams.volume === 'number' && Number.isFinite(ctx.adapterParams.volume)) + body.volume = ctx.adapterParams.volume + + if (typeof extraOptions.sample_rate === 'number' && Number.isFinite(extraOptions.sample_rate)) + body.sample_rate = extraOptions.sample_rate + else if (typeof extraOptions.sampleRate === 'number' && Number.isFinite(extraOptions.sampleRate)) + body.sample_rate = extraOptions.sampleRate + else if (typeof ctx.adapterParams.sampleRate === 'number' && Number.isFinite(ctx.adapterParams.sampleRate)) + body.sample_rate = ctx.adapterParams.sampleRate + + if (isPlainObject(extraOptions.pronunciation_map)) + body.pronunciation_map = extraOptions.pronunciation_map + else if (isPlainObject(extraOptions.pronunciationMap)) + body.pronunciation_map = extraOptions.pronunciationMap + + if (typeof extraOptions.markdown_filter === 'boolean') + body.markdown_filter = extraOptions.markdown_filter + else if (typeof extraOptions.markdownFilter === 'boolean') + body.markdown_filter = extraOptions.markdownFilter + + if (typeof extraOptions.instruction === 'string' && extraOptions.instruction) + body.instruction = extraOptions.instruction + else if (typeof ctx.adapterParams.instruction === 'string' && ctx.adapterParams.instruction) + body.instruction = ctx.adapterParams.instruction + + if (isPlainObject(extraOptions.voice_label)) + body.voice_label = extraOptions.voice_label + else if (isPlainObject(extraOptions.voiceLabel)) + body.voice_label = extraOptions.voiceLabel + + return body +} diff --git a/apps/server/src/services/adapters/tts/types.ts b/apps/server/src/services/adapters/tts/types.ts index a5fd8de91..176d567a2 100644 --- a/apps/server/src/services/adapters/tts/types.ts +++ b/apps/server/src/services/adapters/tts/types.ts @@ -82,7 +82,7 @@ export interface TtsResult { * `./index.ts` — the union is intentionally tight so unknown ids fail at the * type level (router config validation handles runtime). */ -export type TtsAdapterId = 'azure' | 'dashscope-cosyvoice' | 'volcengine' +export type TtsAdapterId = 'azure' | 'dashscope-cosyvoice' | 'stepfun' | 'volcengine' /** * Per-call context for {@link TtsAdapter.getVoiceCatalog}. diff --git a/apps/server/src/services/adapters/tts/unspeech.ts b/apps/server/src/services/adapters/tts/unspeech.ts index 8b2c3cd46..7ad388582 100644 --- a/apps/server/src/services/adapters/tts/unspeech.ts +++ b/apps/server/src/services/adapters/tts/unspeech.ts @@ -12,6 +12,7 @@ interface SendSpeechOptions { model: string input: string voice: string + speed?: number responseFormat: string extraBody?: Record fallbackContentType: string @@ -41,6 +42,7 @@ export async function sendSpeechViaUnSpeech(options: SendSpeechOptions): Promise model, providerLabel, responseFormat, + speed, voice, } = options @@ -52,6 +54,7 @@ export async function sendSpeechViaUnSpeech(options: SendSpeechOptions): Promise input, model, responseFormat, + speed, voice, abortSignal: ctx.abortSignal, extraBody, diff --git a/apps/server/src/services/adapters/tts/volcengine.ts b/apps/server/src/services/adapters/tts/volcengine.ts index 1aa4ed165..a6cadcbc6 100644 --- a/apps/server/src/services/adapters/tts/volcengine.ts +++ b/apps/server/src/services/adapters/tts/volcengine.ts @@ -4,6 +4,7 @@ import type { TtsAdapter, TtsAdapterContext, TtsInput, TtsResult, TtsVoiceCatalo import { createBadRequestError, createInternalError } from '../../../utils/error' import { nanoid } from '../../../utils/id' +import { audioMimeFromFormat } from './audio-format' import { listVoicesViaUnSpeech, sendSpeechViaUnSpeech } from './unspeech' /** @@ -86,7 +87,7 @@ export const volcengineAdapter: TtsAdapter = { audio: { speed_ratio: speed }, request: { reqid: nanoid(), operation: 'query' }, }, - fallbackContentType: encodingToMime(encoding), + fallbackContentType: audioMimeFromFormat(encoding), providerLabel: 'volcengine', }) }, @@ -111,18 +112,3 @@ export const volcengineAdapter: TtsAdapter = { }) }, } - -/** - * Maps Volcengine's `encoding` field to a MIME type for the gateway response. - */ -function encodingToMime(encoding: string): string { - if (encoding === 'mp3') - return 'audio/mpeg' - if (encoding === 'wav') - return 'audio/wav' - if (encoding === 'pcm') - return 'audio/L16' - if (encoding === 'ogg_opus') - return 'audio/ogg' - return 'application/octet-stream' -}