From 5d85f2ae465243fee71bed9b0a43b19160ce9fbe Mon Sep 17 00:00:00 2001 From: Neko Ayaka Date: Sat, 15 Mar 2025 15:04:32 +0800 Subject: [PATCH] fix(stage-ui): build --- .../stage-ui/src/stores/fix/elevenlabs.ts | 186 ++++++++++++++++++ .../stage-ui/src/stores/fix/list-voices.ts | 22 +++ packages/stage-ui/src/stores/fix/microsoft.ts | 174 ++++++++++++++++ packages/stage-ui/src/stores/fix/voice.ts | 41 ++++ packages/stage-ui/src/stores/providers.ts | 23 ++- 5 files changed, 443 insertions(+), 3 deletions(-) create mode 100644 packages/stage-ui/src/stores/fix/elevenlabs.ts create mode 100644 packages/stage-ui/src/stores/fix/list-voices.ts create mode 100644 packages/stage-ui/src/stores/fix/microsoft.ts create mode 100644 packages/stage-ui/src/stores/fix/voice.ts diff --git a/packages/stage-ui/src/stores/fix/elevenlabs.ts b/packages/stage-ui/src/stores/fix/elevenlabs.ts new file mode 100644 index 000000000..ab3d1729e --- /dev/null +++ b/packages/stage-ui/src/stores/fix/elevenlabs.ts @@ -0,0 +1,186 @@ +import type { UnSpeechOptions } from '@xsai-ext/providers-local' +import type { SpeechProviderWithExtraOptions } from '@xsai-ext/shared-providers' +import type { VoiceProviderWithExtraOptions } from './voice' + +import { merge } from '@xsai-ext/shared-providers' +import { objCamelToSnake } from '@xsai/shared' + +/** @see {@link https://elevenlabs.io/docs/api-reference/text-to-speech/convert#request} */ +export interface UnElevenLabsOptions { + /** + * This parameter controls text normalization with three modes: 'auto', 'on', and 'off'. When set to 'auto', + * the system will automatically decide whether to apply text normalization (e.g., spelling out numbers). + * With 'on', text normalization will always be applied, while with 'off', it will be skipped. Cannot be + * turned on for 'eleven_turbo_v2_5' model. + */ + applyTextNormalization?: 'auto' | 'off' | 'on' + /** + * Language code (ISO 639-1) used to enforce a language for the model. Currently only Turbo v2.5 + * supports language enforcement. For other models, an error will be returned if language code is provided. + */ + languageCode?: string + /** + * A list of request_id of the samples that were generated before this generation. Can + * be used to improve the flow of prosody when splitting up a large task into multiple + * requests. The results will be best when the same model is used across the generations. + * + * In case both next_text and next_request_ids is send, next_text will be ignored. + * A maximum of 3 request_ids can be send. + */ + nextRequestIds?: string[] + /** + * The text that comes after the text of the current request. Can be used to improve + * the flow of prosody when concatenating together multiple generations or to influence + * the prosody in the current generation. + */ + nextText?: string + /** + * A list of request_id of the samples that were generated before this generation. Can be + * used to improve the flow of prosody when splitting up a large task into multiple requests. + * The results will be best when the same model is used across the generations. In case both + * previous_text and previous_request_ids is send, previous_text will be ignored. A maximum + * of 3 request_ids can be send. + */ + previousRequestIds?: string[] + /** + * The text that came before the text of the current request. Can be used to improve the + * flow of prosody when concatenating together multiple generations or to influence the + * prosody in the current generation. + */ + previousText?: string + /** + * A list of pronunciation dictionary locators (id, version_id) to be applied to the text. + * They will be applied in order. You may have up to 3 locators per request + */ + pronunciationDictionaryLocators?: { + pronunciationDictionaryId: string + versionId: string + }[] + /** + * If specified, our system will make a best effort to sample deterministically, such that + * repeated requests with the same seed and parameters should return the same result. + * Determinism is not guaranteed. Must be integer between 0 and 4294967295. + */ + seed?: number + /** + * Voice settings overriding stored settings for the given voice. They are applied only on the given request. + */ + voiceSettings?: { + /** + * Determines how closely the AI should adhere to the original voice when attempting to replicate it. + */ + similarityBoost: number + /** + * Controls the speed of the generated speech. Values range from 0.7 to 1.2, with 1.0 being the default + * speed. Lower values create slower, more deliberate speech while higher values produce faster-paced + * speech. Extreme values can impact the quality of the generated speech. + * + * @default 1.0 + */ + speed?: number + /** + * Determines how stable the voice is and the randomness between each generation. Lower values introduce + * broader emotional range for the voice. Higher values can result in a monotonous voice with limited + * emotion. + */ + stability: number + /** + * Determines the style exaggeration of the voice. This setting attempts to amplify the style of the original + * speaker. It does consume additional computational resources and might increase latency if set to anything + * other than 0. + * + * @default 0 + */ + style?: number + /** + * This setting boosts the similarity to the original speaker. Using this setting requires a slightly higher + * computational load, which in turn increases latency. + * + * @default true + */ + useSpeakerBoost?: boolean + } +} + +/** + * [ElevenLabs](https://elevenlabs.io/) provider for [UnSpeech](https://github.com/moeru-ai/unspeech) + * only. + * + * [UnSpeech](https://github.com/moeru-ai/unspeech) is a open-source project that provides a + * OpenAI-compatible audio & speech related API that can be used with various providers such + * as ElevenLabs, Azure TTS, Google TTS, etc. + * + * @param apiKey - ElevenLabs API Key + * @param baseURL - UnSpeech Instance URL + * @returns SpeechProviderWithExtraOptions + */ +export function createUnElevenLabs(apiKey: string, baseURL = 'http://localhost:5933/v1/') { + const toUnSpeechOptions = ({ + applyTextNormalization, + languageCode, + nextRequestIds, + nextText, + previousRequestIds, + previousText, + pronunciationDictionaryLocators, + seed, + voiceSettings, + }: UnElevenLabsOptions): UnSpeechOptions => ({ + extraBody: objCamelToSnake({ + applyTextNormalization, + languageCode, + nextRequestIds, + nextText, + previousRequestIds, + previousText, + pronunciationDictionaryLocators: pronunciationDictionaryLocators + ? pronunciationDictionaryLocators.map(pdl => objCamelToSnake(pdl)) + : undefined, + seed, + voiceSettings: voiceSettings != null + ? objCamelToSnake(voiceSettings) + : { + similarityBoost: 0.75, + stability: 0.5, + }, + }), + }) + + const speechProvider: SpeechProviderWithExtraOptions< + /** @see {@link https://elevenlabs.io/docs/developer-guides/models} */ + 'eleven_english_sts_v2' | 'eleven_flash_v2' | 'eleven_flash_v2_5' | 'eleven_multilingual_sts_v2' | 'eleven_multilingual_v2', + UnElevenLabsOptions + > = { + speech: (model, options) => ({ + ...(options ? toUnSpeechOptions(options) : {}), + apiKey, + baseURL, + model: `elevenlabs/${model}`, + }), + } + + const voiceProvider: VoiceProviderWithExtraOptions< + UnElevenLabsOptions + > = { + voice: (options) => { + if (baseURL.endsWith('v1/')) { + baseURL = baseURL.slice(0, -3) + } + else if (baseURL.endsWith('v1')) { + baseURL = baseURL.slice(0, -2) + } + + return { + query: `provider=elevenlabs`, + ...(options ? toUnSpeechOptions(options) : {}), + apiKey, + baseURL, + } + }, + } + + return merge( + speechProvider, + voiceProvider, + ) +} diff --git a/packages/stage-ui/src/stores/fix/list-voices.ts b/packages/stage-ui/src/stores/fix/list-voices.ts new file mode 100644 index 000000000..65a13ffc4 --- /dev/null +++ b/packages/stage-ui/src/stores/fix/list-voices.ts @@ -0,0 +1,22 @@ +import type { CommonRequestOptions } from '@xsai/shared' +import type { Voice } from './voice' + +import { requestHeaders, requestURL, responseJSON } from '@xsai/shared' + +export interface ListVoicesOptions extends Omit { + query?: string +} + +export interface ListVoicesResponse { + voices: Voice[] +} + +export async function listVoices(options: ListVoicesOptions): Promise { + return (options.fetch ?? globalThis.fetch)(requestURL(options.query ? `api/voices?${options.query}` : 'api/voices', options.baseURL), { + headers: requestHeaders({ ...options.headers }, options.apiKey), + method: 'GET', + signal: options.abortSignal, + }) + .then(responseJSON) + .then(({ voices }) => voices) +} diff --git a/packages/stage-ui/src/stores/fix/microsoft.ts b/packages/stage-ui/src/stores/fix/microsoft.ts new file mode 100644 index 000000000..4480a7fd4 --- /dev/null +++ b/packages/stage-ui/src/stores/fix/microsoft.ts @@ -0,0 +1,174 @@ +import type { UnSpeechOptions } from '@xsai-ext/providers-local' +import type { SpeechProviderWithExtraOptions } from '@xsai-ext/shared-providers' +import type { VoiceProviderWithExtraOptions } from './voice' + +import { merge } from '@xsai-ext/shared-providers' +import { objCamelToSnake } from '@xsai/shared' + +export type MicrosoftRegions = + | 'australiaeast' + | 'brazilsouth' + | 'canadacentral' + | 'centralindia' + | 'centralus' + | 'eastasia' + | 'eastus2' + | 'eastus' + | 'francecentral' + | 'germanywestcentral' + | 'japaneast' + | 'japanwest' + | 'jioindiawest' + | 'koreacentral' + | 'northcentralus' + | 'northeurope' + | 'norwayeast' + | 'southcentralus' + | 'southeastasia' + | 'swedencentral' + | 'switzerlandnorth' + | 'switzerlandwest' + | 'uaenorth' + | 'uksouth' + | 'usgovarizona' + | 'usgovvirginia' + | 'westcentralus' + | 'westeurope' + | 'westus2' + | 'westus3' + | 'westus' + +export interface UnMicrosoftOptionAutoSSML { + gender: + | 'Female' + | 'Male' + | 'Neutral' + | string + lang: + | 'en-US' + | string + /** + * Speech Studio - Voice Gallery + * https://speech.microsoft.com/portal/018ba84135d64cf79106cc99c75ffa6a/voicegallery + */ + voice: + | 'en-US-AndrewMultilingualNeural' + | 'en-US-AriaNeural' + | 'en-US-AvaMultilingualNeural' + | 'en-US-BrianMultilingualNeural' + | 'en-US-ChristopherMultilingualNeural' + | 'en-US-EmmaMultilingualNeural' + | 'en-US-JaneNeural' + | string +} + +export interface UnMicrosoftOptionCommon { + /** + * Text to speech API reference (REST) - Speech service - Azure AI services | Microsoft Learn + * https://learn.microsoft.com/en-us/azure/ai-services/speech-service/rest-text-to-speech?tabs=streaming#custom-neural-voices + */ + deploymentId?: string + /** + * Text to speech API reference (REST) - Speech service - Azure AI services | Microsoft Learn + * https://learn.microsoft.com/en-us/azure/ai-services/speech-service/rest-text-to-speech?tabs=streaming#prebuilt-neural-voices + * + * NOTICE: Voices in preview are available in only these three regions: East US, West Europe, and Southeast Asia. + */ + region: MicrosoftRegions | string + sampleRate?: + | 8000 + | 16000 + | 22050 + | 24000 + | 44100 + | 48000 + | number +} + +export interface UnMicrosoftOptionCustomSSML { + /** + * By default, unspeech service will help you automatically convert OpenAI style plain text input + * into SSML with lang, gender, voice parameters, but if you ever wanted to provide your own SSML + * with all customizable parameters, you can set this option to `true` to disable the automatic + * conversion and use your own SSML instead. + * + * About SSML (Speech Synthesis Markup Language), @see {@link https://learn.microsoft.com/en-us/azure/ai-services/speech-service/speech-synthesis-markup} + */ + disableSsml?: boolean +} + +/** @see {@link https://elevenlabs.io/docs/api-reference/text-to-speech/convert#request} */ +export type UnMicrosoftOptions = (UnMicrosoftOptionAutoSSML | UnMicrosoftOptionCustomSSML) & UnMicrosoftOptionCommon + +/** + * [Microsoft / Azure AI](https://speech.microsoft.com/portal) provider for [UnSpeech](https://github.com/moeru-ai/unspeech) + * only. + * + * [UnSpeech](https://github.com/moeru-ai/unspeech) is a open-source project that provides a + * OpenAI-compatible audio & speech related API that can be used with various providers such + * as ElevenLabs, Azure TTS, Google TTS, etc. + * + * @param apiKey - Microsoft / Azure AI subscription key + * @param baseURL - UnSpeech Instance URL + * @returns SpeechProviderWithExtraOptions + */ +export function createUnMicrosoft(apiKey: string, baseURL = 'http://localhost:5933/v1/') { + const toUnSpeechOptions = (options: UnMicrosoftOptions): UnSpeechOptions => { + const { deploymentId, region, sampleRate } = options + + const extraBody: Record = { + deploymentId, + region, + sampleRate, + } + + if ('disableSsml' in options) { + extraBody.disableSsml = options.disableSsml + } + else if ('lang' in options) { + extraBody.lang = options.lang + extraBody.gender = options.gender + extraBody.voice = options.voice + } + + return { extraBody: objCamelToSnake(extraBody) } + } + + const speechProvider: SpeechProviderWithExtraOptions< + /** @see Currently, cognitive services are on v1 */ + 'microsoft/v1', + UnMicrosoftOptions + > = { + speech: (model, options) => ({ + ...(options ? toUnSpeechOptions(options) : {}), + apiKey, + baseURL, + model: `microsoft/${model}`, + }), + } + + const voiceProvider: VoiceProviderWithExtraOptions< + UnMicrosoftOptions + > = { + voice: (options) => { + if (baseURL.endsWith('v1/')) { + baseURL = baseURL.slice(0, -3) + } + else if (baseURL.endsWith('v1')) { + baseURL = baseURL.slice(0, -2) + } + + return { + query: `region=${options?.region}&provider=microsoft`, + ...(options ? toUnSpeechOptions(options) : {}), + apiKey, + baseURL, + } + }, + } + + return merge( + speechProvider, + voiceProvider, + ) +} diff --git a/packages/stage-ui/src/stores/fix/voice.ts b/packages/stage-ui/src/stores/fix/voice.ts new file mode 100644 index 000000000..884be7971 --- /dev/null +++ b/packages/stage-ui/src/stores/fix/voice.ts @@ -0,0 +1,41 @@ +import type { CommonRequestOptions } from '@xsai/shared' + +export interface Voice { + compatible_models: string[] + description: string + formats: VoiceFormat[] + id: string + labels: Record & { + accent?: string + age?: string + gender?: string + type?: string + } + languages: VoiceLanguage[] + name: string + predefined_options?: Record + preview_audio_url?: string + tags: string[] +} + +export interface VoiceFormat { + bitrate: number + extension: string + format_code: string + mime_type: string + name: string + sample_rate: number +} + +export interface VoiceLanguage { + code: string + title: string +} + +export interface VoiceProvider { + voice: () => Omit & { query?: string } +} + +export interface VoiceProviderWithExtraOptions { + voice: (options?: T) => Omit & { query?: string } & Partial +} diff --git a/packages/stage-ui/src/stores/providers.ts b/packages/stage-ui/src/stores/providers.ts index b60ba9ccd..9a6acda45 100644 --- a/packages/stage-ui/src/stores/providers.ts +++ b/packages/stage-ui/src/stores/providers.ts @@ -9,6 +9,7 @@ import type { TranscriptionProvider, TranscriptionProviderWithExtraOptions, } from '@xsai-ext/shared-providers' +import type { VoiceProviderWithExtraOptions } from './fix/voice' import { useLocalStorage } from '@vueuse/core' import { @@ -24,12 +25,16 @@ import { createWorkersAI, createXAI, } from '@xsai-ext/providers-cloud' -import { createOllama, createUnElevenLabs } from '@xsai-ext/providers-local' +import { createOllama } from '@xsai-ext/providers-local' import { listModels } from '@xsai/model' import { defineStore } from 'pinia' import { computed, ref, watch } from 'vue' import { useI18n } from 'vue-i18n' +import { createUnElevenLabs } from './fix/elevenlabs' +import { listVoices } from './fix/list-voices' +// import { createUnMicrosoft } from './fix/microsoft' + export interface ProviderMetadata { id: string nameKey: string // i18n key for provider name @@ -309,8 +314,20 @@ export const useProvidersStore = defineStore('providers', () => { listModels: async () => { return [] }, - listVoices: async () => { - return [] + listVoices: async (config) => { + const provider = createUnElevenLabs(config.apiKey as string, config.baseUrl as string) as VoiceProviderWithExtraOptions + + const voices = await listVoices({ + ...provider.voice(), + }) + + return voices.map((voice) => { + return { + id: voice.id, + name: voice.name, + provider: 'elevenlabs', + } + }) }, }, },