feat(stage-ui): add Google Gemini TTS provider (#1828)
--------- Co-authored-by: leiyutian <leiyutian@echo.tech> Co-authored-by-agent: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
@@ -1283,6 +1283,9 @@ pages:
|
||||
fireworks:
|
||||
description: fireworks.ai
|
||||
title: Fireworks.ai
|
||||
google-gemini-audio-speech:
|
||||
description: aistudio.google.com
|
||||
title: Google Gemini
|
||||
microsoft-speech:
|
||||
description: speech.microsoft.com
|
||||
fields:
|
||||
|
||||
@@ -77,7 +77,7 @@ function handleDeleteProvider(providerId: string) {
|
||||
<fieldset
|
||||
v-if="persistedChatProvidersMetadata.length > 0"
|
||||
flex="~ row gap-4"
|
||||
min-w-0 of-x-auto scroll-smooth
|
||||
min-w-0 overflow-x-auto scroll-smooth
|
||||
role="radiogroup"
|
||||
>
|
||||
<RadioCardSimple
|
||||
|
||||
@@ -535,7 +535,7 @@ onUnmounted(() => {
|
||||
<fieldset
|
||||
v-if="configuredTranscriptionProvidersMetadata.length > 0"
|
||||
flex="~ row gap-4"
|
||||
min-w-0 of-x-auto scroll-smooth
|
||||
min-w-0 overflow-x-auto scroll-smooth
|
||||
role="radiogroup"
|
||||
>
|
||||
<RadioCardSimple
|
||||
|
||||
@@ -344,7 +344,6 @@ function handleDeleteProvider(providerId: string) {
|
||||
<span>{{ t('settings.pages.modules.speech.sections.section.voice-pack.description') }}</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div v-if="isLoadingVoicePacks" :class="['flex items-center gap-2', 'text-sm text-neutral-400 dark:text-neutral-500']">
|
||||
<div i-solar:spinner-line-duotone class="animate-spin text-base" />
|
||||
<span>{{ t('settings.pages.modules.speech.sections.section.voice-pack.loading') }}</span>
|
||||
@@ -398,7 +397,7 @@ function handleDeleteProvider(providerId: string) {
|
||||
<div max-w-full>
|
||||
<fieldset
|
||||
v-if="selectableSpeechProvidersMetadata.length > 0" flex="~ row gap-4"
|
||||
min-w-0 of-x-auto scroll-smooth role="radiogroup"
|
||||
min-w-0 overflow-x-auto scroll-smooth role="radiogroup"
|
||||
>
|
||||
<RadioCardSimple
|
||||
v-for="metadata in selectableSpeechProvidersMetadata"
|
||||
|
||||
@@ -104,7 +104,7 @@ function formatRelativeTime(timestamp: number | null) {
|
||||
<div :class="['max-w-full']">
|
||||
<fieldset
|
||||
v-if="persistedVisionProvidersMetadata.length > 0"
|
||||
:class="['flex', 'min-w-0', 'flex-row', 'gap-4', 'of-x-auto', 'scroll-smooth']"
|
||||
:class="['flex', 'min-w-0', 'flex-row', 'gap-4', 'overflow-x-auto', 'scroll-smooth']"
|
||||
role="radiogroup"
|
||||
>
|
||||
<RadioCardSimple
|
||||
|
||||
+173
@@ -0,0 +1,173 @@
|
||||
<script setup lang="ts">
|
||||
import type { SpeechProvider } from '@xsai-ext/providers/utils'
|
||||
|
||||
import {
|
||||
Alert,
|
||||
SpeechPlayground,
|
||||
SpeechProviderSettings,
|
||||
} from '@proj-airi/stage-ui/components'
|
||||
import { useProviderValidation } from '@proj-airi/stage-ui/composables/use-provider-validation'
|
||||
import { useSpeechStore } from '@proj-airi/stage-ui/stores/modules/speech'
|
||||
import { useProvidersStore } from '@proj-airi/stage-ui/stores/providers'
|
||||
import { FieldCombobox, FieldRange } from '@proj-airi/ui'
|
||||
import { storeToRefs } from 'pinia'
|
||||
import { computed, onMounted } from 'vue'
|
||||
import { useI18n } from 'vue-i18n'
|
||||
|
||||
const speechStore = useSpeechStore()
|
||||
const providersStore = useProvidersStore()
|
||||
const { providers } = storeToRefs(providersStore)
|
||||
const { t } = useI18n()
|
||||
|
||||
interface GoogleGeminiSpeechProviderConfig {
|
||||
apiKey?: string
|
||||
baseUrl?: string
|
||||
model?: string
|
||||
voice?: string
|
||||
temperature?: number
|
||||
}
|
||||
|
||||
const providerId = 'google-gemini-audio-speech'
|
||||
const defaultModel = 'gemini-2.5-flash-preview-tts'
|
||||
|
||||
const config = computed(() => providers.value[providerId] as GoogleGeminiSpeechProviderConfig | undefined)
|
||||
|
||||
function ensureProviderConfig(): GoogleGeminiSpeechProviderConfig {
|
||||
if (!providers.value[providerId])
|
||||
providers.value[providerId] = {}
|
||||
|
||||
return providers.value[providerId] as GoogleGeminiSpeechProviderConfig
|
||||
}
|
||||
|
||||
const providerModels = computed(() => providersStore.getModelsForProvider(providerId))
|
||||
const modelOptions = computed(() => {
|
||||
return (providerModels.value.length > 0 ? providerModels.value : []).map(model => ({
|
||||
value: model.id,
|
||||
label: model.name,
|
||||
}))
|
||||
})
|
||||
|
||||
const availableVoices = computed(() => speechStore.availableVoices[providerId] || [])
|
||||
|
||||
const model = computed({
|
||||
get: () => config.value?.model || defaultModel,
|
||||
set: (value) => {
|
||||
ensureProviderConfig().model = value
|
||||
},
|
||||
})
|
||||
|
||||
const temperature = computed({
|
||||
get: () => config.value?.temperature ?? 1.0,
|
||||
set: (value) => {
|
||||
ensureProviderConfig().temperature = value
|
||||
},
|
||||
})
|
||||
|
||||
const apiKeyConfigured = computed(() => !!providers.value[providerId]?.apiKey)
|
||||
|
||||
onMounted(async () => {
|
||||
ensureProviderConfig()
|
||||
|
||||
if (!config.value?.model)
|
||||
model.value = defaultModel
|
||||
|
||||
await providersStore.loadModelsForConfiguredProviders()
|
||||
await providersStore.fetchModelsForProvider(providerId)
|
||||
await speechStore.loadVoicesForProvider(providerId)
|
||||
})
|
||||
|
||||
async function handleGenerateSpeech(input: string, voiceId: string, _useSSML: boolean, modelId?: string) {
|
||||
const provider = await providersStore.getProviderInstance<SpeechProvider<string>>(providerId)
|
||||
if (!provider)
|
||||
throw new Error('Failed to initialize speech provider')
|
||||
|
||||
const providerConfig = providersStore.getProviderConfig(providerId)
|
||||
const modelToUse = modelId || model.value || defaultModel
|
||||
const voiceToUse = voiceId || '' as string
|
||||
|
||||
return await speechStore.speech(
|
||||
provider,
|
||||
modelToUse,
|
||||
input,
|
||||
voiceToUse,
|
||||
providerConfig,
|
||||
)
|
||||
}
|
||||
|
||||
const {
|
||||
isValidating,
|
||||
isValid,
|
||||
validationMessage,
|
||||
forceValid,
|
||||
} = useProviderValidation(providerId)
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<SpeechProviderSettings
|
||||
:provider-id="providerId"
|
||||
:default-model="defaultModel"
|
||||
>
|
||||
<template #voice-settings>
|
||||
<FieldCombobox
|
||||
v-model="model"
|
||||
label="Model"
|
||||
description="Select the Gemini TTS model to use for speech generation"
|
||||
:options="modelOptions"
|
||||
placeholder="Select a Gemini model..."
|
||||
/>
|
||||
<FieldRange
|
||||
v-model="temperature"
|
||||
label="Temperature"
|
||||
description="Controls randomness in speech generation. Lower values make speech more predictable, higher values make it more creative."
|
||||
:min="0"
|
||||
:max="2"
|
||||
:step="0.1"
|
||||
:format-value="(value) => value.toFixed(1)"
|
||||
/>
|
||||
</template>
|
||||
|
||||
<template #playground>
|
||||
<SpeechPlayground
|
||||
:available-voices="availableVoices"
|
||||
:generate-speech="handleGenerateSpeech"
|
||||
:api-key-configured="apiKeyConfigured"
|
||||
:voices-loading="speechStore.isLoadingSpeechProviderVoices"
|
||||
default-text="Hello! This is a test of the Google Gemini Speech."
|
||||
/>
|
||||
</template>
|
||||
|
||||
<template #advanced-settings>
|
||||
<Alert v-if="!isValid && isValidating === 0 && validationMessage" type="error">
|
||||
<template #title>
|
||||
<div class="w-full flex items-center justify-between">
|
||||
<span>{{ t('settings.dialogs.onboarding.validationFailed') }}</span>
|
||||
<button
|
||||
type="button"
|
||||
class="ml-2 rounded bg-red-100 px-2 py-0.5 text-xs text-red-600 font-medium transition-colors dark:bg-red-800/30 hover:bg-red-200 dark:text-red-300 dark:hover:bg-red-700/40"
|
||||
@click="forceValid"
|
||||
>
|
||||
{{ t('settings.pages.providers.common.continueAnyway') }}
|
||||
</button>
|
||||
</div>
|
||||
</template>
|
||||
<template v-if="validationMessage" #content>
|
||||
<div class="whitespace-pre-wrap break-all">
|
||||
{{ validationMessage }}
|
||||
</div>
|
||||
</template>
|
||||
</Alert>
|
||||
<Alert v-if="isValid && isValidating === 0" type="success">
|
||||
<template #title>
|
||||
{{ t('settings.dialogs.onboarding.validationSuccess') }}
|
||||
</template>
|
||||
</Alert>
|
||||
</template>
|
||||
</SpeechProviderSettings>
|
||||
</template>
|
||||
|
||||
<route lang="yaml">
|
||||
meta:
|
||||
layout: settings
|
||||
stageTransition:
|
||||
name: slide
|
||||
</route>
|
||||
@@ -59,6 +59,7 @@ import { useAuthStore } from './auth'
|
||||
import { createAliyunNLSProvider as createAliyunNlsStreamProvider } from './providers/aliyun/stream-transcription'
|
||||
import { convertProviderDefinitionsToMetadata } from './providers/converters'
|
||||
import { models as elevenLabsModels } from './providers/elevenlabs/list-models'
|
||||
import { buildGoogleGeminiSpeechProvider } from './providers/google-gemini-speech'
|
||||
import { buildOpenAICompatibleProvider } from './providers/openai-compatible-builder'
|
||||
import { buildOpenRouterAudioSpeechProvider } from './providers/openrouter/audio-speech'
|
||||
import { createWebSpeechAPIProvider } from './providers/web-speech-api'
|
||||
@@ -2239,6 +2240,7 @@ export const useProvidersStore = defineStore('providers', () => {
|
||||
},
|
||||
},
|
||||
},
|
||||
'google-gemini-audio-speech': buildGoogleGeminiSpeechProvider(v => baseUrlValidator.value(v)),
|
||||
}
|
||||
|
||||
const VISION_PROVIDER_ID_PREFIX = 'vision-'
|
||||
|
||||
@@ -0,0 +1,372 @@
|
||||
import type { SpeechProviderWithExtraOptions } from '@xsai-ext/providers/utils'
|
||||
|
||||
import type { ModelInfo, VoiceInfo } from '../providers'
|
||||
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
|
||||
import { buildGoogleGeminiSpeechProvider } from './google-gemini-speech'
|
||||
|
||||
function createBaseUrlValidator() {
|
||||
return (baseUrl: unknown) => {
|
||||
if (!baseUrl || typeof baseUrl !== 'string' || baseUrl.length === 0) {
|
||||
return { errors: [new Error('Base URL is required.')], reason: 'Base URL is required.', valid: false }
|
||||
}
|
||||
// Simulate the real isUrl check: a bare word without scheme is invalid
|
||||
if (typeof baseUrl === 'string' && !baseUrl.startsWith('http')) {
|
||||
return { errors: [new Error('Base URL is not absolute.')], reason: 'Base URL is not absolute.', valid: false }
|
||||
}
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
const baseUrlValidator = createBaseUrlValidator()
|
||||
|
||||
function buildProvider() {
|
||||
return buildGoogleGeminiSpeechProvider(baseUrlValidator)
|
||||
}
|
||||
|
||||
async function getSpeechProvider(config: Record<string, unknown>): Promise<SpeechProviderWithExtraOptions<string, Record<string, unknown>>> {
|
||||
const metadata = buildProvider()
|
||||
return (await metadata.createProvider(config)) as SpeechProviderWithExtraOptions<string, Record<string, unknown>>
|
||||
}
|
||||
|
||||
describe('googleGeminiSpeech provider metadata', () => {
|
||||
const metadata = buildProvider()
|
||||
|
||||
it('has the correct provider ID', () => {
|
||||
expect(metadata.id).toBe('google-gemini-audio-speech')
|
||||
})
|
||||
|
||||
it('is in the speech category', () => {
|
||||
expect(metadata.category).toBe('speech')
|
||||
})
|
||||
|
||||
it('has text-to-speech task', () => {
|
||||
expect(metadata.tasks).toContain('text-to-speech')
|
||||
})
|
||||
|
||||
it('has a name and description', () => {
|
||||
expect(metadata.name).toBe('Google Gemini')
|
||||
expect(metadata.description).toBe('aistudio.google.com')
|
||||
})
|
||||
|
||||
it('has the correct i18n keys', () => {
|
||||
expect(metadata.nameKey).toBe('settings.pages.providers.provider.google-gemini-audio-speech.title')
|
||||
expect(metadata.descriptionKey).toBe('settings.pages.providers.provider.google-gemini-audio-speech.description')
|
||||
})
|
||||
|
||||
it('has default options with baseUrl', () => {
|
||||
const defaults = metadata.defaultOptions?.()
|
||||
expect(defaults?.baseUrl).toBe('https://generativelanguage.googleapis.com/v1beta/')
|
||||
})
|
||||
})
|
||||
|
||||
describe('googleGeminiSpeech listModels', () => {
|
||||
const metadata = buildProvider()
|
||||
|
||||
it('returns three Gemini TTS models', async () => {
|
||||
const models = await metadata.capabilities.listModels?.({})
|
||||
expect(models).toHaveLength(3)
|
||||
})
|
||||
|
||||
it('includes gemini-2.5-flash-preview-tts', async () => {
|
||||
const models = await metadata.capabilities.listModels?.({})
|
||||
expect(models?.map((m: ModelInfo) => m.id)).toContain('gemini-2.5-flash-preview-tts')
|
||||
})
|
||||
|
||||
it('includes gemini-2.5-pro-preview-tts', async () => {
|
||||
const models = await metadata.capabilities.listModels?.({})
|
||||
expect(models?.map((m: ModelInfo) => m.id)).toContain('gemini-2.5-pro-preview-tts')
|
||||
})
|
||||
|
||||
it('includes gemini-3.1-flash-tts-preview', async () => {
|
||||
const models = await metadata.capabilities.listModels?.({})
|
||||
expect(models?.map((m: ModelInfo) => m.id)).toContain('gemini-3.1-flash-tts-preview')
|
||||
})
|
||||
|
||||
it('each model has the correct provider ID', async () => {
|
||||
const models = await metadata.capabilities.listModels?.({})
|
||||
for (const model of models ?? []) {
|
||||
expect(model.provider).toBe('google-gemini-audio-speech')
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
describe('googleGeminiSpeech listVoices', () => {
|
||||
const metadata = buildProvider()
|
||||
|
||||
it('returns 30 voices', async () => {
|
||||
const voices = await metadata.capabilities.listVoices?.({})
|
||||
expect(voices).toHaveLength(30)
|
||||
})
|
||||
|
||||
it('includes Kore voice', async () => {
|
||||
const voices = await metadata.capabilities.listVoices?.({})
|
||||
expect(voices?.map((v: VoiceInfo) => v.id)).toContain('Kore')
|
||||
})
|
||||
|
||||
it('each voice has compatibleModels with all three models', async () => {
|
||||
const voices = await metadata.capabilities.listVoices?.({})
|
||||
for (const voice of voices ?? []) {
|
||||
expect(voice.compatibleModels).toEqual([
|
||||
'gemini-2.5-flash-preview-tts',
|
||||
'gemini-2.5-pro-preview-tts',
|
||||
'gemini-3.1-flash-tts-preview',
|
||||
])
|
||||
}
|
||||
})
|
||||
|
||||
it('each voice has the correct provider ID', async () => {
|
||||
const voices = await metadata.capabilities.listVoices?.({})
|
||||
for (const voice of voices ?? []) {
|
||||
expect(voice.provider).toBe('google-gemini-audio-speech')
|
||||
}
|
||||
})
|
||||
|
||||
it('each voice has a description', async () => {
|
||||
const voices = await metadata.capabilities.listVoices?.({})
|
||||
for (const voice of voices ?? []) {
|
||||
expect(voice.description).toBeTruthy()
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
describe('googleGeminiSpeech validation', () => {
|
||||
const metadata = buildProvider()
|
||||
|
||||
it('fails validation without API key', async () => {
|
||||
const result = await metadata.validators.validateProviderConfig({})
|
||||
expect(result.valid).toBe(false)
|
||||
expect(result.errors.some((e: any) => e.message?.includes('API Key'))).toBe(true)
|
||||
})
|
||||
|
||||
it('fails validation with empty API key string', async () => {
|
||||
const result = await metadata.validators.validateProviderConfig({ apiKey: '' })
|
||||
expect(result.valid).toBe(false)
|
||||
})
|
||||
|
||||
it('fails validation with whitespace-only API key', async () => {
|
||||
const result = await metadata.validators.validateProviderConfig({ apiKey: ' ' })
|
||||
expect(result.valid).toBe(false)
|
||||
})
|
||||
|
||||
it('passes validation with a valid API key', async () => {
|
||||
const result = await metadata.validators.validateProviderConfig({ apiKey: 'test-api-key' })
|
||||
expect(result.valid).toBe(true)
|
||||
})
|
||||
|
||||
it('uses baseUrl validator when baseUrl is provided', async () => {
|
||||
const result = await metadata.validators.validateProviderConfig({ apiKey: 'k', baseUrl: 'invalid' })
|
||||
expect(result.valid).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
describe('googleGeminiSpeech request construction', () => {
|
||||
it('creates a speech provider with correct shape', async () => {
|
||||
const provider = await getSpeechProvider({ apiKey: 'test-key', baseUrl: 'https://example.com/v1beta' })
|
||||
const speechResult = provider.speech('gemini-2.5-flash-preview-tts')
|
||||
|
||||
expect(speechResult.model).toBe('gemini-2.5-flash-preview-tts')
|
||||
expect(speechResult.baseURL).toBe('https://example.com/v1beta/')
|
||||
expect(typeof speechResult.fetch).toBe('function')
|
||||
})
|
||||
|
||||
it('uses default model when none specified', async () => {
|
||||
const provider = await getSpeechProvider({ apiKey: 'test-key' })
|
||||
const speechResult = provider.speech('' as any)
|
||||
expect(speechResult.model).toBe('gemini-2.5-flash-preview-tts')
|
||||
})
|
||||
|
||||
it('uses default base URL when none specified', async () => {
|
||||
const provider = await getSpeechProvider({ apiKey: 'test-key' })
|
||||
const speechResult = provider.speech('gemini-2.5-flash-preview-tts')
|
||||
expect(speechResult.baseURL).toBe('https://generativelanguage.googleapis.com/v1beta/')
|
||||
})
|
||||
|
||||
it('spreads extra options into speech() return value', async () => {
|
||||
const provider = await getSpeechProvider({ apiKey: 'test-key' })
|
||||
const speechResult = provider.speech('gemini-2.5-flash-preview-tts', { temperature: 0.7 })
|
||||
expect(speechResult.temperature).toBe(0.7)
|
||||
})
|
||||
|
||||
it('explicit model argument takes precedence over options.model', async () => {
|
||||
const provider = await getSpeechProvider({ apiKey: 'test-key' })
|
||||
const speechResult = provider.speech('gemini-2.5-flash-preview-tts', { model: 'gemini-2.5-pro-preview-tts' })
|
||||
expect(speechResult.model).toBe('gemini-2.5-flash-preview-tts')
|
||||
})
|
||||
|
||||
it('fetch adapter throws when body is missing', async () => {
|
||||
const provider = await getSpeechProvider({ apiKey: 'test-key' })
|
||||
const speechResult = provider.speech('gemini-2.5-flash-preview-tts')
|
||||
const fetchFn = speechResult.fetch!
|
||||
|
||||
await expect(fetchFn(new URL('http://test'), {})).rejects.toThrow('Invalid request body')
|
||||
})
|
||||
|
||||
it('fetch adapter throws when input text is missing', async () => {
|
||||
const provider = await getSpeechProvider({ apiKey: 'test-key' })
|
||||
const speechResult = provider.speech('gemini-2.5-flash-preview-tts')
|
||||
const fetchFn = speechResult.fetch!
|
||||
|
||||
await expect(fetchFn(new URL('http://test'), {
|
||||
body: JSON.stringify({ model: 'test-model' }),
|
||||
})).rejects.toThrow('Missing input text')
|
||||
})
|
||||
|
||||
it('fetch adapter constructs correct Gemini URL', async () => {
|
||||
const mockResponse = new Response(JSON.stringify({
|
||||
candidates: [{ content: { parts: [{ inlineData: { data: 'AAAA' } }] } }],
|
||||
}), { status: 200, headers: { 'Content-Type': 'application/json' } })
|
||||
|
||||
globalThis.fetch = vi.fn().mockResolvedValue(mockResponse)
|
||||
|
||||
const provider = await getSpeechProvider({ apiKey: 'test-key', baseUrl: 'https://example.com/v1beta' })
|
||||
const speechResult = provider.speech('gemini-2.5-flash-preview-tts')
|
||||
const fetchFn = speechResult.fetch!
|
||||
|
||||
await fetchFn(new URL('http://test'), {
|
||||
body: JSON.stringify({ model: 'gemini-2.5-flash-preview-tts', input: 'Hello', voice: 'Kore' }),
|
||||
})
|
||||
|
||||
expect(globalThis.fetch).toHaveBeenCalledWith(
|
||||
'https://example.com/v1beta/models/gemini-2.5-flash-preview-tts:generateContent',
|
||||
expect.objectContaining({
|
||||
method: 'POST',
|
||||
headers: expect.objectContaining({
|
||||
'x-goog-api-key': 'test-key',
|
||||
'Content-Type': 'application/json',
|
||||
}),
|
||||
}),
|
||||
)
|
||||
})
|
||||
|
||||
it('fetch adapter sends correct Gemini request body', async () => {
|
||||
const mockResponse = new Response(JSON.stringify({
|
||||
candidates: [{ content: { parts: [{ inlineData: { data: 'AAAA' } }] } }],
|
||||
}), { status: 200, headers: { 'Content-Type': 'application/json' } })
|
||||
|
||||
globalThis.fetch = vi.fn().mockResolvedValue(mockResponse)
|
||||
|
||||
const provider = await getSpeechProvider({ apiKey: 'test-key', baseUrl: 'https://example.com/v1beta' })
|
||||
const speechResult = provider.speech('gemini-2.5-flash-preview-tts')
|
||||
const fetchFn = speechResult.fetch!
|
||||
|
||||
await fetchFn(new URL('http://test'), {
|
||||
body: JSON.stringify({ model: 'gemini-2.5-flash-preview-tts', input: 'Hello from AIRI', voice: 'Kore' }),
|
||||
})
|
||||
|
||||
const callArg = (globalThis.fetch as any).mock.calls[0][1]
|
||||
const requestBody = JSON.parse(callArg.body)
|
||||
|
||||
expect(requestBody.generationConfig.responseModalities).toEqual(['AUDIO'])
|
||||
expect(requestBody.generationConfig.speechConfig.voiceConfig.prebuiltVoiceConfig.voiceName).toBe('Kore')
|
||||
expect(requestBody.contents[0].parts[0].text).toBe('Hello from AIRI')
|
||||
})
|
||||
|
||||
it('includes temperature in generationConfig when provided', async () => {
|
||||
const mockResponse = new Response(JSON.stringify({
|
||||
candidates: [{ content: { parts: [{ inlineData: { data: 'AAAA' } }] } }],
|
||||
}), { status: 200, headers: { 'Content-Type': 'application/json' } })
|
||||
|
||||
globalThis.fetch = vi.fn().mockResolvedValue(mockResponse)
|
||||
|
||||
const provider = await getSpeechProvider({ apiKey: 'test-key' })
|
||||
const speechResult = provider.speech('gemini-2.5-flash-preview-tts')
|
||||
const fetchFn = speechResult.fetch!
|
||||
|
||||
await fetchFn(new URL('http://test'), {
|
||||
body: JSON.stringify({ model: 'gemini-2.5-flash-preview-tts', input: 'Test', voice: 'Kore', temperature: 0.7 }),
|
||||
})
|
||||
|
||||
const callArg = (globalThis.fetch as any).mock.calls[0][1]
|
||||
const requestBody = JSON.parse(callArg.body)
|
||||
|
||||
expect(requestBody.generationConfig.temperature).toBe(0.7)
|
||||
})
|
||||
|
||||
it('omits temperature from generationConfig when not provided', async () => {
|
||||
const mockResponse = new Response(JSON.stringify({
|
||||
candidates: [{ content: { parts: [{ inlineData: { data: 'AAAA' } }] } }],
|
||||
}), { status: 200, headers: { 'Content-Type': 'application/json' } })
|
||||
|
||||
globalThis.fetch = vi.fn().mockResolvedValue(mockResponse)
|
||||
|
||||
const provider = await getSpeechProvider({ apiKey: 'test-key' })
|
||||
const speechResult = provider.speech('gemini-2.5-flash-preview-tts')
|
||||
const fetchFn = speechResult.fetch!
|
||||
|
||||
await fetchFn(new URL('http://test'), {
|
||||
body: JSON.stringify({ model: 'gemini-2.5-flash-preview-tts', input: 'Test', voice: 'Kore' }),
|
||||
})
|
||||
|
||||
const callArg = (globalThis.fetch as any).mock.calls[0][1]
|
||||
const requestBody = JSON.parse(callArg.body)
|
||||
|
||||
expect(requestBody.generationConfig.temperature).toBeUndefined()
|
||||
})
|
||||
})
|
||||
|
||||
describe('googleGeminiSpeech audio conversion', () => {
|
||||
it('returns WAV audio from Gemini response', async () => {
|
||||
const pcmBase64 = btoa('\x00\x00\x00\x00')
|
||||
const mockResponse = new Response(JSON.stringify({
|
||||
candidates: [{ content: { parts: [{ inlineData: { data: pcmBase64 } }] } }],
|
||||
}), { status: 200, headers: { 'Content-Type': 'application/json' } })
|
||||
|
||||
globalThis.fetch = vi.fn().mockResolvedValue(mockResponse)
|
||||
|
||||
const provider = await getSpeechProvider({ apiKey: 'test-key' })
|
||||
const speechResult = provider.speech('gemini-2.5-flash-preview-tts')
|
||||
const fetchFn = speechResult.fetch!
|
||||
|
||||
const response = await fetchFn(new URL('http://test'), {
|
||||
body: JSON.stringify({ model: 'gemini-2.5-flash-preview-tts', input: 'Test', voice: 'Kore' }),
|
||||
})
|
||||
|
||||
expect(response.headers.get('Content-Type')).toBe('audio/wav')
|
||||
|
||||
const buffer = await response.arrayBuffer()
|
||||
const bytes = new Uint8Array(buffer)
|
||||
|
||||
// RIFF header
|
||||
expect(String.fromCharCode(bytes[0])).toBe('R')
|
||||
expect(String.fromCharCode(bytes[1])).toBe('I')
|
||||
expect(String.fromCharCode(bytes[2])).toBe('F')
|
||||
expect(String.fromCharCode(bytes[3])).toBe('F')
|
||||
|
||||
// WAVE format
|
||||
expect(String.fromCharCode(bytes[8])).toBe('W')
|
||||
expect(String.fromCharCode(bytes[9])).toBe('A')
|
||||
expect(String.fromCharCode(bytes[10])).toBe('V')
|
||||
expect(String.fromCharCode(bytes[11])).toBe('E')
|
||||
})
|
||||
|
||||
it('throws when Gemini response has no audio data', async () => {
|
||||
const mockResponse = new Response(JSON.stringify({
|
||||
candidates: [{ content: { parts: [{ text: 'no audio here' }] } }],
|
||||
}), { status: 200, headers: { 'Content-Type': 'application/json' } })
|
||||
|
||||
globalThis.fetch = vi.fn().mockResolvedValue(mockResponse)
|
||||
|
||||
const provider = await getSpeechProvider({ apiKey: 'test-key' })
|
||||
const speechResult = provider.speech('gemini-2.5-flash-preview-tts')
|
||||
const fetchFn = speechResult.fetch!
|
||||
|
||||
await expect(fetchFn(new URL('http://test'), {
|
||||
body: JSON.stringify({ model: 'gemini-2.5-flash-preview-tts', input: 'Test', voice: 'Kore' }),
|
||||
})).rejects.toThrow('Gemini TTS response missing audio data')
|
||||
})
|
||||
|
||||
it('throws when Gemini API returns non-OK status', async () => {
|
||||
const mockResponse = new Response('Unauthorized', { status: 401 })
|
||||
globalThis.fetch = vi.fn().mockResolvedValue(mockResponse)
|
||||
|
||||
const provider = await getSpeechProvider({ apiKey: 'bad-key' })
|
||||
const speechResult = provider.speech('gemini-2.5-flash-preview-tts')
|
||||
const fetchFn = speechResult.fetch!
|
||||
|
||||
await expect(fetchFn(new URL('http://test'), {
|
||||
body: JSON.stringify({ model: 'gemini-2.5-flash-preview-tts', input: 'Test', voice: 'Kore' }),
|
||||
})).rejects.toThrow('Gemini TTS request failed: 401')
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,265 @@
|
||||
import type { SpeechProviderWithExtraOptions } from '@xsai-ext/providers/utils'
|
||||
|
||||
import type { ModelInfo, ProviderMetadata, VoiceInfo } from '../providers'
|
||||
|
||||
const PROVIDER_ID = 'google-gemini-audio-speech'
|
||||
const DEFAULT_BASE_URL = 'https://generativelanguage.googleapis.com/v1beta'
|
||||
const DEFAULT_MODEL = 'gemini-2.5-flash-preview-tts'
|
||||
|
||||
const GOOGLE_GEMINI_TTS_MODELS = [
|
||||
'gemini-2.5-flash-preview-tts',
|
||||
'gemini-2.5-pro-preview-tts',
|
||||
'gemini-3.1-flash-tts-preview',
|
||||
] as const
|
||||
|
||||
const GOOGLE_GEMINI_TTS_VOICES: [string, string][] = [
|
||||
['Zephyr', 'Bright'],
|
||||
['Puck', 'Upbeat'],
|
||||
['Charon', 'Informative'],
|
||||
['Kore', 'Firm'],
|
||||
['Fenrir', 'Excitable'],
|
||||
['Leda', 'Youthful'],
|
||||
['Orus', 'Firm'],
|
||||
['Aoede', 'Breezy'],
|
||||
['Callirrhoe', 'Easy-going'],
|
||||
['Autonoe', 'Bright'],
|
||||
['Enceladus', 'Breathy'],
|
||||
['Iapetus', 'Clear'],
|
||||
['Umbriel', 'Easy-going'],
|
||||
['Algieba', 'Smooth'],
|
||||
['Despina', 'Smooth'],
|
||||
['Erinome', 'Clear'],
|
||||
['Algenib', 'Gravelly'],
|
||||
['Rasalgethi', 'Informative'],
|
||||
['Laomedeia', 'Upbeat'],
|
||||
['Achernar', 'Soft'],
|
||||
['Alnilam', 'Firm'],
|
||||
['Schedar', 'Even'],
|
||||
['Gacrux', 'Mature'],
|
||||
['Pulcherrima', 'Forward'],
|
||||
['Achird', 'Friendly'],
|
||||
['Zubenelgenubi', 'Casual'],
|
||||
['Vindemiatrix', 'Gentle'],
|
||||
['Sadachbia', 'Lively'],
|
||||
['Sadaltager', 'Knowledgeable'],
|
||||
['Sulafat', 'Warm'],
|
||||
]
|
||||
|
||||
/** Wraps raw PCM16 mono data in a minimal WAV container. */
|
||||
function wrapPCM16InWAV(pcmBytes: Uint8Array, sampleRate = 24000): Uint8Array {
|
||||
const numChannels = 1
|
||||
const bitsPerSample = 16
|
||||
const byteRate = sampleRate * numChannels * (bitsPerSample / 8)
|
||||
const blockAlign = numChannels * (bitsPerSample / 8)
|
||||
const header = new ArrayBuffer(44)
|
||||
const view = new DataView(header)
|
||||
|
||||
const writeStr = (offset: number, str: string) => {
|
||||
for (let i = 0; i < str.length; i++)
|
||||
view.setUint8(offset + i, str.charCodeAt(i))
|
||||
}
|
||||
|
||||
writeStr(0, 'RIFF')
|
||||
view.setUint32(4, 36 + pcmBytes.length, true)
|
||||
writeStr(8, 'WAVE')
|
||||
|
||||
writeStr(12, 'fmt ')
|
||||
view.setUint32(16, 16, true)
|
||||
view.setUint16(20, 1, true)
|
||||
view.setUint16(22, numChannels, true)
|
||||
view.setUint32(24, sampleRate, true)
|
||||
view.setUint32(28, byteRate, true)
|
||||
view.setUint16(32, blockAlign, true)
|
||||
view.setUint16(34, bitsPerSample, true)
|
||||
|
||||
writeStr(36, 'data')
|
||||
view.setUint32(40, pcmBytes.length, true)
|
||||
|
||||
const wav = new Uint8Array(44 + pcmBytes.length)
|
||||
wav.set(new Uint8Array(header), 0)
|
||||
wav.set(pcmBytes, 44)
|
||||
return wav
|
||||
}
|
||||
|
||||
/** Decodes a base64 string into a Uint8Array. */
|
||||
function base64ToBytes(base64: string): Uint8Array {
|
||||
const binaryString = atob(base64)
|
||||
const bytes = new Uint8Array(binaryString.length)
|
||||
for (let i = 0; i < binaryString.length; i++)
|
||||
bytes[i] = binaryString.charCodeAt(i)
|
||||
return bytes
|
||||
}
|
||||
|
||||
function normalizeBaseUrl(value: unknown): string {
|
||||
let base = typeof value === 'string' ? value.trim() : ''
|
||||
if (!base)
|
||||
base = DEFAULT_BASE_URL
|
||||
if (!base.endsWith('/'))
|
||||
base += '/'
|
||||
return base
|
||||
}
|
||||
|
||||
function normalizeApiKey(value: unknown): string {
|
||||
return typeof value === 'string' ? value.trim() : ''
|
||||
}
|
||||
|
||||
/**
|
||||
* Custom fetch adapter that translates an OpenAI-compatible TTS request
|
||||
* into a Gemini generateContent call with AUDIO response modality.
|
||||
*/
|
||||
function createAudioFetch(apiKey: string, baseUrl: string) {
|
||||
return async (_input: RequestInfo | URL, init?: RequestInit): Promise<Response> => {
|
||||
if (!init?.body || typeof init.body !== 'string')
|
||||
throw new Error('Invalid request body')
|
||||
|
||||
const body = JSON.parse(init.body)
|
||||
const model = body.model as string
|
||||
const input = body.input as string
|
||||
const temperature = typeof body.temperature === 'number' ? body.temperature : undefined
|
||||
|
||||
if (!input)
|
||||
throw new Error('Missing input text for Gemini TTS')
|
||||
if (!model)
|
||||
throw new Error('Missing model for Gemini TTS')
|
||||
|
||||
function buildGenerationConfig(): Record<string, unknown> {
|
||||
const voiceConfig: Record<string, unknown> = {
|
||||
prebuiltVoiceConfig: {
|
||||
voiceName: (body.voice as string) || 'Kore',
|
||||
},
|
||||
}
|
||||
|
||||
return {
|
||||
responseModalities: ['AUDIO'],
|
||||
speechConfig: { voiceConfig },
|
||||
...(temperature !== undefined ? { temperature } : {}),
|
||||
}
|
||||
}
|
||||
|
||||
const response = await globalThis.fetch(`${baseUrl}models/${model}:generateContent`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'x-goog-api-key': apiKey,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
contents: [
|
||||
{
|
||||
parts: [
|
||||
{ text: input },
|
||||
],
|
||||
},
|
||||
],
|
||||
generationConfig: buildGenerationConfig(),
|
||||
}),
|
||||
})
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text().catch(() => '')
|
||||
throw new Error(`Gemini TTS request failed: ${response.status} ${errorText}`)
|
||||
}
|
||||
|
||||
const json = await response.json()
|
||||
const audioBase64 = json.candidates?.[0]?.content?.parts?.find(
|
||||
(part: { inlineData?: { data?: string } }) => part.inlineData,
|
||||
)?.inlineData?.data
|
||||
|
||||
if (!audioBase64) {
|
||||
throw new Error('Gemini TTS response missing audio data')
|
||||
}
|
||||
|
||||
const pcmBytes = base64ToBytes(audioBase64)
|
||||
const wavBytes = wrapPCM16InWAV(pcmBytes)
|
||||
|
||||
// NOTICE: wrapPCM16InWAV always creates a fresh Uint8Array, so .buffer is the full
|
||||
// backing ArrayBuffer (not a subarray view into a larger buffer). The `as ArrayBuffer`
|
||||
// cast is needed because .buffer returns ArrayBufferLike in newer TypeScript.
|
||||
return new Response(wavBytes.buffer as ArrayBuffer, {
|
||||
status: 200,
|
||||
headers: { 'Content-Type': 'audio/wav' },
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
function createSpeechProvider(apiKey: string, baseUrl: string): SpeechProviderWithExtraOptions<string, Record<string, unknown>> {
|
||||
return {
|
||||
speech: (model?: string, options?: Record<string, unknown>) => ({
|
||||
baseURL: `${baseUrl}`,
|
||||
fetch: createAudioFetch(apiKey, baseUrl),
|
||||
...options,
|
||||
model: model || (options?.model as string | undefined) || DEFAULT_MODEL,
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
function listModels(): ModelInfo[] {
|
||||
return GOOGLE_GEMINI_TTS_MODELS.map(id => ({
|
||||
id,
|
||||
name: id
|
||||
.split('-')
|
||||
.map(w => w.charAt(0).toUpperCase() + w.slice(1))
|
||||
.join(' '),
|
||||
provider: PROVIDER_ID,
|
||||
description: 'Gemini API text-to-speech model',
|
||||
capabilities: ['text-to-speech'],
|
||||
} satisfies ModelInfo))
|
||||
}
|
||||
|
||||
function listVoices(): VoiceInfo[] {
|
||||
return GOOGLE_GEMINI_TTS_VOICES.map(([voiceName, style]) => ({
|
||||
id: voiceName,
|
||||
name: voiceName,
|
||||
provider: PROVIDER_ID,
|
||||
description: style,
|
||||
languages: [{ code: 'auto', title: 'Auto' }],
|
||||
compatibleModels: [...GOOGLE_GEMINI_TTS_MODELS],
|
||||
} satisfies VoiceInfo))
|
||||
}
|
||||
|
||||
export function buildGoogleGeminiSpeechProvider(
|
||||
baseUrlValidator: (baseUrl: unknown) => { errors: unknown[], reason: string, valid: boolean } | null | undefined,
|
||||
): ProviderMetadata {
|
||||
return {
|
||||
id: PROVIDER_ID,
|
||||
category: 'speech',
|
||||
tasks: ['text-to-speech', 'tts'],
|
||||
nameKey: 'settings.pages.providers.provider.google-gemini-audio-speech.title',
|
||||
name: 'Google Gemini',
|
||||
descriptionKey: 'settings.pages.providers.provider.google-gemini-audio-speech.description',
|
||||
description: 'aistudio.google.com',
|
||||
icon: 'i-lobe-icons:gemini',
|
||||
iconColor: 'i-lobe-icons:gemini-color',
|
||||
defaultOptions: () => ({
|
||||
baseUrl: `${DEFAULT_BASE_URL}/`,
|
||||
}),
|
||||
createProvider: async (config: Record<string, unknown>) => {
|
||||
const apiKey = normalizeApiKey(config.apiKey)
|
||||
const baseUrl = normalizeBaseUrl(config.baseUrl)
|
||||
return createSpeechProvider(apiKey, baseUrl)
|
||||
},
|
||||
capabilities: {
|
||||
listModels: async () => listModels(),
|
||||
listVoices: async () => listVoices(),
|
||||
},
|
||||
validators: {
|
||||
chatPingCheckAvailable: false,
|
||||
validateProviderConfig: (config: Record<string, unknown>) => {
|
||||
const errors: Error[] = []
|
||||
if (!normalizeApiKey(config.apiKey))
|
||||
errors.push(new Error('API Key is required.'))
|
||||
|
||||
if (config.baseUrl) {
|
||||
const res = baseUrlValidator(config.baseUrl)
|
||||
if (res)
|
||||
return res
|
||||
}
|
||||
|
||||
return {
|
||||
errors,
|
||||
reason: errors.map(e => e.message).join(', '),
|
||||
valid: errors.length === 0,
|
||||
}
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user