feat(stage-ui): add Google Gemini TTS provider (#1828)

---------

Co-authored-by: leiyutian <leiyutian@echo.tech>
Co-authored-by-agent: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
Ryan Lei
2026-06-18 06:41:45 +00:00
committed by GitHub
co-authored by leiyutian
parent 4a9275cb45
commit 70d1c317df
9 changed files with 819 additions and 5 deletions
@@ -1283,6 +1283,9 @@ pages:
fireworks:
description: fireworks.ai
title: Fireworks.ai
google-gemini-audio-speech:
description: aistudio.google.com
title: Google Gemini
microsoft-speech:
description: speech.microsoft.com
fields:
@@ -77,7 +77,7 @@ function handleDeleteProvider(providerId: string) {
<fieldset
v-if="persistedChatProvidersMetadata.length > 0"
flex="~ row gap-4"
min-w-0 of-x-auto scroll-smooth
min-w-0 overflow-x-auto scroll-smooth
role="radiogroup"
>
<RadioCardSimple
@@ -535,7 +535,7 @@ onUnmounted(() => {
<fieldset
v-if="configuredTranscriptionProvidersMetadata.length > 0"
flex="~ row gap-4"
min-w-0 of-x-auto scroll-smooth
min-w-0 overflow-x-auto scroll-smooth
role="radiogroup"
>
<RadioCardSimple
@@ -344,7 +344,6 @@ function handleDeleteProvider(providerId: string) {
<span>{{ t('settings.pages.modules.speech.sections.section.voice-pack.description') }}</span>
</div>
</div>
<div v-if="isLoadingVoicePacks" :class="['flex items-center gap-2', 'text-sm text-neutral-400 dark:text-neutral-500']">
<div i-solar:spinner-line-duotone class="animate-spin text-base" />
<span>{{ t('settings.pages.modules.speech.sections.section.voice-pack.loading') }}</span>
@@ -398,7 +397,7 @@ function handleDeleteProvider(providerId: string) {
<div max-w-full>
<fieldset
v-if="selectableSpeechProvidersMetadata.length > 0" flex="~ row gap-4"
min-w-0 of-x-auto scroll-smooth role="radiogroup"
min-w-0 overflow-x-auto scroll-smooth role="radiogroup"
>
<RadioCardSimple
v-for="metadata in selectableSpeechProvidersMetadata"
@@ -104,7 +104,7 @@ function formatRelativeTime(timestamp: number | null) {
<div :class="['max-w-full']">
<fieldset
v-if="persistedVisionProvidersMetadata.length > 0"
:class="['flex', 'min-w-0', 'flex-row', 'gap-4', 'of-x-auto', 'scroll-smooth']"
:class="['flex', 'min-w-0', 'flex-row', 'gap-4', 'overflow-x-auto', 'scroll-smooth']"
role="radiogroup"
>
<RadioCardSimple
@@ -0,0 +1,173 @@
<script setup lang="ts">
import type { SpeechProvider } from '@xsai-ext/providers/utils'
import {
Alert,
SpeechPlayground,
SpeechProviderSettings,
} from '@proj-airi/stage-ui/components'
import { useProviderValidation } from '@proj-airi/stage-ui/composables/use-provider-validation'
import { useSpeechStore } from '@proj-airi/stage-ui/stores/modules/speech'
import { useProvidersStore } from '@proj-airi/stage-ui/stores/providers'
import { FieldCombobox, FieldRange } from '@proj-airi/ui'
import { storeToRefs } from 'pinia'
import { computed, onMounted } from 'vue'
import { useI18n } from 'vue-i18n'
const speechStore = useSpeechStore()
const providersStore = useProvidersStore()
const { providers } = storeToRefs(providersStore)
const { t } = useI18n()
interface GoogleGeminiSpeechProviderConfig {
apiKey?: string
baseUrl?: string
model?: string
voice?: string
temperature?: number
}
const providerId = 'google-gemini-audio-speech'
const defaultModel = 'gemini-2.5-flash-preview-tts'
const config = computed(() => providers.value[providerId] as GoogleGeminiSpeechProviderConfig | undefined)
function ensureProviderConfig(): GoogleGeminiSpeechProviderConfig {
if (!providers.value[providerId])
providers.value[providerId] = {}
return providers.value[providerId] as GoogleGeminiSpeechProviderConfig
}
const providerModels = computed(() => providersStore.getModelsForProvider(providerId))
const modelOptions = computed(() => {
return (providerModels.value.length > 0 ? providerModels.value : []).map(model => ({
value: model.id,
label: model.name,
}))
})
const availableVoices = computed(() => speechStore.availableVoices[providerId] || [])
const model = computed({
get: () => config.value?.model || defaultModel,
set: (value) => {
ensureProviderConfig().model = value
},
})
const temperature = computed({
get: () => config.value?.temperature ?? 1.0,
set: (value) => {
ensureProviderConfig().temperature = value
},
})
const apiKeyConfigured = computed(() => !!providers.value[providerId]?.apiKey)
onMounted(async () => {
ensureProviderConfig()
if (!config.value?.model)
model.value = defaultModel
await providersStore.loadModelsForConfiguredProviders()
await providersStore.fetchModelsForProvider(providerId)
await speechStore.loadVoicesForProvider(providerId)
})
async function handleGenerateSpeech(input: string, voiceId: string, _useSSML: boolean, modelId?: string) {
const provider = await providersStore.getProviderInstance<SpeechProvider<string>>(providerId)
if (!provider)
throw new Error('Failed to initialize speech provider')
const providerConfig = providersStore.getProviderConfig(providerId)
const modelToUse = modelId || model.value || defaultModel
const voiceToUse = voiceId || '' as string
return await speechStore.speech(
provider,
modelToUse,
input,
voiceToUse,
providerConfig,
)
}
const {
isValidating,
isValid,
validationMessage,
forceValid,
} = useProviderValidation(providerId)
</script>
<template>
<SpeechProviderSettings
:provider-id="providerId"
:default-model="defaultModel"
>
<template #voice-settings>
<FieldCombobox
v-model="model"
label="Model"
description="Select the Gemini TTS model to use for speech generation"
:options="modelOptions"
placeholder="Select a Gemini model..."
/>
<FieldRange
v-model="temperature"
label="Temperature"
description="Controls randomness in speech generation. Lower values make speech more predictable, higher values make it more creative."
:min="0"
:max="2"
:step="0.1"
:format-value="(value) => value.toFixed(1)"
/>
</template>
<template #playground>
<SpeechPlayground
:available-voices="availableVoices"
:generate-speech="handleGenerateSpeech"
:api-key-configured="apiKeyConfigured"
:voices-loading="speechStore.isLoadingSpeechProviderVoices"
default-text="Hello! This is a test of the Google Gemini Speech."
/>
</template>
<template #advanced-settings>
<Alert v-if="!isValid && isValidating === 0 && validationMessage" type="error">
<template #title>
<div class="w-full flex items-center justify-between">
<span>{{ t('settings.dialogs.onboarding.validationFailed') }}</span>
<button
type="button"
class="ml-2 rounded bg-red-100 px-2 py-0.5 text-xs text-red-600 font-medium transition-colors dark:bg-red-800/30 hover:bg-red-200 dark:text-red-300 dark:hover:bg-red-700/40"
@click="forceValid"
>
{{ t('settings.pages.providers.common.continueAnyway') }}
</button>
</div>
</template>
<template v-if="validationMessage" #content>
<div class="whitespace-pre-wrap break-all">
{{ validationMessage }}
</div>
</template>
</Alert>
<Alert v-if="isValid && isValidating === 0" type="success">
<template #title>
{{ t('settings.dialogs.onboarding.validationSuccess') }}
</template>
</Alert>
</template>
</SpeechProviderSettings>
</template>
<route lang="yaml">
meta:
layout: settings
stageTransition:
name: slide
</route>
@@ -59,6 +59,7 @@ import { useAuthStore } from './auth'
import { createAliyunNLSProvider as createAliyunNlsStreamProvider } from './providers/aliyun/stream-transcription'
import { convertProviderDefinitionsToMetadata } from './providers/converters'
import { models as elevenLabsModels } from './providers/elevenlabs/list-models'
import { buildGoogleGeminiSpeechProvider } from './providers/google-gemini-speech'
import { buildOpenAICompatibleProvider } from './providers/openai-compatible-builder'
import { buildOpenRouterAudioSpeechProvider } from './providers/openrouter/audio-speech'
import { createWebSpeechAPIProvider } from './providers/web-speech-api'
@@ -2239,6 +2240,7 @@ export const useProvidersStore = defineStore('providers', () => {
},
},
},
'google-gemini-audio-speech': buildGoogleGeminiSpeechProvider(v => baseUrlValidator.value(v)),
}
const VISION_PROVIDER_ID_PREFIX = 'vision-'
@@ -0,0 +1,372 @@
import type { SpeechProviderWithExtraOptions } from '@xsai-ext/providers/utils'
import type { ModelInfo, VoiceInfo } from '../providers'
import { describe, expect, it, vi } from 'vitest'
import { buildGoogleGeminiSpeechProvider } from './google-gemini-speech'
function createBaseUrlValidator() {
return (baseUrl: unknown) => {
if (!baseUrl || typeof baseUrl !== 'string' || baseUrl.length === 0) {
return { errors: [new Error('Base URL is required.')], reason: 'Base URL is required.', valid: false }
}
// Simulate the real isUrl check: a bare word without scheme is invalid
if (typeof baseUrl === 'string' && !baseUrl.startsWith('http')) {
return { errors: [new Error('Base URL is not absolute.')], reason: 'Base URL is not absolute.', valid: false }
}
return null
}
}
const baseUrlValidator = createBaseUrlValidator()
function buildProvider() {
return buildGoogleGeminiSpeechProvider(baseUrlValidator)
}
async function getSpeechProvider(config: Record<string, unknown>): Promise<SpeechProviderWithExtraOptions<string, Record<string, unknown>>> {
const metadata = buildProvider()
return (await metadata.createProvider(config)) as SpeechProviderWithExtraOptions<string, Record<string, unknown>>
}
describe('googleGeminiSpeech provider metadata', () => {
const metadata = buildProvider()
it('has the correct provider ID', () => {
expect(metadata.id).toBe('google-gemini-audio-speech')
})
it('is in the speech category', () => {
expect(metadata.category).toBe('speech')
})
it('has text-to-speech task', () => {
expect(metadata.tasks).toContain('text-to-speech')
})
it('has a name and description', () => {
expect(metadata.name).toBe('Google Gemini')
expect(metadata.description).toBe('aistudio.google.com')
})
it('has the correct i18n keys', () => {
expect(metadata.nameKey).toBe('settings.pages.providers.provider.google-gemini-audio-speech.title')
expect(metadata.descriptionKey).toBe('settings.pages.providers.provider.google-gemini-audio-speech.description')
})
it('has default options with baseUrl', () => {
const defaults = metadata.defaultOptions?.()
expect(defaults?.baseUrl).toBe('https://generativelanguage.googleapis.com/v1beta/')
})
})
describe('googleGeminiSpeech listModels', () => {
const metadata = buildProvider()
it('returns three Gemini TTS models', async () => {
const models = await metadata.capabilities.listModels?.({})
expect(models).toHaveLength(3)
})
it('includes gemini-2.5-flash-preview-tts', async () => {
const models = await metadata.capabilities.listModels?.({})
expect(models?.map((m: ModelInfo) => m.id)).toContain('gemini-2.5-flash-preview-tts')
})
it('includes gemini-2.5-pro-preview-tts', async () => {
const models = await metadata.capabilities.listModels?.({})
expect(models?.map((m: ModelInfo) => m.id)).toContain('gemini-2.5-pro-preview-tts')
})
it('includes gemini-3.1-flash-tts-preview', async () => {
const models = await metadata.capabilities.listModels?.({})
expect(models?.map((m: ModelInfo) => m.id)).toContain('gemini-3.1-flash-tts-preview')
})
it('each model has the correct provider ID', async () => {
const models = await metadata.capabilities.listModels?.({})
for (const model of models ?? []) {
expect(model.provider).toBe('google-gemini-audio-speech')
}
})
})
describe('googleGeminiSpeech listVoices', () => {
const metadata = buildProvider()
it('returns 30 voices', async () => {
const voices = await metadata.capabilities.listVoices?.({})
expect(voices).toHaveLength(30)
})
it('includes Kore voice', async () => {
const voices = await metadata.capabilities.listVoices?.({})
expect(voices?.map((v: VoiceInfo) => v.id)).toContain('Kore')
})
it('each voice has compatibleModels with all three models', async () => {
const voices = await metadata.capabilities.listVoices?.({})
for (const voice of voices ?? []) {
expect(voice.compatibleModels).toEqual([
'gemini-2.5-flash-preview-tts',
'gemini-2.5-pro-preview-tts',
'gemini-3.1-flash-tts-preview',
])
}
})
it('each voice has the correct provider ID', async () => {
const voices = await metadata.capabilities.listVoices?.({})
for (const voice of voices ?? []) {
expect(voice.provider).toBe('google-gemini-audio-speech')
}
})
it('each voice has a description', async () => {
const voices = await metadata.capabilities.listVoices?.({})
for (const voice of voices ?? []) {
expect(voice.description).toBeTruthy()
}
})
})
describe('googleGeminiSpeech validation', () => {
const metadata = buildProvider()
it('fails validation without API key', async () => {
const result = await metadata.validators.validateProviderConfig({})
expect(result.valid).toBe(false)
expect(result.errors.some((e: any) => e.message?.includes('API Key'))).toBe(true)
})
it('fails validation with empty API key string', async () => {
const result = await metadata.validators.validateProviderConfig({ apiKey: '' })
expect(result.valid).toBe(false)
})
it('fails validation with whitespace-only API key', async () => {
const result = await metadata.validators.validateProviderConfig({ apiKey: ' ' })
expect(result.valid).toBe(false)
})
it('passes validation with a valid API key', async () => {
const result = await metadata.validators.validateProviderConfig({ apiKey: 'test-api-key' })
expect(result.valid).toBe(true)
})
it('uses baseUrl validator when baseUrl is provided', async () => {
const result = await metadata.validators.validateProviderConfig({ apiKey: 'k', baseUrl: 'invalid' })
expect(result.valid).toBe(false)
})
})
describe('googleGeminiSpeech request construction', () => {
it('creates a speech provider with correct shape', async () => {
const provider = await getSpeechProvider({ apiKey: 'test-key', baseUrl: 'https://example.com/v1beta' })
const speechResult = provider.speech('gemini-2.5-flash-preview-tts')
expect(speechResult.model).toBe('gemini-2.5-flash-preview-tts')
expect(speechResult.baseURL).toBe('https://example.com/v1beta/')
expect(typeof speechResult.fetch).toBe('function')
})
it('uses default model when none specified', async () => {
const provider = await getSpeechProvider({ apiKey: 'test-key' })
const speechResult = provider.speech('' as any)
expect(speechResult.model).toBe('gemini-2.5-flash-preview-tts')
})
it('uses default base URL when none specified', async () => {
const provider = await getSpeechProvider({ apiKey: 'test-key' })
const speechResult = provider.speech('gemini-2.5-flash-preview-tts')
expect(speechResult.baseURL).toBe('https://generativelanguage.googleapis.com/v1beta/')
})
it('spreads extra options into speech() return value', async () => {
const provider = await getSpeechProvider({ apiKey: 'test-key' })
const speechResult = provider.speech('gemini-2.5-flash-preview-tts', { temperature: 0.7 })
expect(speechResult.temperature).toBe(0.7)
})
it('explicit model argument takes precedence over options.model', async () => {
const provider = await getSpeechProvider({ apiKey: 'test-key' })
const speechResult = provider.speech('gemini-2.5-flash-preview-tts', { model: 'gemini-2.5-pro-preview-tts' })
expect(speechResult.model).toBe('gemini-2.5-flash-preview-tts')
})
it('fetch adapter throws when body is missing', async () => {
const provider = await getSpeechProvider({ apiKey: 'test-key' })
const speechResult = provider.speech('gemini-2.5-flash-preview-tts')
const fetchFn = speechResult.fetch!
await expect(fetchFn(new URL('http://test'), {})).rejects.toThrow('Invalid request body')
})
it('fetch adapter throws when input text is missing', async () => {
const provider = await getSpeechProvider({ apiKey: 'test-key' })
const speechResult = provider.speech('gemini-2.5-flash-preview-tts')
const fetchFn = speechResult.fetch!
await expect(fetchFn(new URL('http://test'), {
body: JSON.stringify({ model: 'test-model' }),
})).rejects.toThrow('Missing input text')
})
it('fetch adapter constructs correct Gemini URL', async () => {
const mockResponse = new Response(JSON.stringify({
candidates: [{ content: { parts: [{ inlineData: { data: 'AAAA' } }] } }],
}), { status: 200, headers: { 'Content-Type': 'application/json' } })
globalThis.fetch = vi.fn().mockResolvedValue(mockResponse)
const provider = await getSpeechProvider({ apiKey: 'test-key', baseUrl: 'https://example.com/v1beta' })
const speechResult = provider.speech('gemini-2.5-flash-preview-tts')
const fetchFn = speechResult.fetch!
await fetchFn(new URL('http://test'), {
body: JSON.stringify({ model: 'gemini-2.5-flash-preview-tts', input: 'Hello', voice: 'Kore' }),
})
expect(globalThis.fetch).toHaveBeenCalledWith(
'https://example.com/v1beta/models/gemini-2.5-flash-preview-tts:generateContent',
expect.objectContaining({
method: 'POST',
headers: expect.objectContaining({
'x-goog-api-key': 'test-key',
'Content-Type': 'application/json',
}),
}),
)
})
it('fetch adapter sends correct Gemini request body', async () => {
const mockResponse = new Response(JSON.stringify({
candidates: [{ content: { parts: [{ inlineData: { data: 'AAAA' } }] } }],
}), { status: 200, headers: { 'Content-Type': 'application/json' } })
globalThis.fetch = vi.fn().mockResolvedValue(mockResponse)
const provider = await getSpeechProvider({ apiKey: 'test-key', baseUrl: 'https://example.com/v1beta' })
const speechResult = provider.speech('gemini-2.5-flash-preview-tts')
const fetchFn = speechResult.fetch!
await fetchFn(new URL('http://test'), {
body: JSON.stringify({ model: 'gemini-2.5-flash-preview-tts', input: 'Hello from AIRI', voice: 'Kore' }),
})
const callArg = (globalThis.fetch as any).mock.calls[0][1]
const requestBody = JSON.parse(callArg.body)
expect(requestBody.generationConfig.responseModalities).toEqual(['AUDIO'])
expect(requestBody.generationConfig.speechConfig.voiceConfig.prebuiltVoiceConfig.voiceName).toBe('Kore')
expect(requestBody.contents[0].parts[0].text).toBe('Hello from AIRI')
})
it('includes temperature in generationConfig when provided', async () => {
const mockResponse = new Response(JSON.stringify({
candidates: [{ content: { parts: [{ inlineData: { data: 'AAAA' } }] } }],
}), { status: 200, headers: { 'Content-Type': 'application/json' } })
globalThis.fetch = vi.fn().mockResolvedValue(mockResponse)
const provider = await getSpeechProvider({ apiKey: 'test-key' })
const speechResult = provider.speech('gemini-2.5-flash-preview-tts')
const fetchFn = speechResult.fetch!
await fetchFn(new URL('http://test'), {
body: JSON.stringify({ model: 'gemini-2.5-flash-preview-tts', input: 'Test', voice: 'Kore', temperature: 0.7 }),
})
const callArg = (globalThis.fetch as any).mock.calls[0][1]
const requestBody = JSON.parse(callArg.body)
expect(requestBody.generationConfig.temperature).toBe(0.7)
})
it('omits temperature from generationConfig when not provided', async () => {
const mockResponse = new Response(JSON.stringify({
candidates: [{ content: { parts: [{ inlineData: { data: 'AAAA' } }] } }],
}), { status: 200, headers: { 'Content-Type': 'application/json' } })
globalThis.fetch = vi.fn().mockResolvedValue(mockResponse)
const provider = await getSpeechProvider({ apiKey: 'test-key' })
const speechResult = provider.speech('gemini-2.5-flash-preview-tts')
const fetchFn = speechResult.fetch!
await fetchFn(new URL('http://test'), {
body: JSON.stringify({ model: 'gemini-2.5-flash-preview-tts', input: 'Test', voice: 'Kore' }),
})
const callArg = (globalThis.fetch as any).mock.calls[0][1]
const requestBody = JSON.parse(callArg.body)
expect(requestBody.generationConfig.temperature).toBeUndefined()
})
})
describe('googleGeminiSpeech audio conversion', () => {
it('returns WAV audio from Gemini response', async () => {
const pcmBase64 = btoa('\x00\x00\x00\x00')
const mockResponse = new Response(JSON.stringify({
candidates: [{ content: { parts: [{ inlineData: { data: pcmBase64 } }] } }],
}), { status: 200, headers: { 'Content-Type': 'application/json' } })
globalThis.fetch = vi.fn().mockResolvedValue(mockResponse)
const provider = await getSpeechProvider({ apiKey: 'test-key' })
const speechResult = provider.speech('gemini-2.5-flash-preview-tts')
const fetchFn = speechResult.fetch!
const response = await fetchFn(new URL('http://test'), {
body: JSON.stringify({ model: 'gemini-2.5-flash-preview-tts', input: 'Test', voice: 'Kore' }),
})
expect(response.headers.get('Content-Type')).toBe('audio/wav')
const buffer = await response.arrayBuffer()
const bytes = new Uint8Array(buffer)
// RIFF header
expect(String.fromCharCode(bytes[0])).toBe('R')
expect(String.fromCharCode(bytes[1])).toBe('I')
expect(String.fromCharCode(bytes[2])).toBe('F')
expect(String.fromCharCode(bytes[3])).toBe('F')
// WAVE format
expect(String.fromCharCode(bytes[8])).toBe('W')
expect(String.fromCharCode(bytes[9])).toBe('A')
expect(String.fromCharCode(bytes[10])).toBe('V')
expect(String.fromCharCode(bytes[11])).toBe('E')
})
it('throws when Gemini response has no audio data', async () => {
const mockResponse = new Response(JSON.stringify({
candidates: [{ content: { parts: [{ text: 'no audio here' }] } }],
}), { status: 200, headers: { 'Content-Type': 'application/json' } })
globalThis.fetch = vi.fn().mockResolvedValue(mockResponse)
const provider = await getSpeechProvider({ apiKey: 'test-key' })
const speechResult = provider.speech('gemini-2.5-flash-preview-tts')
const fetchFn = speechResult.fetch!
await expect(fetchFn(new URL('http://test'), {
body: JSON.stringify({ model: 'gemini-2.5-flash-preview-tts', input: 'Test', voice: 'Kore' }),
})).rejects.toThrow('Gemini TTS response missing audio data')
})
it('throws when Gemini API returns non-OK status', async () => {
const mockResponse = new Response('Unauthorized', { status: 401 })
globalThis.fetch = vi.fn().mockResolvedValue(mockResponse)
const provider = await getSpeechProvider({ apiKey: 'bad-key' })
const speechResult = provider.speech('gemini-2.5-flash-preview-tts')
const fetchFn = speechResult.fetch!
await expect(fetchFn(new URL('http://test'), {
body: JSON.stringify({ model: 'gemini-2.5-flash-preview-tts', input: 'Test', voice: 'Kore' }),
})).rejects.toThrow('Gemini TTS request failed: 401')
})
})
@@ -0,0 +1,265 @@
import type { SpeechProviderWithExtraOptions } from '@xsai-ext/providers/utils'
import type { ModelInfo, ProviderMetadata, VoiceInfo } from '../providers'
const PROVIDER_ID = 'google-gemini-audio-speech'
const DEFAULT_BASE_URL = 'https://generativelanguage.googleapis.com/v1beta'
const DEFAULT_MODEL = 'gemini-2.5-flash-preview-tts'
const GOOGLE_GEMINI_TTS_MODELS = [
'gemini-2.5-flash-preview-tts',
'gemini-2.5-pro-preview-tts',
'gemini-3.1-flash-tts-preview',
] as const
const GOOGLE_GEMINI_TTS_VOICES: [string, string][] = [
['Zephyr', 'Bright'],
['Puck', 'Upbeat'],
['Charon', 'Informative'],
['Kore', 'Firm'],
['Fenrir', 'Excitable'],
['Leda', 'Youthful'],
['Orus', 'Firm'],
['Aoede', 'Breezy'],
['Callirrhoe', 'Easy-going'],
['Autonoe', 'Bright'],
['Enceladus', 'Breathy'],
['Iapetus', 'Clear'],
['Umbriel', 'Easy-going'],
['Algieba', 'Smooth'],
['Despina', 'Smooth'],
['Erinome', 'Clear'],
['Algenib', 'Gravelly'],
['Rasalgethi', 'Informative'],
['Laomedeia', 'Upbeat'],
['Achernar', 'Soft'],
['Alnilam', 'Firm'],
['Schedar', 'Even'],
['Gacrux', 'Mature'],
['Pulcherrima', 'Forward'],
['Achird', 'Friendly'],
['Zubenelgenubi', 'Casual'],
['Vindemiatrix', 'Gentle'],
['Sadachbia', 'Lively'],
['Sadaltager', 'Knowledgeable'],
['Sulafat', 'Warm'],
]
/** Wraps raw PCM16 mono data in a minimal WAV container. */
function wrapPCM16InWAV(pcmBytes: Uint8Array, sampleRate = 24000): Uint8Array {
const numChannels = 1
const bitsPerSample = 16
const byteRate = sampleRate * numChannels * (bitsPerSample / 8)
const blockAlign = numChannels * (bitsPerSample / 8)
const header = new ArrayBuffer(44)
const view = new DataView(header)
const writeStr = (offset: number, str: string) => {
for (let i = 0; i < str.length; i++)
view.setUint8(offset + i, str.charCodeAt(i))
}
writeStr(0, 'RIFF')
view.setUint32(4, 36 + pcmBytes.length, true)
writeStr(8, 'WAVE')
writeStr(12, 'fmt ')
view.setUint32(16, 16, true)
view.setUint16(20, 1, true)
view.setUint16(22, numChannels, true)
view.setUint32(24, sampleRate, true)
view.setUint32(28, byteRate, true)
view.setUint16(32, blockAlign, true)
view.setUint16(34, bitsPerSample, true)
writeStr(36, 'data')
view.setUint32(40, pcmBytes.length, true)
const wav = new Uint8Array(44 + pcmBytes.length)
wav.set(new Uint8Array(header), 0)
wav.set(pcmBytes, 44)
return wav
}
/** Decodes a base64 string into a Uint8Array. */
function base64ToBytes(base64: string): Uint8Array {
const binaryString = atob(base64)
const bytes = new Uint8Array(binaryString.length)
for (let i = 0; i < binaryString.length; i++)
bytes[i] = binaryString.charCodeAt(i)
return bytes
}
function normalizeBaseUrl(value: unknown): string {
let base = typeof value === 'string' ? value.trim() : ''
if (!base)
base = DEFAULT_BASE_URL
if (!base.endsWith('/'))
base += '/'
return base
}
function normalizeApiKey(value: unknown): string {
return typeof value === 'string' ? value.trim() : ''
}
/**
* Custom fetch adapter that translates an OpenAI-compatible TTS request
* into a Gemini generateContent call with AUDIO response modality.
*/
function createAudioFetch(apiKey: string, baseUrl: string) {
return async (_input: RequestInfo | URL, init?: RequestInit): Promise<Response> => {
if (!init?.body || typeof init.body !== 'string')
throw new Error('Invalid request body')
const body = JSON.parse(init.body)
const model = body.model as string
const input = body.input as string
const temperature = typeof body.temperature === 'number' ? body.temperature : undefined
if (!input)
throw new Error('Missing input text for Gemini TTS')
if (!model)
throw new Error('Missing model for Gemini TTS')
function buildGenerationConfig(): Record<string, unknown> {
const voiceConfig: Record<string, unknown> = {
prebuiltVoiceConfig: {
voiceName: (body.voice as string) || 'Kore',
},
}
return {
responseModalities: ['AUDIO'],
speechConfig: { voiceConfig },
...(temperature !== undefined ? { temperature } : {}),
}
}
const response = await globalThis.fetch(`${baseUrl}models/${model}:generateContent`, {
method: 'POST',
headers: {
'x-goog-api-key': apiKey,
'Content-Type': 'application/json',
},
body: JSON.stringify({
contents: [
{
parts: [
{ text: input },
],
},
],
generationConfig: buildGenerationConfig(),
}),
})
if (!response.ok) {
const errorText = await response.text().catch(() => '')
throw new Error(`Gemini TTS request failed: ${response.status} ${errorText}`)
}
const json = await response.json()
const audioBase64 = json.candidates?.[0]?.content?.parts?.find(
(part: { inlineData?: { data?: string } }) => part.inlineData,
)?.inlineData?.data
if (!audioBase64) {
throw new Error('Gemini TTS response missing audio data')
}
const pcmBytes = base64ToBytes(audioBase64)
const wavBytes = wrapPCM16InWAV(pcmBytes)
// NOTICE: wrapPCM16InWAV always creates a fresh Uint8Array, so .buffer is the full
// backing ArrayBuffer (not a subarray view into a larger buffer). The `as ArrayBuffer`
// cast is needed because .buffer returns ArrayBufferLike in newer TypeScript.
return new Response(wavBytes.buffer as ArrayBuffer, {
status: 200,
headers: { 'Content-Type': 'audio/wav' },
})
}
}
function createSpeechProvider(apiKey: string, baseUrl: string): SpeechProviderWithExtraOptions<string, Record<string, unknown>> {
return {
speech: (model?: string, options?: Record<string, unknown>) => ({
baseURL: `${baseUrl}`,
fetch: createAudioFetch(apiKey, baseUrl),
...options,
model: model || (options?.model as string | undefined) || DEFAULT_MODEL,
}),
}
}
function listModels(): ModelInfo[] {
return GOOGLE_GEMINI_TTS_MODELS.map(id => ({
id,
name: id
.split('-')
.map(w => w.charAt(0).toUpperCase() + w.slice(1))
.join(' '),
provider: PROVIDER_ID,
description: 'Gemini API text-to-speech model',
capabilities: ['text-to-speech'],
} satisfies ModelInfo))
}
function listVoices(): VoiceInfo[] {
return GOOGLE_GEMINI_TTS_VOICES.map(([voiceName, style]) => ({
id: voiceName,
name: voiceName,
provider: PROVIDER_ID,
description: style,
languages: [{ code: 'auto', title: 'Auto' }],
compatibleModels: [...GOOGLE_GEMINI_TTS_MODELS],
} satisfies VoiceInfo))
}
export function buildGoogleGeminiSpeechProvider(
baseUrlValidator: (baseUrl: unknown) => { errors: unknown[], reason: string, valid: boolean } | null | undefined,
): ProviderMetadata {
return {
id: PROVIDER_ID,
category: 'speech',
tasks: ['text-to-speech', 'tts'],
nameKey: 'settings.pages.providers.provider.google-gemini-audio-speech.title',
name: 'Google Gemini',
descriptionKey: 'settings.pages.providers.provider.google-gemini-audio-speech.description',
description: 'aistudio.google.com',
icon: 'i-lobe-icons:gemini',
iconColor: 'i-lobe-icons:gemini-color',
defaultOptions: () => ({
baseUrl: `${DEFAULT_BASE_URL}/`,
}),
createProvider: async (config: Record<string, unknown>) => {
const apiKey = normalizeApiKey(config.apiKey)
const baseUrl = normalizeBaseUrl(config.baseUrl)
return createSpeechProvider(apiKey, baseUrl)
},
capabilities: {
listModels: async () => listModels(),
listVoices: async () => listVoices(),
},
validators: {
chatPingCheckAvailable: false,
validateProviderConfig: (config: Record<string, unknown>) => {
const errors: Error[] = []
if (!normalizeApiKey(config.apiKey))
errors.push(new Error('API Key is required.'))
if (config.baseUrl) {
const res = baseUrlValidator(config.baseUrl)
if (res)
return res
}
return {
errors,
reason: errors.map(e => e.message).join(', '),
valid: errors.length === 0,
}
},
},
}
}