diff --git a/packages/i18n/src/locales/en/settings.yaml b/packages/i18n/src/locales/en/settings.yaml index 21d5bb848..e6bb99113 100644 --- a/packages/i18n/src/locales/en/settings.yaml +++ b/packages/i18n/src/locales/en/settings.yaml @@ -1283,6 +1283,9 @@ pages: fireworks: description: fireworks.ai title: Fireworks.ai + google-gemini-audio-speech: + description: aistudio.google.com + title: Google Gemini microsoft-speech: description: speech.microsoft.com fields: diff --git a/packages/stage-pages/src/pages/settings/modules/consciousness.vue b/packages/stage-pages/src/pages/settings/modules/consciousness.vue index a88097e7e..cc21ad9cc 100644 --- a/packages/stage-pages/src/pages/settings/modules/consciousness.vue +++ b/packages/stage-pages/src/pages/settings/modules/consciousness.vue @@ -77,7 +77,7 @@ function handleDeleteProvider(providerId: string) {
{
{{ t('settings.pages.modules.speech.sections.section.voice-pack.description') }} -
{{ t('settings.pages.modules.speech.sections.section.voice-pack.loading') }} @@ -398,7 +397,7 @@ function handleDeleteProvider(providerId: string) {
+import type { SpeechProvider } from '@xsai-ext/providers/utils' + +import { + Alert, + SpeechPlayground, + SpeechProviderSettings, +} from '@proj-airi/stage-ui/components' +import { useProviderValidation } from '@proj-airi/stage-ui/composables/use-provider-validation' +import { useSpeechStore } from '@proj-airi/stage-ui/stores/modules/speech' +import { useProvidersStore } from '@proj-airi/stage-ui/stores/providers' +import { FieldCombobox, FieldRange } from '@proj-airi/ui' +import { storeToRefs } from 'pinia' +import { computed, onMounted } from 'vue' +import { useI18n } from 'vue-i18n' + +const speechStore = useSpeechStore() +const providersStore = useProvidersStore() +const { providers } = storeToRefs(providersStore) +const { t } = useI18n() + +interface GoogleGeminiSpeechProviderConfig { + apiKey?: string + baseUrl?: string + model?: string + voice?: string + temperature?: number +} + +const providerId = 'google-gemini-audio-speech' +const defaultModel = 'gemini-2.5-flash-preview-tts' + +const config = computed(() => providers.value[providerId] as GoogleGeminiSpeechProviderConfig | undefined) + +function ensureProviderConfig(): GoogleGeminiSpeechProviderConfig { + if (!providers.value[providerId]) + providers.value[providerId] = {} + + return providers.value[providerId] as GoogleGeminiSpeechProviderConfig +} + +const providerModels = computed(() => providersStore.getModelsForProvider(providerId)) +const modelOptions = computed(() => { + return (providerModels.value.length > 0 ? providerModels.value : []).map(model => ({ + value: model.id, + label: model.name, + })) +}) + +const availableVoices = computed(() => speechStore.availableVoices[providerId] || []) + +const model = computed({ + get: () => config.value?.model || defaultModel, + set: (value) => { + ensureProviderConfig().model = value + }, +}) + +const temperature = computed({ + get: () => config.value?.temperature ?? 1.0, + set: (value) => { + ensureProviderConfig().temperature = value + }, +}) + +const apiKeyConfigured = computed(() => !!providers.value[providerId]?.apiKey) + +onMounted(async () => { + ensureProviderConfig() + + if (!config.value?.model) + model.value = defaultModel + + await providersStore.loadModelsForConfiguredProviders() + await providersStore.fetchModelsForProvider(providerId) + await speechStore.loadVoicesForProvider(providerId) +}) + +async function handleGenerateSpeech(input: string, voiceId: string, _useSSML: boolean, modelId?: string) { + const provider = await providersStore.getProviderInstance>(providerId) + if (!provider) + throw new Error('Failed to initialize speech provider') + + const providerConfig = providersStore.getProviderConfig(providerId) + const modelToUse = modelId || model.value || defaultModel + const voiceToUse = voiceId || '' as string + + return await speechStore.speech( + provider, + modelToUse, + input, + voiceToUse, + providerConfig, + ) +} + +const { + isValidating, + isValid, + validationMessage, + forceValid, +} = useProviderValidation(providerId) + + + + + +meta: + layout: settings + stageTransition: + name: slide + diff --git a/packages/stage-ui/src/stores/providers.ts b/packages/stage-ui/src/stores/providers.ts index 5526b5ea3..360f4082f 100644 --- a/packages/stage-ui/src/stores/providers.ts +++ b/packages/stage-ui/src/stores/providers.ts @@ -59,6 +59,7 @@ import { useAuthStore } from './auth' import { createAliyunNLSProvider as createAliyunNlsStreamProvider } from './providers/aliyun/stream-transcription' import { convertProviderDefinitionsToMetadata } from './providers/converters' import { models as elevenLabsModels } from './providers/elevenlabs/list-models' +import { buildGoogleGeminiSpeechProvider } from './providers/google-gemini-speech' import { buildOpenAICompatibleProvider } from './providers/openai-compatible-builder' import { buildOpenRouterAudioSpeechProvider } from './providers/openrouter/audio-speech' import { createWebSpeechAPIProvider } from './providers/web-speech-api' @@ -2239,6 +2240,7 @@ export const useProvidersStore = defineStore('providers', () => { }, }, }, + 'google-gemini-audio-speech': buildGoogleGeminiSpeechProvider(v => baseUrlValidator.value(v)), } const VISION_PROVIDER_ID_PREFIX = 'vision-' diff --git a/packages/stage-ui/src/stores/providers/google-gemini-speech.test.ts b/packages/stage-ui/src/stores/providers/google-gemini-speech.test.ts new file mode 100644 index 000000000..fca46c6dc --- /dev/null +++ b/packages/stage-ui/src/stores/providers/google-gemini-speech.test.ts @@ -0,0 +1,372 @@ +import type { SpeechProviderWithExtraOptions } from '@xsai-ext/providers/utils' + +import type { ModelInfo, VoiceInfo } from '../providers' + +import { describe, expect, it, vi } from 'vitest' + +import { buildGoogleGeminiSpeechProvider } from './google-gemini-speech' + +function createBaseUrlValidator() { + return (baseUrl: unknown) => { + if (!baseUrl || typeof baseUrl !== 'string' || baseUrl.length === 0) { + return { errors: [new Error('Base URL is required.')], reason: 'Base URL is required.', valid: false } + } + // Simulate the real isUrl check: a bare word without scheme is invalid + if (typeof baseUrl === 'string' && !baseUrl.startsWith('http')) { + return { errors: [new Error('Base URL is not absolute.')], reason: 'Base URL is not absolute.', valid: false } + } + return null + } +} + +const baseUrlValidator = createBaseUrlValidator() + +function buildProvider() { + return buildGoogleGeminiSpeechProvider(baseUrlValidator) +} + +async function getSpeechProvider(config: Record): Promise>> { + const metadata = buildProvider() + return (await metadata.createProvider(config)) as SpeechProviderWithExtraOptions> +} + +describe('googleGeminiSpeech provider metadata', () => { + const metadata = buildProvider() + + it('has the correct provider ID', () => { + expect(metadata.id).toBe('google-gemini-audio-speech') + }) + + it('is in the speech category', () => { + expect(metadata.category).toBe('speech') + }) + + it('has text-to-speech task', () => { + expect(metadata.tasks).toContain('text-to-speech') + }) + + it('has a name and description', () => { + expect(metadata.name).toBe('Google Gemini') + expect(metadata.description).toBe('aistudio.google.com') + }) + + it('has the correct i18n keys', () => { + expect(metadata.nameKey).toBe('settings.pages.providers.provider.google-gemini-audio-speech.title') + expect(metadata.descriptionKey).toBe('settings.pages.providers.provider.google-gemini-audio-speech.description') + }) + + it('has default options with baseUrl', () => { + const defaults = metadata.defaultOptions?.() + expect(defaults?.baseUrl).toBe('https://generativelanguage.googleapis.com/v1beta/') + }) +}) + +describe('googleGeminiSpeech listModels', () => { + const metadata = buildProvider() + + it('returns three Gemini TTS models', async () => { + const models = await metadata.capabilities.listModels?.({}) + expect(models).toHaveLength(3) + }) + + it('includes gemini-2.5-flash-preview-tts', async () => { + const models = await metadata.capabilities.listModels?.({}) + expect(models?.map((m: ModelInfo) => m.id)).toContain('gemini-2.5-flash-preview-tts') + }) + + it('includes gemini-2.5-pro-preview-tts', async () => { + const models = await metadata.capabilities.listModels?.({}) + expect(models?.map((m: ModelInfo) => m.id)).toContain('gemini-2.5-pro-preview-tts') + }) + + it('includes gemini-3.1-flash-tts-preview', async () => { + const models = await metadata.capabilities.listModels?.({}) + expect(models?.map((m: ModelInfo) => m.id)).toContain('gemini-3.1-flash-tts-preview') + }) + + it('each model has the correct provider ID', async () => { + const models = await metadata.capabilities.listModels?.({}) + for (const model of models ?? []) { + expect(model.provider).toBe('google-gemini-audio-speech') + } + }) +}) + +describe('googleGeminiSpeech listVoices', () => { + const metadata = buildProvider() + + it('returns 30 voices', async () => { + const voices = await metadata.capabilities.listVoices?.({}) + expect(voices).toHaveLength(30) + }) + + it('includes Kore voice', async () => { + const voices = await metadata.capabilities.listVoices?.({}) + expect(voices?.map((v: VoiceInfo) => v.id)).toContain('Kore') + }) + + it('each voice has compatibleModels with all three models', async () => { + const voices = await metadata.capabilities.listVoices?.({}) + for (const voice of voices ?? []) { + expect(voice.compatibleModels).toEqual([ + 'gemini-2.5-flash-preview-tts', + 'gemini-2.5-pro-preview-tts', + 'gemini-3.1-flash-tts-preview', + ]) + } + }) + + it('each voice has the correct provider ID', async () => { + const voices = await metadata.capabilities.listVoices?.({}) + for (const voice of voices ?? []) { + expect(voice.provider).toBe('google-gemini-audio-speech') + } + }) + + it('each voice has a description', async () => { + const voices = await metadata.capabilities.listVoices?.({}) + for (const voice of voices ?? []) { + expect(voice.description).toBeTruthy() + } + }) +}) + +describe('googleGeminiSpeech validation', () => { + const metadata = buildProvider() + + it('fails validation without API key', async () => { + const result = await metadata.validators.validateProviderConfig({}) + expect(result.valid).toBe(false) + expect(result.errors.some((e: any) => e.message?.includes('API Key'))).toBe(true) + }) + + it('fails validation with empty API key string', async () => { + const result = await metadata.validators.validateProviderConfig({ apiKey: '' }) + expect(result.valid).toBe(false) + }) + + it('fails validation with whitespace-only API key', async () => { + const result = await metadata.validators.validateProviderConfig({ apiKey: ' ' }) + expect(result.valid).toBe(false) + }) + + it('passes validation with a valid API key', async () => { + const result = await metadata.validators.validateProviderConfig({ apiKey: 'test-api-key' }) + expect(result.valid).toBe(true) + }) + + it('uses baseUrl validator when baseUrl is provided', async () => { + const result = await metadata.validators.validateProviderConfig({ apiKey: 'k', baseUrl: 'invalid' }) + expect(result.valid).toBe(false) + }) +}) + +describe('googleGeminiSpeech request construction', () => { + it('creates a speech provider with correct shape', async () => { + const provider = await getSpeechProvider({ apiKey: 'test-key', baseUrl: 'https://example.com/v1beta' }) + const speechResult = provider.speech('gemini-2.5-flash-preview-tts') + + expect(speechResult.model).toBe('gemini-2.5-flash-preview-tts') + expect(speechResult.baseURL).toBe('https://example.com/v1beta/') + expect(typeof speechResult.fetch).toBe('function') + }) + + it('uses default model when none specified', async () => { + const provider = await getSpeechProvider({ apiKey: 'test-key' }) + const speechResult = provider.speech('' as any) + expect(speechResult.model).toBe('gemini-2.5-flash-preview-tts') + }) + + it('uses default base URL when none specified', async () => { + const provider = await getSpeechProvider({ apiKey: 'test-key' }) + const speechResult = provider.speech('gemini-2.5-flash-preview-tts') + expect(speechResult.baseURL).toBe('https://generativelanguage.googleapis.com/v1beta/') + }) + + it('spreads extra options into speech() return value', async () => { + const provider = await getSpeechProvider({ apiKey: 'test-key' }) + const speechResult = provider.speech('gemini-2.5-flash-preview-tts', { temperature: 0.7 }) + expect(speechResult.temperature).toBe(0.7) + }) + + it('explicit model argument takes precedence over options.model', async () => { + const provider = await getSpeechProvider({ apiKey: 'test-key' }) + const speechResult = provider.speech('gemini-2.5-flash-preview-tts', { model: 'gemini-2.5-pro-preview-tts' }) + expect(speechResult.model).toBe('gemini-2.5-flash-preview-tts') + }) + + it('fetch adapter throws when body is missing', async () => { + const provider = await getSpeechProvider({ apiKey: 'test-key' }) + const speechResult = provider.speech('gemini-2.5-flash-preview-tts') + const fetchFn = speechResult.fetch! + + await expect(fetchFn(new URL('http://test'), {})).rejects.toThrow('Invalid request body') + }) + + it('fetch adapter throws when input text is missing', async () => { + const provider = await getSpeechProvider({ apiKey: 'test-key' }) + const speechResult = provider.speech('gemini-2.5-flash-preview-tts') + const fetchFn = speechResult.fetch! + + await expect(fetchFn(new URL('http://test'), { + body: JSON.stringify({ model: 'test-model' }), + })).rejects.toThrow('Missing input text') + }) + + it('fetch adapter constructs correct Gemini URL', async () => { + const mockResponse = new Response(JSON.stringify({ + candidates: [{ content: { parts: [{ inlineData: { data: 'AAAA' } }] } }], + }), { status: 200, headers: { 'Content-Type': 'application/json' } }) + + globalThis.fetch = vi.fn().mockResolvedValue(mockResponse) + + const provider = await getSpeechProvider({ apiKey: 'test-key', baseUrl: 'https://example.com/v1beta' }) + const speechResult = provider.speech('gemini-2.5-flash-preview-tts') + const fetchFn = speechResult.fetch! + + await fetchFn(new URL('http://test'), { + body: JSON.stringify({ model: 'gemini-2.5-flash-preview-tts', input: 'Hello', voice: 'Kore' }), + }) + + expect(globalThis.fetch).toHaveBeenCalledWith( + 'https://example.com/v1beta/models/gemini-2.5-flash-preview-tts:generateContent', + expect.objectContaining({ + method: 'POST', + headers: expect.objectContaining({ + 'x-goog-api-key': 'test-key', + 'Content-Type': 'application/json', + }), + }), + ) + }) + + it('fetch adapter sends correct Gemini request body', async () => { + const mockResponse = new Response(JSON.stringify({ + candidates: [{ content: { parts: [{ inlineData: { data: 'AAAA' } }] } }], + }), { status: 200, headers: { 'Content-Type': 'application/json' } }) + + globalThis.fetch = vi.fn().mockResolvedValue(mockResponse) + + const provider = await getSpeechProvider({ apiKey: 'test-key', baseUrl: 'https://example.com/v1beta' }) + const speechResult = provider.speech('gemini-2.5-flash-preview-tts') + const fetchFn = speechResult.fetch! + + await fetchFn(new URL('http://test'), { + body: JSON.stringify({ model: 'gemini-2.5-flash-preview-tts', input: 'Hello from AIRI', voice: 'Kore' }), + }) + + const callArg = (globalThis.fetch as any).mock.calls[0][1] + const requestBody = JSON.parse(callArg.body) + + expect(requestBody.generationConfig.responseModalities).toEqual(['AUDIO']) + expect(requestBody.generationConfig.speechConfig.voiceConfig.prebuiltVoiceConfig.voiceName).toBe('Kore') + expect(requestBody.contents[0].parts[0].text).toBe('Hello from AIRI') + }) + + it('includes temperature in generationConfig when provided', async () => { + const mockResponse = new Response(JSON.stringify({ + candidates: [{ content: { parts: [{ inlineData: { data: 'AAAA' } }] } }], + }), { status: 200, headers: { 'Content-Type': 'application/json' } }) + + globalThis.fetch = vi.fn().mockResolvedValue(mockResponse) + + const provider = await getSpeechProvider({ apiKey: 'test-key' }) + const speechResult = provider.speech('gemini-2.5-flash-preview-tts') + const fetchFn = speechResult.fetch! + + await fetchFn(new URL('http://test'), { + body: JSON.stringify({ model: 'gemini-2.5-flash-preview-tts', input: 'Test', voice: 'Kore', temperature: 0.7 }), + }) + + const callArg = (globalThis.fetch as any).mock.calls[0][1] + const requestBody = JSON.parse(callArg.body) + + expect(requestBody.generationConfig.temperature).toBe(0.7) + }) + + it('omits temperature from generationConfig when not provided', async () => { + const mockResponse = new Response(JSON.stringify({ + candidates: [{ content: { parts: [{ inlineData: { data: 'AAAA' } }] } }], + }), { status: 200, headers: { 'Content-Type': 'application/json' } }) + + globalThis.fetch = vi.fn().mockResolvedValue(mockResponse) + + const provider = await getSpeechProvider({ apiKey: 'test-key' }) + const speechResult = provider.speech('gemini-2.5-flash-preview-tts') + const fetchFn = speechResult.fetch! + + await fetchFn(new URL('http://test'), { + body: JSON.stringify({ model: 'gemini-2.5-flash-preview-tts', input: 'Test', voice: 'Kore' }), + }) + + const callArg = (globalThis.fetch as any).mock.calls[0][1] + const requestBody = JSON.parse(callArg.body) + + expect(requestBody.generationConfig.temperature).toBeUndefined() + }) +}) + +describe('googleGeminiSpeech audio conversion', () => { + it('returns WAV audio from Gemini response', async () => { + const pcmBase64 = btoa('\x00\x00\x00\x00') + const mockResponse = new Response(JSON.stringify({ + candidates: [{ content: { parts: [{ inlineData: { data: pcmBase64 } }] } }], + }), { status: 200, headers: { 'Content-Type': 'application/json' } }) + + globalThis.fetch = vi.fn().mockResolvedValue(mockResponse) + + const provider = await getSpeechProvider({ apiKey: 'test-key' }) + const speechResult = provider.speech('gemini-2.5-flash-preview-tts') + const fetchFn = speechResult.fetch! + + const response = await fetchFn(new URL('http://test'), { + body: JSON.stringify({ model: 'gemini-2.5-flash-preview-tts', input: 'Test', voice: 'Kore' }), + }) + + expect(response.headers.get('Content-Type')).toBe('audio/wav') + + const buffer = await response.arrayBuffer() + const bytes = new Uint8Array(buffer) + + // RIFF header + expect(String.fromCharCode(bytes[0])).toBe('R') + expect(String.fromCharCode(bytes[1])).toBe('I') + expect(String.fromCharCode(bytes[2])).toBe('F') + expect(String.fromCharCode(bytes[3])).toBe('F') + + // WAVE format + expect(String.fromCharCode(bytes[8])).toBe('W') + expect(String.fromCharCode(bytes[9])).toBe('A') + expect(String.fromCharCode(bytes[10])).toBe('V') + expect(String.fromCharCode(bytes[11])).toBe('E') + }) + + it('throws when Gemini response has no audio data', async () => { + const mockResponse = new Response(JSON.stringify({ + candidates: [{ content: { parts: [{ text: 'no audio here' }] } }], + }), { status: 200, headers: { 'Content-Type': 'application/json' } }) + + globalThis.fetch = vi.fn().mockResolvedValue(mockResponse) + + const provider = await getSpeechProvider({ apiKey: 'test-key' }) + const speechResult = provider.speech('gemini-2.5-flash-preview-tts') + const fetchFn = speechResult.fetch! + + await expect(fetchFn(new URL('http://test'), { + body: JSON.stringify({ model: 'gemini-2.5-flash-preview-tts', input: 'Test', voice: 'Kore' }), + })).rejects.toThrow('Gemini TTS response missing audio data') + }) + + it('throws when Gemini API returns non-OK status', async () => { + const mockResponse = new Response('Unauthorized', { status: 401 }) + globalThis.fetch = vi.fn().mockResolvedValue(mockResponse) + + const provider = await getSpeechProvider({ apiKey: 'bad-key' }) + const speechResult = provider.speech('gemini-2.5-flash-preview-tts') + const fetchFn = speechResult.fetch! + + await expect(fetchFn(new URL('http://test'), { + body: JSON.stringify({ model: 'gemini-2.5-flash-preview-tts', input: 'Test', voice: 'Kore' }), + })).rejects.toThrow('Gemini TTS request failed: 401') + }) +}) diff --git a/packages/stage-ui/src/stores/providers/google-gemini-speech.ts b/packages/stage-ui/src/stores/providers/google-gemini-speech.ts new file mode 100644 index 000000000..510c4d566 --- /dev/null +++ b/packages/stage-ui/src/stores/providers/google-gemini-speech.ts @@ -0,0 +1,265 @@ +import type { SpeechProviderWithExtraOptions } from '@xsai-ext/providers/utils' + +import type { ModelInfo, ProviderMetadata, VoiceInfo } from '../providers' + +const PROVIDER_ID = 'google-gemini-audio-speech' +const DEFAULT_BASE_URL = 'https://generativelanguage.googleapis.com/v1beta' +const DEFAULT_MODEL = 'gemini-2.5-flash-preview-tts' + +const GOOGLE_GEMINI_TTS_MODELS = [ + 'gemini-2.5-flash-preview-tts', + 'gemini-2.5-pro-preview-tts', + 'gemini-3.1-flash-tts-preview', +] as const + +const GOOGLE_GEMINI_TTS_VOICES: [string, string][] = [ + ['Zephyr', 'Bright'], + ['Puck', 'Upbeat'], + ['Charon', 'Informative'], + ['Kore', 'Firm'], + ['Fenrir', 'Excitable'], + ['Leda', 'Youthful'], + ['Orus', 'Firm'], + ['Aoede', 'Breezy'], + ['Callirrhoe', 'Easy-going'], + ['Autonoe', 'Bright'], + ['Enceladus', 'Breathy'], + ['Iapetus', 'Clear'], + ['Umbriel', 'Easy-going'], + ['Algieba', 'Smooth'], + ['Despina', 'Smooth'], + ['Erinome', 'Clear'], + ['Algenib', 'Gravelly'], + ['Rasalgethi', 'Informative'], + ['Laomedeia', 'Upbeat'], + ['Achernar', 'Soft'], + ['Alnilam', 'Firm'], + ['Schedar', 'Even'], + ['Gacrux', 'Mature'], + ['Pulcherrima', 'Forward'], + ['Achird', 'Friendly'], + ['Zubenelgenubi', 'Casual'], + ['Vindemiatrix', 'Gentle'], + ['Sadachbia', 'Lively'], + ['Sadaltager', 'Knowledgeable'], + ['Sulafat', 'Warm'], +] + +/** Wraps raw PCM16 mono data in a minimal WAV container. */ +function wrapPCM16InWAV(pcmBytes: Uint8Array, sampleRate = 24000): Uint8Array { + const numChannels = 1 + const bitsPerSample = 16 + const byteRate = sampleRate * numChannels * (bitsPerSample / 8) + const blockAlign = numChannels * (bitsPerSample / 8) + const header = new ArrayBuffer(44) + const view = new DataView(header) + + const writeStr = (offset: number, str: string) => { + for (let i = 0; i < str.length; i++) + view.setUint8(offset + i, str.charCodeAt(i)) + } + + writeStr(0, 'RIFF') + view.setUint32(4, 36 + pcmBytes.length, true) + writeStr(8, 'WAVE') + + writeStr(12, 'fmt ') + view.setUint32(16, 16, true) + view.setUint16(20, 1, true) + view.setUint16(22, numChannels, true) + view.setUint32(24, sampleRate, true) + view.setUint32(28, byteRate, true) + view.setUint16(32, blockAlign, true) + view.setUint16(34, bitsPerSample, true) + + writeStr(36, 'data') + view.setUint32(40, pcmBytes.length, true) + + const wav = new Uint8Array(44 + pcmBytes.length) + wav.set(new Uint8Array(header), 0) + wav.set(pcmBytes, 44) + return wav +} + +/** Decodes a base64 string into a Uint8Array. */ +function base64ToBytes(base64: string): Uint8Array { + const binaryString = atob(base64) + const bytes = new Uint8Array(binaryString.length) + for (let i = 0; i < binaryString.length; i++) + bytes[i] = binaryString.charCodeAt(i) + return bytes +} + +function normalizeBaseUrl(value: unknown): string { + let base = typeof value === 'string' ? value.trim() : '' + if (!base) + base = DEFAULT_BASE_URL + if (!base.endsWith('/')) + base += '/' + return base +} + +function normalizeApiKey(value: unknown): string { + return typeof value === 'string' ? value.trim() : '' +} + +/** + * Custom fetch adapter that translates an OpenAI-compatible TTS request + * into a Gemini generateContent call with AUDIO response modality. + */ +function createAudioFetch(apiKey: string, baseUrl: string) { + return async (_input: RequestInfo | URL, init?: RequestInit): Promise => { + if (!init?.body || typeof init.body !== 'string') + throw new Error('Invalid request body') + + const body = JSON.parse(init.body) + const model = body.model as string + const input = body.input as string + const temperature = typeof body.temperature === 'number' ? body.temperature : undefined + + if (!input) + throw new Error('Missing input text for Gemini TTS') + if (!model) + throw new Error('Missing model for Gemini TTS') + + function buildGenerationConfig(): Record { + const voiceConfig: Record = { + prebuiltVoiceConfig: { + voiceName: (body.voice as string) || 'Kore', + }, + } + + return { + responseModalities: ['AUDIO'], + speechConfig: { voiceConfig }, + ...(temperature !== undefined ? { temperature } : {}), + } + } + + const response = await globalThis.fetch(`${baseUrl}models/${model}:generateContent`, { + method: 'POST', + headers: { + 'x-goog-api-key': apiKey, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + contents: [ + { + parts: [ + { text: input }, + ], + }, + ], + generationConfig: buildGenerationConfig(), + }), + }) + + if (!response.ok) { + const errorText = await response.text().catch(() => '') + throw new Error(`Gemini TTS request failed: ${response.status} ${errorText}`) + } + + const json = await response.json() + const audioBase64 = json.candidates?.[0]?.content?.parts?.find( + (part: { inlineData?: { data?: string } }) => part.inlineData, + )?.inlineData?.data + + if (!audioBase64) { + throw new Error('Gemini TTS response missing audio data') + } + + const pcmBytes = base64ToBytes(audioBase64) + const wavBytes = wrapPCM16InWAV(pcmBytes) + + // NOTICE: wrapPCM16InWAV always creates a fresh Uint8Array, so .buffer is the full + // backing ArrayBuffer (not a subarray view into a larger buffer). The `as ArrayBuffer` + // cast is needed because .buffer returns ArrayBufferLike in newer TypeScript. + return new Response(wavBytes.buffer as ArrayBuffer, { + status: 200, + headers: { 'Content-Type': 'audio/wav' }, + }) + } +} + +function createSpeechProvider(apiKey: string, baseUrl: string): SpeechProviderWithExtraOptions> { + return { + speech: (model?: string, options?: Record) => ({ + baseURL: `${baseUrl}`, + fetch: createAudioFetch(apiKey, baseUrl), + ...options, + model: model || (options?.model as string | undefined) || DEFAULT_MODEL, + }), + } +} + +function listModels(): ModelInfo[] { + return GOOGLE_GEMINI_TTS_MODELS.map(id => ({ + id, + name: id + .split('-') + .map(w => w.charAt(0).toUpperCase() + w.slice(1)) + .join(' '), + provider: PROVIDER_ID, + description: 'Gemini API text-to-speech model', + capabilities: ['text-to-speech'], + } satisfies ModelInfo)) +} + +function listVoices(): VoiceInfo[] { + return GOOGLE_GEMINI_TTS_VOICES.map(([voiceName, style]) => ({ + id: voiceName, + name: voiceName, + provider: PROVIDER_ID, + description: style, + languages: [{ code: 'auto', title: 'Auto' }], + compatibleModels: [...GOOGLE_GEMINI_TTS_MODELS], + } satisfies VoiceInfo)) +} + +export function buildGoogleGeminiSpeechProvider( + baseUrlValidator: (baseUrl: unknown) => { errors: unknown[], reason: string, valid: boolean } | null | undefined, +): ProviderMetadata { + return { + id: PROVIDER_ID, + category: 'speech', + tasks: ['text-to-speech', 'tts'], + nameKey: 'settings.pages.providers.provider.google-gemini-audio-speech.title', + name: 'Google Gemini', + descriptionKey: 'settings.pages.providers.provider.google-gemini-audio-speech.description', + description: 'aistudio.google.com', + icon: 'i-lobe-icons:gemini', + iconColor: 'i-lobe-icons:gemini-color', + defaultOptions: () => ({ + baseUrl: `${DEFAULT_BASE_URL}/`, + }), + createProvider: async (config: Record) => { + const apiKey = normalizeApiKey(config.apiKey) + const baseUrl = normalizeBaseUrl(config.baseUrl) + return createSpeechProvider(apiKey, baseUrl) + }, + capabilities: { + listModels: async () => listModels(), + listVoices: async () => listVoices(), + }, + validators: { + chatPingCheckAvailable: false, + validateProviderConfig: (config: Record) => { + const errors: Error[] = [] + if (!normalizeApiKey(config.apiKey)) + errors.push(new Error('API Key is required.')) + + if (config.baseUrl) { + const res = baseUrlValidator(config.baseUrl) + if (res) + return res + } + + return { + errors, + reason: errors.map(e => e.message).join(', '), + valid: errors.length === 0, + } + }, + }, + } +}