feat(stage-web|stage-tamagotchi): microsoft speech & azure speech

This commit is contained in:
Neko Ayaka
2025-03-18 11:07:26 +08:00
parent 6c2e36e15c
commit 1c83a8201b
24 changed files with 1524 additions and 415 deletions
+7 -7
View File
@@ -61,7 +61,7 @@
"culori": "^4.0.1",
"defu": "^6.1.4",
"drizzle-kit": "^0.30.5",
"drizzle-orm": "^0.40.0",
"drizzle-orm": "^0.40.1",
"jszip": "^3.10.1",
"localforage": "^1.10.0",
"nprogress": "^0.2.0",
@@ -79,7 +79,7 @@
"three": "^0.174.0",
"unified": "^11.0.5",
"valibot": "1.0.0-beta.9",
"vaul-vue": "^0.4.0",
"vaul-vue": "^0.4.1",
"vue": "^3.5.13",
"vue-demi": "^0.14.10",
"vue-i18n": "^11.1.2",
@@ -92,17 +92,17 @@
"@electron-toolkit/tsconfig": "^1.0.1",
"@iconify-json/carbon": "^1.2.8",
"@iconify-json/eos-icons": "^1.2.2",
"@iconify-json/lucide": "^1.2.29",
"@iconify-json/lucide": "^1.2.30",
"@iconify-json/mingcute": "^1.2.3",
"@iconify-json/simple-icons": "^1.2.28",
"@iconify-json/solar": "^1.2.2",
"@iconify-json/svg-spinners": "^1.2.2",
"@iconify-json/vscode-icons": "^1.2.16",
"@iconify/utils": "^2.3.0",
"@intlify/unplugin-vue-i18n": "^6.0.3",
"@intlify/unplugin-vue-i18n": "^6.0.4",
"@proj-airi/drizzle-duckdb-wasm": "workspace:^",
"@proj-airi/elevenlabs": "workspace:^",
"@proj-airi/lobe-icons": "^0.3.6",
"@proj-airi/lobe-icons": "^1.0.3",
"@proj-airi/provider-transformers": "workspace:^",
"@proj-airi/ui-transitions": "workspace:^",
"@proj-airi/unplugin-fetch": "^0.1.4",
@@ -114,7 +114,7 @@
"@types/three": "^0.174.0",
"@types/yauzl": "^2.10.3",
"@unocss/reset": "^66.1.0-beta.5",
"@vitejs/plugin-vue": "^5.2.1",
"@vitejs/plugin-vue": "^5.2.3",
"@vue-macros/volar": "^3.0.0-beta.5",
"@vueuse/motion": "^3.0.3",
"electron": "^34.3.3",
@@ -132,6 +132,6 @@
"vite-plugin-pwa": "^0.21.1",
"vite-plugin-vue-devtools": "^7.7.2",
"vite-plugin-vue-layouts": "^0.11.0",
"vue-tsc": "^2.2.8"
"vue-tsc": "^3.0.0-alpha.2"
}
}
@@ -95,7 +95,7 @@ function handleLanguageChange(event: Event) {
<div
v-motion
grid="~ cols-[150px_1fr]"
bg="neutral-100 dark:neutral-800"
bg="neutral-50 dark:neutral-800"
hover="bg-neutral-200 dark:bg-neutral-700"
transition="all ease-in-out duration-250" items-center gap-1.5 rounded-lg px-4
py-3
@@ -129,7 +129,7 @@ function handleLanguageChange(event: Event) {
<!-- Theme Setting -->
<label
v-motion
bg="neutral-100 dark:neutral-800"
bg="neutral-50 dark:neutral-800"
hover="bg-neutral-200 dark:bg-neutral-700"
transition="all ease-in-out duration-250"
w-full flex cursor-pointer rounded-lg px-4 py-3
@@ -169,7 +169,7 @@ function handleLanguageChange(event: Event) {
<!-- Developer Settings -->
<label
v-motion
bg="neutral-100 dark:neutral-800"
bg="neutral-50 dark:neutral-800"
hover="bg-neutral-200 dark:bg-neutral-700"
transition="all ease-in-out duration-250"
w-full flex cursor-pointer rounded-lg px-4 py-3
@@ -208,7 +208,7 @@ function handleLanguageChange(event: Event) {
</label>
</div>
<div text="neutral-200/50 dark:neutral-500/20" pointer-events-none fixed bottom-0 right-0 z--1 translate-x-10 translate-y-10>
<div v-motion text="40" i-lucide:cog />
<div v-motion text="60" i-lucide:cog />
</div>
</div>
</template>
@@ -8,7 +8,7 @@ import { RouterLink, useRouter } from 'vue-router'
const providersStore = useProvidersStore()
const consciousnessStore = useConsciousnessStore()
const { availableProviders, availableProvidersMetadata } = storeToRefs(providersStore)
const { availableProviders, availableTextGenerationsProvidersMetadata } = storeToRefs(providersStore)
const {
activeProvider,
activeModel,
@@ -78,7 +78,7 @@ function updateCustomModelName(value: string) {
role="radiogroup"
>
<RadioCardSimple
v-for="metadata in availableProvidersMetadata"
v-for="metadata in availableTextGenerationsProvidersMetadata"
:id="metadata.id"
:key="metadata.id"
v-model="activeProvider"
@@ -92,6 +92,14 @@ const modulesList = computed<Module[]>(() => [
to: '',
configured: false,
},
{
id: 'x',
name: t('settings.pages.modules.x.title'),
description: t('settings.pages.modules.x.description'),
icon: 'i-simple-icons:x',
to: '',
configured: false,
},
])
</script>
@@ -1,6 +1,7 @@
<script setup lang="ts">
import {
FieldCheckbox,
FieldInput,
FieldRange,
RadioCardSimple,
VoiceCardManySelect,
@@ -16,7 +17,7 @@ const providersStore = useProvidersStore()
const speechStore = useSpeechStore()
const {
availableProviders,
availableProvidersMetadata,
availableAudioSpeechProvidersMetadata,
} = storeToRefs(providersStore)
const {
activeSpeechProvider,
@@ -49,11 +50,6 @@ watch(activeSpeechProvider, async () => {
await speechStore.loadVoicesForProvider(activeSpeechProvider.value)
})
function updateVoiceName(value: string) {
voiceId.value = value
updateSSMLExample()
}
function updateCustomVoiceName(value: string) {
voiceId.value = value
updateSSMLExample()
@@ -74,6 +70,8 @@ function handleVoiceSelection(value: string) {
voiceId.value = value
updateSSMLExample()
}
watch(voiceId, updateSSMLExample)
</script>
<template>
@@ -110,7 +108,7 @@ function handleVoiceSelection(value: string) {
min-w-0 of-x-scroll scroll-smooth role="radiogroup"
>
<RadioCardSimple
v-for="metadata in availableProvidersMetadata"
v-for="metadata in availableAudioSpeechProvidersMetadata"
:id="metadata.id"
:key="metadata.id"
v-model="activeSpeechProvider"
@@ -262,20 +260,13 @@ function handleVoiceSelection(value: string) {
v-if="!availableVoices[activeSpeechProvider] || availableVoices[activeSpeechProvider].length === 0"
class="mt-2 space-y-6"
>
<div>
<label class="mb-1 block text-sm font-medium">
Voice Name
</label>
<input
v-model="voiceId" type="text"
class="w-full border border-neutral-300 rounded bg-white px-3 py-2 dark:border-neutral-700 dark:bg-neutral-900"
placeholder="Enter voice name (e.g., 'Rachel', 'Josh')"
@input="(event) => updateVoiceName((event.target as HTMLInputElement).value)"
>
<p class="mt-1 text-xs text-neutral-500">
For ElevenLabs, enter the exact voice name from your account
</p>
</div>
<FieldInput
v-model="voiceId"
type="text"
label="Voice ID"
description="Enter the voice ID for your custom voice"
placeholder="Enter voice name (e.g., 'Rachel', 'Josh')"
/>
<!-- Model selection for ElevenLabs -->
<div v-if="activeSpeechProvider === 'elevenlabs'">
@@ -0,0 +1,413 @@
<script setup lang="ts">
import type { Voice } from '@proj-airi/stage-ui/constants'
import type { RemovableRef } from '@vueuse/core'
import type { UnMicrosoftOptions } from '@xsai-ext/providers-local'
import type { SpeechProviderWithExtraOptions } from '@xsai-ext/shared-providers'
import {
FieldCheckbox,
FieldInput,
FieldRange,
ProviderAdvancedSettings,
ProviderApiKeyInput,
ProviderBaseUrlInput,
ProviderBasicSettings,
ProviderSettingsContainer,
ProviderSettingsLayout,
TestDummyMarker,
} from '@proj-airi/stage-ui/components'
import { voiceMap } from '@proj-airi/stage-ui/constants'
import { useProvidersStore, useSpeechStore } from '@proj-airi/stage-ui/stores'
import { generateSpeech } from '@xsai/generate-speech'
import { storeToRefs } from 'pinia'
import { computed, onMounted, onUnmounted, ref, watch } from 'vue'
import { useI18n } from 'vue-i18n'
import { useRouter } from 'vue-router'
const { t } = useI18n()
const router = useRouter()
const providersStore = useProvidersStore()
const speechStore = useSpeechStore()
const { providers } = storeToRefs(providersStore) as { providers: RemovableRef<Record<string, any>> }
// For playground
const testText = ref('Hello! This is a test of the Microsoft Speech voice synthesis.')
const isGenerating = ref(false)
const audioUrl = ref('')
const errorMessage = ref('')
const audioPlayer = ref<HTMLAudioElement | null>(null)
// Get provider metadata
const providerId = 'microsoft-speech'
const providerMetadata = computed(() => providersStore.getProviderMetadata(providerId))
const apiKey = computed({
get: () => providers.value[providerId]?.apiKey || '',
set: (value) => {
if (!providers.value[providerId])
providers.value[providerId] = {}
providers.value[providerId].apiKey = value
},
})
const baseUrl = computed({
get: () => providers.value[providerId]?.baseUrl || providerMetadata.value?.defaultOptions?.baseUrl || '',
set: (value) => {
if (!providers.value[providerId])
providers.value[providerId] = {}
providers.value[providerId].baseUrl = value
},
})
const region = computed({
get: () => providers.value[providerId]?.region || providerMetadata.value?.defaultOptions?.region || '',
set: (value) => {
if (!providers.value[providerId])
providers.value[providerId] = {}
providers.value[providerId].region = value
},
})
// Voice settings as individual computed properties
const similarityBoost = computed({
get: () => (providers.value[providerId]?.voiceSettings as any)?.similarityBoost ?? 0.75,
set: (value) => {
if (!providers.value[providerId])
providers.value[providerId] = {}
if (!providers.value[providerId].voiceSettings)
providers.value[providerId].voiceSettings = {}
providers.value[providerId].voiceSettings.similarityBoost = value
},
})
const stability = computed({
get: () => (providers.value[providerId]?.voiceSettings as any)?.stability ?? 0.5,
set: (value) => {
if (!providers.value[providerId])
providers.value[providerId] = {}
if (!providers.value[providerId].voiceSettings)
providers.value[providerId].voiceSettings = {}
providers.value[providerId].voiceSettings.stability = value
},
})
const speed = computed({
get: () => (providers.value[providerId]?.voiceSettings as any)?.speed ?? 1.0,
set: (value) => {
if (!providers.value[providerId])
providers.value[providerId] = {}
if (!providers.value[providerId].voiceSettings)
providers.value[providerId].voiceSettings = {}
providers.value[providerId].voiceSettings.speed = value
},
})
const style = computed({
get: () => (providers.value[providerId]?.voiceSettings as any)?.style ?? 0,
set: (value) => {
if (!providers.value[providerId])
providers.value[providerId] = {}
if (!providers.value[providerId].voiceSettings)
providers.value[providerId].voiceSettings = {}
providers.value[providerId].style = value
},
})
const useSpeakerBoost = computed({
get: () => (providers.value[providerId]?.voiceSettings as any)?.useSpeakerBoost !== false,
set: (value) => {
if (!providers.value[providerId])
providers.value[providerId] = {}
if (!providers.value[providerId].voiceSettings)
providers.value[providerId].voiceSettings = {}
providers.value[providerId].voiceSettings.useSpeakerBoost = value
},
})
// Speech settings
const selectedLanguage = ref(speechStore.selectedLanguage)
const selectedVoice = ref(speechStore.voiceName)
const availableVoices = computed(() => speechStore.availableVoicesForLanguage)
onMounted(() => {
providersStore.initializeProvider(providerId)
// Initialize refs with current values
apiKey.value = providers.value[providerId]?.apiKey || ''
baseUrl.value = providers.value[providerId]?.baseUrl || providerMetadata.value?.defaultOptions?.baseUrl || ''
// Initialize voice settings refs
if (providers.value[providerId]?.voiceSettings) {
similarityBoost.value = (providers.value[providerId].voiceSettings as any)?.similarityBoost ?? 0.75
stability.value = (providers.value[providerId].voiceSettings as any)?.stability ?? 0.5
speed.value = (providers.value[providerId].voiceSettings as any)?.speed ?? 1.0
style.value = (providers.value[providerId].voiceSettings as any)?.style ?? 0
useSpeakerBoost.value = (providers.value[providerId].voiceSettings as any)?.useSpeakerBoost !== false
}
// Load voices if provider is configured
if (providersStore.configuredProviders[providerId]) {
speechStore.loadVoicesForProvider(providerId)
}
})
// Watch all settings and update the provider configuration
watch([apiKey, baseUrl, similarityBoost, stability, speed, style, useSpeakerBoost], () => {
providers.value[providerId] = {
...providers.value[providerId],
apiKey: apiKey.value,
baseUrl: baseUrl.value || providerMetadata.value?.defaultOptions?.baseUrl || '',
}
})
// Function to generate speech
async function generateTestSpeech() {
if (!testText.value.trim())
return
const provider = providersStore.getProviderInstance(providerId) as SpeechProviderWithExtraOptions<string, UnMicrosoftOptions>
if (!provider) {
console.error('Failed to initialize speech provider')
return
}
isGenerating.value = true
errorMessage.value = ''
try {
// Stop any currently playing audio
if (audioUrl.value) {
stopTestAudio()
}
const response = await generateSpeech({
...provider.speech('v1'),
input: testText.value,
voice: voiceMap[selectedVoice.value as Voice],
})
// Convert the response to a blob and create an object URL
audioUrl.value = URL.createObjectURL(new Blob([response]))
// Play the audio
setTimeout(() => {
if (audioPlayer.value) {
audioPlayer.value.play()
}
}, 100)
}
catch (error) {
console.error('Error generating speech:', error)
errorMessage.value = error instanceof Error ? error.message : 'An unknown error occurred'
}
finally {
isGenerating.value = false
}
}
// Function to stop audio playback
function stopTestAudio() {
if (audioPlayer.value) {
audioPlayer.value.pause()
audioPlayer.value.currentTime = 0
}
// Clean up the object URL to prevent memory leaks
if (audioUrl.value) {
URL.revokeObjectURL(audioUrl.value)
audioUrl.value = ''
}
}
// Clean up when component is unmounted
onUnmounted(() => {
if (audioUrl.value) {
URL.revokeObjectURL(audioUrl.value)
}
})
function handleResetVoiceSettings() {
providers.value[providerId] = {
...(providerMetadata.value?.defaultOptions as any),
}
}
</script>
<template>
<ProviderSettingsLayout
:provider-name="providerMetadata?.localizedName" :provider-icon="providerMetadata?.icon"
:on-back="() => router.back()"
>
<div flex="~ col md:row gap-6">
<ProviderSettingsContainer class="w-full md:w-[40%]">
<ProviderBasicSettings
:title="t('settings.pages.providers.common.section.basic.title')"
:description="t('settings.pages.providers.common.section.basic.description')"
:on-reset="handleResetVoiceSettings"
>
<ProviderApiKeyInput v-model="apiKey" :provider-name="providerMetadata?.localizedName" placeholder="sk-" />
<FieldInput
v-model="region"
label="Region"
description="Speech Service region"
placeholder="eastasia"
required
type="text"
/>
</ProviderBasicSettings>
<div flex="~ col gap-6">
<h2 class="text-lg text-neutral-500 md:text-2xl dark:text-neutral-400">
{{ t('settings.pages.providers.common.section.voice.title') }}
</h2>
<div flex="~ col gap-4">
<FieldRange
v-model="similarityBoost"
:label="t('settings.pages.providers.provider.elevenlabs.fields.field.simularity-boost.label')"
:description="t('settings.pages.providers.provider.elevenlabs.fields.field.simularity-boost.description')"
:min="0" :max="1" :step="0.01"
/>
<FieldRange
v-model="stability"
:label="t('settings.pages.providers.provider.elevenlabs.fields.field.stability.label')"
:description="t('settings.pages.providers.provider.elevenlabs.fields.field.stability.description')"
:min="0" :max="1" :step="0.01"
/>
<FieldRange
v-model="speed"
:label="t('settings.pages.providers.provider.elevenlabs.fields.field.speed.label')"
:description="t('settings.pages.providers.provider.elevenlabs.fields.field.speed.description')" :min="0.7"
:max="1.2" :step="0.01"
/>
<FieldRange
v-model="style"
:label="t('settings.pages.providers.provider.elevenlabs.fields.field.style.label')"
:description="t('settings.pages.providers.provider.elevenlabs.fields.field.style.description')" :min="0"
:max="1" :step="0.01"
/>
<FieldCheckbox
v-model="useSpeakerBoost"
:label="t('settings.pages.providers.provider.elevenlabs.fields.field.speaker-boost.label')"
:description="t('settings.pages.providers.provider.elevenlabs.fields.field.speaker-boost.description')"
/>
</div>
</div>
<ProviderAdvancedSettings :title="t('settings.pages.providers.common.section.advanced.title')">
<ProviderBaseUrlInput
v-model="baseUrl"
:placeholder="providerMetadata?.defaultOptions?.baseUrl as string || ''" required
/>
</ProviderAdvancedSettings>
</ProviderSettingsContainer>
<div flex="~ col gap-6" class="w-full md:w-[60%]">
<div w-full rounded-xl>
<h2 class="mb-4 text-lg text-neutral-500 md:text-2xl dark:text-neutral-400" w-full>
<div class="inline-flex items-center gap-4">
<TestDummyMarker />
<div>
{{ t('settings.pages.providers.provider.elevenlabs.playground.title') }}
</div>
</div>
</h2>
<div flex="~ col gap-4">
<textarea
v-model="testText"
:placeholder="t('settings.pages.providers.provider.elevenlabs.playground.fields.field.input.placeholder')"
border="neutral-100 dark:neutral-800 solid 2 focus:neutral-200 dark:focus:neutral-700"
transition="all duration-250 ease-in-out"
bg="neutral-100 dark:neutral-800 focus:neutral-50 dark:focus:neutral-900"
h-24 w-full rounded-lg px-3 py-2 text-sm outline-none
/>
<div flex="~ col gap-6">
<label grid="~ cols-2 gap-4">
<div>
<div class="flex items-center gap-1 text-sm font-medium">
{{ t('settings.pages.providers.provider.elevenlabs.playground.fields.field.language.label') }}
</div>
<div class="text-xs text-neutral-500 dark:text-neutral-400">
{{ t('settings.pages.providers.provider.elevenlabs.playground.fields.field.language.description') }}
</div>
</div>
<select
v-model="selectedLanguage"
border="neutral-300 dark:neutral-800 solid 2 focus:neutral-400 dark:focus:neutral-600"
transition="border duration-250 ease-in-out" w-full rounded-lg px-2 py-1 text-nowrap text-sm
outline-none
>
<option v-for="language in speechStore.availableLanguages" :key="language" :value="language">
{{ language }}
</option>
</select>
</label>
<label grid="~ cols-2 gap-4">
<div>
<div class="flex items-center gap-1 text-sm font-medium">
{{ t('settings.pages.providers.provider.elevenlabs.playground.fields.field.voice.label') }}
</div>
<div class="text-xs text-neutral-500 dark:text-neutral-400">
{{ t('settings.pages.providers.provider.elevenlabs.playground.fields.field.voice.description') }}
</div>
</div>
<select
v-model="selectedVoice"
border="neutral-300 dark:neutral-800 solid 2 focus:neutral-400 dark:focus:neutral-600"
transition="border duration-250 ease-in-out" w-full rounded-lg px-2 py-1 text-nowrap text-sm
outline-none
>
<option v-for="voice in availableVoices" :key="voice.id" :value="voice.name">
{{ voice.name }}
</option>
</select>
</label>
</div>
<div flex="~ row" gap-4>
<button
border="neutral-800 dark:neutral-200 solid 2" transition="border duration-250 ease-in-out"
rounded-lg px-4 text="neutral-100 dark:neutral-900" py-2 text-sm
:disabled="isGenerating || !testText.trim() || !apiKey"
:class="{ 'opacity-50 cursor-not-allowed': isGenerating || !testText.trim() || !apiKey }"
bg="neutral-700 dark:neutral-300" @click="generateTestSpeech"
>
<div flex="~ row" items-center gap-2>
<div i-solar:play-circle-bold-duotone />
<span>{{ isGenerating ? t('settings.pages.providers.provider.elevenlabs.playground.buttons.button.test-voice.generating') : t('settings.pages.providers.provider.elevenlabs.playground.buttons.button.test-voice.label') }}</span>
</div>
</button>
<button
v-if="audioUrl" border="primary-300 dark:primary-800 solid 2"
transition="border duration-250 ease-in-out" rounded-lg px-4 py-2 text-sm @click="stopTestAudio"
>
<div flex="~ row" items-center gap-2>
<div i-solar:stop-circle-bold-duotone />
<span>Stop</span>
</div>
</button>
</div>
<div v-if="!apiKey" class="mt-2 text-sm text-red-500">
{{ t('settings.pages.providers.provider.elevenlabs.playground.validation.error-missing-api-key') }}
</div>
<div v-if="errorMessage" class="mt-2 text-sm text-red-500">
{{ errorMessage }}
</div>
<audio v-if="audioUrl" ref="audioPlayer" :src="audioUrl" controls class="mt-2 w-full" />
</div>
</div>
</div>
</div>
</ProviderSettingsLayout>
</template>