fix(stage-*): OpenAI Compatible Speech and Hearing modules activation (#947)

---------

Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
Co-authored-by: Neko <neko@ayaka.moe>
This commit is contained in:
lockrush-dev
2026-01-17 00:16:42 +08:00
committed by GitHub
co-authored by autofix-ci[bot] Neko
parent 153eb95c8b
commit 815ce36bce
19 changed files with 1058 additions and 387 deletions
+17 -6
View File
@@ -44,17 +44,28 @@ runs:
shell: bash
run: |
if [[ "${{ inputs.version }}" == "latest" ]]; then
VERSION=$(curl -s https://api.github.com/repos/realm/SwiftLint/releases/latest | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/')
if [[ -z "$VERSION" ]]; then
echo "Error: Failed to get latest version"
# Fetch the latest release tag using a more robust method
API_RESPONSE=$(curl -s https://api.github.com/repos/realm/SwiftLint/releases/latest)
# Try using jq if available (most GitHub Actions runners have it)
if command -v jq >/dev/null 2>&1; then
VERSION=$(echo "$API_RESPONSE" | jq -r '.tag_name // empty')
else
# Fallback: parse JSON manually with sed, being very specific about the tag_name field
VERSION=$(echo "$API_RESPONSE" | sed -n 's/.*"tag_name"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' | head -1)
fi
# Validate the version looks correct (should contain version numbers)
if [[ -z "$VERSION" ]] || [[ "$VERSION" == "null" ]] || [[ ! "$VERSION" =~ ^[0-9] ]]; then
echo "Error: Failed to get latest version or got invalid version: '$VERSION'"
echo "API Response (first 50 lines):"
echo "$API_RESPONSE" | head -50
exit 1
fi
echo "Latest version: $VERSION"
else
VERSION="${{ inputs.version }}"
if [[ ! "$VERSION" =~ ^v ]]; then
VERSION="v$VERSION"
fi
# Remove 'v' prefix if user provided it, since SwiftLint tags don't use it
VERSION="${VERSION#v}"
echo "Using specified version: $VERSION"
fi
echo "version=$VERSION" >> $GITHUB_OUTPUT
@@ -8,10 +8,10 @@ import { useAudioContext } from '@proj-airi/stage-ui/stores/audio'
import { useHearingSpeechInputPipeline, useHearingStore } from '@proj-airi/stage-ui/stores/modules/hearing'
import { useProvidersStore } from '@proj-airi/stage-ui/stores/providers'
import { useSettingsAudioDevice } from '@proj-airi/stage-ui/stores/settings'
import { Button, FieldCheckbox, FieldRange, FieldSelect } from '@proj-airi/ui'
import { Button, FieldCheckbox, FieldInput, FieldRange, FieldSelect } from '@proj-airi/ui'
import { until } from '@vueuse/core'
import { storeToRefs } from 'pinia'
import { computed, onUnmounted, ref, watch } from 'vue'
import { computed, onMounted, onUnmounted, ref, watch } from 'vue'
import { useI18n } from 'vue-i18n'
const { t } = useI18n()
@@ -222,8 +222,29 @@ const speakingIndicatorClass = computed(() => {
}
})
function updateCustomModelName(value: string) {
activeCustomModelName.value = value
function updateCustomModelName(value: string | undefined) {
const modelValue = value || ''
activeCustomModelName.value = modelValue
activeTranscriptionModel.value = modelValue
}
// Sync OpenAI Compatible model from provider config
function syncOpenAICompatibleSettings() {
if (activeTranscriptionProvider.value !== 'openai-compatible-audio-transcription')
return
const providerConfig = providersStore.getProviderConfig(activeTranscriptionProvider.value)
// Always sync model from provider config (override any existing value from previous provider)
if (providerConfig?.model) {
activeTranscriptionModel.value = providerConfig.model as string
updateCustomModelName(providerConfig.model as string)
}
else {
// If no model in provider config, use default
const defaultModel = 'whisper-1'
activeTranscriptionModel.value = defaultModel
updateCustomModelName(defaultModel)
}
}
onStopRecord(async (recording) => {
@@ -427,6 +448,7 @@ watch(activeTranscriptionProvider, async (provider) => {
return
await hearingStore.loadModelsForProvider(provider)
syncOpenAICompatibleSettings()
// Auto-select first model for Web Speech API if no model is selected
if (provider === 'browser-web-speech-api' && !activeTranscriptionModel.value) {
@@ -438,6 +460,11 @@ watch(activeTranscriptionProvider, async (provider) => {
}
}, { immediate: true })
onMounted(async () => {
// Audio devices are loaded on demand when user requests them
syncOpenAICompatibleSettings()
})
onUnmounted(() => {
stopSTTTest()
stopAudioMonitoring()
@@ -537,19 +564,25 @@ onUnmounted(() => {
</div>
<!-- Model selection section -->
<div v-if="activeTranscriptionProvider && supportsModelListing">
<div v-if="activeTranscriptionProvider">
<div flex="~ col gap-4">
<div>
<h2 class="text-lg md:text-2xl">
{{ t('settings.pages.modules.consciousness.sections.section.provider-model-selection.title') }}
</h2>
<div text="neutral-400 dark:neutral-400">
<span>{{ t('settings.pages.modules.consciousness.sections.section.provider-model-selection.subtitle') }}</span>
<!-- Show different description based on whether provider supports model listing and has models -->
<span v-if="supportsModelListing && providerModels.length > 0">
{{ t('settings.pages.modules.consciousness.sections.section.provider-model-selection.subtitle') }}
</span>
<span v-else>
Enter the transcription model to use (e.g., 'whisper-1', 'gpt-4o-transcribe')
</span>
</div>
</div>
<!-- Loading state -->
<div v-if="isLoadingActiveProviderModels" class="flex items-center justify-center py-4">
<div v-if="isLoadingActiveProviderModels && supportsModelListing" class="flex items-center justify-center py-4">
<div class="mr-2 animate-spin">
<div i-solar:spinner-line-duotone text-xl />
</div>
@@ -558,14 +591,26 @@ onUnmounted(() => {
<!-- Error state -->
<ErrorContainer
v-else-if="activeProviderModelError"
v-else-if="activeProviderModelError && supportsModelListing"
:title="t('settings.pages.modules.consciousness.sections.section.provider-model-selection.error')"
:error="activeProviderModelError"
/>
<!-- No models available -->
<!-- Manual input for providers without model listing or when no models are available -->
<div
v-else-if="!supportsModelListing || (activeTranscriptionProvider === 'openai-compatible-audio-transcription' && providerModels.length === 0 && !isLoadingActiveProviderModels)"
class="mt-2"
>
<FieldInput
:model-value="activeTranscriptionModel || activeCustomModelName || ''"
placeholder="whisper-1"
@update:model-value="updateCustomModelName"
/>
</div>
<!-- No models available (for other providers with model listing but no models) -->
<Alert
v-else-if="providerModels.length === 0 && !isLoadingActiveProviderModels"
v-else-if="providerModels.length === 0 && !isLoadingActiveProviderModels && supportsModelListing"
type="warning"
>
<template #title>
@@ -576,8 +621,8 @@ onUnmounted(() => {
</template>
</Alert>
<!-- Using the new RadioCardManySelect component -->
<template v-else-if="providerModels.length > 0">
<!-- Using the new RadioCardManySelect component for providers with models -->
<template v-else-if="providerModels.length > 0 && supportsModelListing">
<RadioCardManySelect
v-model="activeTranscriptionModel"
v-model:search-query="transcriptionModelSearchQuery"
@@ -57,14 +57,43 @@ const audioUrl = ref('')
const audioPlayer = ref<HTMLAudioElement | null>(null)
const errorMessage = ref('')
// Sync OpenAI Compatible model and voice from provider config
function syncOpenAICompatibleSettings() {
if (activeSpeechProvider.value !== 'openai-compatible-audio-speech')
return
const providerConfig = providersStore.getProviderConfig(activeSpeechProvider.value)
// Sync model from provider config (override any existing value from previous provider)
if (providerConfig?.model) {
activeSpeechModel.value = providerConfig.model as string
}
else {
// If no model in provider config, use default
activeSpeechModel.value = 'tts-1'
}
// Sync voice from provider config (override any existing value from previous provider)
// Use updateCustomVoiceName to ensure proper reactivity
if (providerConfig?.voice) {
activeSpeechVoiceId.value = providerConfig.voice as string
updateCustomVoiceName(providerConfig.voice as string)
}
else {
// If no voice in provider config, use default
activeSpeechVoiceId.value = 'alloy'
updateCustomVoiceName('alloy')
}
}
onMounted(async () => {
await providersStore.loadModelsForConfiguredProviders()
await speechStore.loadVoicesForProvider(activeSpeechProvider.value)
syncOpenAICompatibleSettings()
})
watch(activeSpeechProvider, async () => {
watch(activeSpeechProvider, async (newProvider) => {
await providersStore.loadModelsForConfiguredProviders()
await speechStore.loadVoicesForProvider(activeSpeechProvider.value)
await speechStore.loadVoicesForProvider(newProvider)
syncOpenAICompatibleSettings()
})
// Function to generate speech
@@ -75,16 +104,6 @@ async function generateTestSpeech() {
if (useSSML.value && !ssmlText.value.trim())
return
if (!activeSpeechModel.value) {
console.error('No model selected')
return
}
if (!activeSpeechVoice.value) {
console.error('No voice selected')
return
}
const provider = await providersStore.getProviderInstance(activeSpeechProvider.value) as SpeechProviderWithExtraOptions<string, any>
if (!provider) {
console.error('Failed to initialize speech provider')
@@ -93,6 +112,37 @@ async function generateTestSpeech() {
const providerConfig = providersStore.getProviderConfig(activeSpeechProvider.value)
// For OpenAI Compatible providers, fall back to provider config for model and voice
let model = activeSpeechModel.value
let voice = activeSpeechVoice.value
if (activeSpeechProvider.value === 'openai-compatible-audio-speech') {
if (!model && providerConfig?.model) {
model = providerConfig.model as string
}
if (!voice && providerConfig?.voice) {
voice = {
id: providerConfig.voice as string,
name: providerConfig.voice as string,
description: providerConfig.voice as string,
previewURL: '',
languages: [{ code: 'en', title: 'English' }],
provider: activeSpeechProvider.value,
gender: 'neutral',
}
}
}
if (!model) {
console.error('No model selected')
return
}
if (!voice) {
console.error('No voice selected')
return
}
isGenerating.value = true
errorMessage.value = ''
@@ -104,12 +154,12 @@ async function generateTestSpeech() {
const input = useSSML.value
? ssmlText.value
: speechStore.supportsSSML ? speechStore.generateSSML(testText.value, activeSpeechVoice.value, { ...providerConfig, pitch: pitch.value }) : testText.value
: speechStore.supportsSSML ? speechStore.generateSSML(testText.value, voice, { ...providerConfig, pitch: pitch.value }) : testText.value
const response = await generateSpeech({
...provider.speech(activeSpeechModel.value, providerConfig),
...provider.speech(model, providerConfig),
input,
voice: activeSpeechVoice.value.id,
voice: voice.id,
})
// Convert the response to a blob and create an object URL
@@ -169,8 +219,8 @@ function updateCustomVoiceName(value: string | undefined) {
}
}
function updateCustomModelName(value: string) {
activeSpeechModel.value = value
function updateCustomModelName(value: string | undefined) {
activeSpeechModel.value = value || ''
}
</script>
@@ -237,7 +287,7 @@ function updateCustomModelName(value: string) {
</div>
<div>
<!-- Model selection section -->
<div v-if="activeSpeechProvider && supportsModelListing">
<div v-if="activeSpeechProvider">
<div flex="~ col gap-4">
<div>
<h2 class="text-lg md:text-2xl">
@@ -248,6 +298,19 @@ function updateCustomModelName(value: string) {
</div>
</div>
<!-- Manual input for OpenAI Compatible -->
<div v-if="activeSpeechProvider === 'openai-compatible-audio-speech'">
<FieldInput
:model-value="activeSpeechModel || ''"
label="Model"
description="Enter the TTS model to use for speech generation"
placeholder="tts-1"
@update:model-value="updateCustomModelName"
/>
</div>
<!-- Model listing for other providers -->
<div v-else-if="supportsModelListing">
<!-- Loading state -->
<div v-if="isLoadingActiveProviderModels" class="flex items-center justify-center py-4">
<div class="mr-2 animate-spin">
@@ -294,6 +357,7 @@ function updateCustomModelName(value: string) {
</div>
</div>
</div>
</div>
<!-- Voice Configuration Section -->
<div v-if="activeSpeechProvider">
@@ -331,15 +395,20 @@ function updateCustomModelName(value: string) {
</div>
<!-- Error state -->
<!-- Voice selection with RadioCardManySelect -->
<!-- Voice selection with RadioCardManySelect (skip for OpenAI Compatible) -->
<div
v-else-if="availableVoices[activeSpeechProvider] && availableVoices[activeSpeechProvider].length > 0"
v-else-if="activeSpeechProvider !== 'openai-compatible-audio-speech' && availableVoices[activeSpeechProvider] && availableVoices[activeSpeechProvider].length > 0"
class="space-y-6"
>
<VoiceCardManySelect
v-model:search-query="voiceSearchQuery"
v-model:voice-id="activeSpeechVoiceId"
:voices="availableVoices[activeSpeechProvider]?.filter(voice => {
// If no model is selected, show all voices
if (!activeSpeechModel) {
return true
}
// If a model is selected, filter by compatibility
return !voice.compatibleModels || voice.compatibleModels.includes(activeSpeechModel)
}).map(voice => ({
id: voice.id,
@@ -404,16 +473,17 @@ function updateCustomModelName(value: string) {
/>
</div>
<!-- Manual voice input when no voices are available -->
<!-- Manual voice input when no voices are available or for OpenAI Compatible -->
<div
v-if="!availableVoices[activeSpeechProvider] || availableVoices[activeSpeechProvider].length === 0"
v-if="activeSpeechProvider === 'openai-compatible-audio-speech' || !availableVoices[activeSpeechProvider] || availableVoices[activeSpeechProvider].length === 0"
class="mt-2 space-y-6"
>
<FieldInput
type="text"
:model-value="activeSpeechVoiceId || ''"
label="Voice Name"
description="Enter the voice name for your custom voice"
placeholder="Enter voice name (e.g., 'Rachel', 'Josh')"
placeholder="Enter voice name (e.g., 'alloy', 'echo')"
@update:model-value="updateCustomVoiceName"
/>
@@ -7,9 +7,9 @@ import {
} from '@proj-airi/stage-ui/components'
import { useSpeechStore } from '@proj-airi/stage-ui/stores/modules/speech'
import { useProvidersStore } from '@proj-airi/stage-ui/stores/providers'
import { FieldRange } from '@proj-airi/ui'
import { FieldRange, FieldSelect } from '@proj-airi/ui'
import { storeToRefs } from 'pinia'
import { computed, ref, watch } from 'vue'
import { computed, onMounted, ref, watch } from 'vue'
import { useI18n } from 'vue-i18n'
const speechStore = useSpeechStore()
@@ -27,14 +27,55 @@ const defaultModel = 'gpt-4o-mini-tts'
const speed = ref<number>(1.0)
// Model selection
const model = computed({
get: () => providers.value[providerId]?.model as string | undefined || defaultModel,
set: (value) => {
if (!providers.value[providerId])
providers.value[providerId] = {}
providers.value[providerId].model = value
},
})
// Load models and voices
const providerModels = computed(() => {
return providersStore.getModelsForProvider(providerId)
})
const isLoadingModels = computed(() => {
return providersStore.isLoadingModels[providerId] || false
})
// Check if API key is configured
const apiKeyConfigured = computed(() => !!providers.value[providerId]?.apiKey)
// Filter voices based on the selected model's compatibility
const availableVoices = computed(() => {
return speechStore.availableVoices[providerId] || []
const allVoices = speechStore.availableVoices[providerId] || []
const selectedModel = model.value || defaultModel
// Filter voices to only show those compatible with the selected model
return allVoices.filter((voice) => {
// If voice has no compatibleModels array, include it (backward compatibility)
if (!voice.compatibleModels || voice.compatibleModels.length === 0) {
return true
}
// Check if the selected model is in the voice's compatibleModels array
return voice.compatibleModels.includes(selectedModel)
})
})
// Generate speech with ElevenLabs-specific parameters
// Load models and voices on mount
onMounted(async () => {
await providersStore.loadModelsForConfiguredProviders()
await providersStore.fetchModelsForProvider(providerId)
// Load voices
// NOTE: OpenAI does not provide an API endpoint to retrieve available voices.
// Voices are hardcoded in provider metadata - this is a provider limitation, not an application limitation.
await speechStore.loadVoicesForProvider(providerId)
})
// Generate speech with OpenAI-specific parameters
async function handleGenerateSpeech(input: string, voiceId: string, _useSSML: boolean) {
const provider = await providersStore.getProviderInstance<SpeechProvider<string>>(providerId)
if (!provider) {
@@ -44,13 +85,12 @@ async function handleGenerateSpeech(input: string, voiceId: string, _useSSML: bo
// Get provider configuration
const providerConfig = providersStore.getProviderConfig(providerId)
// Get model from configuration or use default
const model = providerConfig.model as string | undefined || defaultModel
// Use the reactive model computed property (not a local variable)
const modelToUse = model.value || defaultModel
// ElevenLabs doesn't need SSML conversion, but if SSML is provided, use it directly
return await speechStore.speech(
provider,
model,
modelToUse,
input,
voiceId,
{
@@ -64,6 +104,14 @@ watch(speed, async () => {
const providerConfig = providersStore.getProviderConfig(providerId)
providerConfig.speed = speed.value
})
watch(model, async () => {
const providerConfig = providersStore.getProviderConfig(providerId)
providerConfig.model = model.value
// Reload voices when model changes to ensure compatibility filtering is applied
// Note: Voice compatibility varies by model - some voices (ballad, verse, marin, cedar) are only compatible with gpt-4o-mini-tts models
await speechStore.loadVoicesForProvider(providerId)
})
</script>
<template>
@@ -72,8 +120,17 @@ watch(speed, async () => {
:default-model="defaultModel"
:additional-settings="defaultVoiceSettings"
>
<!-- Voice settings specific to ElevenLabs -->
<!-- Voice settings specific to OpenAI -->
<template #voice-settings>
<!-- Model selection -->
<FieldSelect
v-model="model"
label="Model"
description="Select the TTS model to use for speech generation"
:options="providerModels.map(m => ({ value: m.id, label: m.name }))"
:disabled="isLoadingModels || providerModels.length === 0"
placeholder="Select a model..."
/>
<!-- Speed control - common to most providers -->
<FieldRange
v-model="speed"
@@ -1,27 +1,23 @@
<script setup lang="ts">
import type { RemovableRef } from '@vueuse/core'
import type { SpeechProvider } from '@xsai-ext/providers/utils'
import {
Alert,
ProviderAdvancedSettings,
ProviderApiKeyInput,
ProviderBaseUrlInput,
ProviderBasicSettings,
ProviderSettingsContainer,
ProviderSettingsLayout,
SpeechPlaygroundOpenAICompatible,
SpeechProviderSettings,
} from '@proj-airi/stage-ui/components'
import { useProviderValidation } from '@proj-airi/stage-ui/composables/use-provider-validation'
import { useSpeechStore } from '@proj-airi/stage-ui/stores/modules/speech'
import { useProvidersStore } from '@proj-airi/stage-ui/stores/providers'
import { FieldRange } from '@proj-airi/ui'
import { FieldInput, FieldRange } from '@proj-airi/ui'
import { storeToRefs } from 'pinia'
import { computed, ref } from 'vue'
import { computed, onMounted, ref, watch } from 'vue'
import { useI18n } from 'vue-i18n'
const speechStore = useSpeechStore()
const providersStore = useProvidersStore()
const { providers } = storeToRefs(providersStore) as { providers: RemovableRef<Record<string, any>> }
const { providers } = storeToRefs(providersStore)
const { t } = useI18n()
const defaultVoiceSettings = {
speed: 1.0,
@@ -29,28 +25,21 @@ const defaultVoiceSettings = {
// Get provider metadata
const providerId = 'openai-compatible-audio-speech'
const defaultModel = 'tts-1'
// Settings refs
const apiKey = computed({
get: () => providers.value[providerId]?.apiKey || '',
set: (value) => {
if (providers.value[providerId])
providers.value[providerId].apiKey = value
},
})
const baseUrl = computed({
get: () => providers.value[providerId]?.baseUrl || '',
set: (value) => {
if (providers.value[providerId])
providers.value[providerId].baseUrl = value
},
})
// Initialize speed from provider config or default
const speed = ref<number>(
(providers.value[providerId] as any)?.voiceSettings?.speed
|| (providers.value[providerId] as any)?.speed
|| defaultVoiceSettings.speed,
)
// Model selection
const model = computed({
get: () => providers.value[providerId]?.model || 'tts-1',
get: () => providers.value[providerId]?.model as string | undefined || defaultModel,
set: (value) => {
if (providers.value[providerId])
if (!providers.value[providerId])
providers.value[providerId] = {}
providers.value[providerId].model = value
},
})
@@ -58,29 +47,77 @@ const model = computed({
const voice = computed({
get: () => providers.value[providerId]?.voice || 'alloy',
set: (value) => {
if (providers.value[providerId])
if (!providers.value[providerId])
providers.value[providerId] = {}
providers.value[providerId].voice = value
},
})
const speed = ref<number>(1.0)
// TODO: use `useRefHistory` for this
// Watch provider config changes to sync local refs (for reset functionality)
watch(
() => providers.value[providerId],
(newConfig) => {
if (newConfig) {
// Sync speed from voiceSettings or direct speed property
const config = newConfig as any
const newSpeed = config.voiceSettings?.speed || config.speed || defaultVoiceSettings.speed
if (Math.abs(speed.value - newSpeed) > 0.001) // Use small epsilon for float comparison
speed.value = newSpeed
// Sync model if it was reset
if (!config.model && model.value !== defaultModel)
model.value = defaultModel
// Sync voice if it was reset
if (!config.voice && voice.value !== 'alloy')
voice.value = 'alloy'
}
else {
// Provider config was reset, reset our local refs to defaults
speed.value = defaultVoiceSettings.speed
model.value = defaultModel
voice.value = 'alloy'
}
},
{ deep: true, immediate: true },
)
// Check if API key is configured
const apiKeyConfigured = computed(() => !!providers.value[providerId]?.apiKey)
// Generate speech with specific parameters
// Ensure provider config is initialized on mount
onMounted(() => {
if (!providers.value[providerId]) {
providers.value[providerId] = {}
}
// Initialize model and voice if they don't exist
if (!providers.value[providerId].model) {
providers.value[providerId].model = defaultModel
}
if (!providers.value[providerId].voice) {
providers.value[providerId].voice = 'alloy'
}
})
// Generate speech with OpenAI-compatible parameters
async function handleGenerateSpeech(input: string, voiceId: string, _useSSML: boolean, modelId?: string) {
const provider = await providersStore.getProviderInstance<SpeechProvider<string>>(providerId)
if (!provider)
if (!provider) {
throw new Error('Failed to initialize speech provider')
}
// Get provider configuration
const providerConfig = providersStore.getProviderConfig(providerId)
// Use the reactive model computed property (not a local variable)
const modelToUse = modelId || model.value || defaultModel
return await speechStore.speech(
provider,
modelId || model.value,
modelToUse,
input,
voiceId || voice.value,
voiceId || (voice.value as string),
{
...providerConfig,
...defaultVoiceSettings,
@@ -89,44 +126,54 @@ async function handleGenerateSpeech(input: string, voiceId: string, _useSSML: bo
)
}
watch(speed, async () => {
if (!providers.value[providerId])
providers.value[providerId] = {}
providers.value[providerId].speed = speed.value
})
watch(model, () => {
// Ensure provider config exists
if (!providers.value[providerId])
providers.value[providerId] = {}
// Save model to provider config (this persists to localStorage automatically)
providers.value[providerId].model = model.value
})
watch(voice, () => {
// Ensure provider config exists
if (!providers.value[providerId])
providers.value[providerId] = {}
// Save voice to provider config (this persists to localStorage automatically)
providers.value[providerId].voice = voice.value
})
// Use the composable to get validation logic and state
const {
t,
router,
providerMetadata,
isValidating,
isValid,
validationMessage,
handleResetSettings,
forceValid,
} = useProviderValidation(providerId)
</script>
<template>
<ProviderSettingsLayout
:provider-name="providerMetadata?.localizedName"
:provider-icon-color="providerMetadata?.iconColor"
:on-back="() => router.back()"
>
<ProviderSettingsContainer>
<ProviderBasicSettings
:title="t('settings.pages.providers.common.section.basic.title')"
:description="t('settings.pages.providers.common.section.basic.description')"
:on-reset="handleResetSettings"
>
<ProviderApiKeyInput
v-model="apiKey"
:required="false"
:provider-name="providerMetadata?.localizedName"
<SpeechProviderSettings
:provider-id="providerId"
:default-model="defaultModel"
:additional-settings="defaultVoiceSettings"
placeholder="sk-..."
>
<!-- Voice settings specific to OpenAI Compatible -->
<template #voice-settings>
<!-- Model input -->
<FieldInput
v-model="model"
label="Model"
description="Enter the TTS model to use for speech generation"
placeholder="tts-1"
/>
</ProviderBasicSettings>
<ProviderAdvancedSettings :title="t('settings.pages.providers.common.section.advanced.title')">
<ProviderBaseUrlInput
v-model="baseUrl"
placeholder="https://api.openai.com/v1/"
/>
<!-- Speed control - common to most providers -->
<FieldRange
v-model="speed"
:label="t('settings.pages.providers.provider.common.fields.field.speed.label')"
@@ -134,9 +181,20 @@ const {
:min="0.5"
:max="2.0" :step="0.01"
/>
</ProviderAdvancedSettings>
</template>
<template #playground>
<SpeechPlaygroundOpenAICompatible
v-model:model-value="model"
v-model:voice="voice as any"
:generate-speech="handleGenerateSpeech"
:api-key-configured="apiKeyConfigured"
default-text="Hello! This is a test of the OpenAI Compatible Speech."
/>
</template>
<!-- Validation Status -->
<template #advanced-settings>
<Alert v-if="!isValid && isValidating === 0 && validationMessage" type="error">
<template #title>
<div class="w-full flex items-center justify-between">
@@ -161,16 +219,8 @@ const {
{{ t('settings.dialogs.onboarding.validationSuccess') }}
</template>
</Alert>
</ProviderSettingsContainer>
<SpeechPlaygroundOpenAICompatible
v-model:model-value="model"
v-model:voice="voice"
:generate-speech="handleGenerateSpeech"
:api-key-configured="apiKeyConfigured"
default-text="Hello! This is a test of the OpenAI Compatible Speech."
/>
</ProviderSettingsLayout>
</template>
</SpeechProviderSettings>
</template>
<route lang="yaml">
@@ -104,8 +104,9 @@ const isWebSpeechAPIAvailable = computed(() => {
&& ('webkitSpeechRecognition' in window || 'SpeechRecognition' in window)
})
onMounted(() => {
onMounted(async () => {
ensureProviderSettings()
// Audio devices are loaded on demand when user requests them
})
// Speech-to-Text test state (always uses Web Speech API)
@@ -388,14 +389,8 @@ onUnmounted(() => {
</div>
</div>
<div v-else-if="!selectedAudioInput" class="border border-amber-200 rounded-lg bg-amber-50 p-3 dark:border-amber-800 dark:bg-amber-900/20">
<div class="flex items-center gap-2 text-amber-700 dark:text-amber-400">
<div i-solar:warning-circle-line-duotone class="text-lg" />
<span class="text-sm font-medium">Please select an audio input device to test</span>
</div>
</div>
<div v-else class="flex flex-col gap-4">
<!-- Audio Input Device Selector - Always visible when Web Speech API is available -->
<div class="flex items-center gap-2">
<FieldSelect
v-model="selectedAudioInput"
@@ -411,9 +406,17 @@ onUnmounted(() => {
/>
</div>
<!-- Warning if no device selected -->
<div v-if="!selectedAudioInput" class="border border-amber-200 rounded-lg bg-amber-50 p-3 dark:border-amber-800 dark:bg-amber-900/20">
<div class="flex items-center gap-2 text-amber-700 dark:text-amber-400">
<div i-solar:warning-circle-line-duotone class="text-lg" />
<span class="text-sm font-medium">Please select an audio input device to test</span>
</div>
</div>
<div class="flex items-center gap-2">
<Button
:disabled="isTranscribing && !isTestingSTT"
:disabled="!selectedAudioInput || (isTranscribing && !isTestingSTT)"
class="flex-1"
@click="isTestingSTT ? stopSTTTest() : startSTTTest()"
>
@@ -7,8 +7,9 @@ import {
} from '@proj-airi/stage-ui/components'
import { useHearingStore } from '@proj-airi/stage-ui/stores/modules/hearing'
import { useProvidersStore } from '@proj-airi/stage-ui/stores/providers'
import { FieldSelect } from '@proj-airi/ui'
import { storeToRefs } from 'pinia'
import { computed } from 'vue'
import { computed, onMounted, watch } from 'vue'
const hearingStore = useHearingStore()
const providersStore = useProvidersStore()
@@ -18,31 +19,60 @@ const { providers } = storeToRefs(providersStore)
const providerId = 'openai-audio-transcription'
const defaultModel = 'whisper-1'
// Model selection
const model = computed({
get: () => providers.value[providerId]?.model as string | undefined || defaultModel,
set: (value) => {
if (!providers.value[providerId])
providers.value[providerId] = {}
providers.value[providerId].model = value
},
})
// Load models
const providerModels = computed(() => {
return providersStore.getModelsForProvider(providerId)
})
const isLoadingModels = computed(() => {
return providersStore.isLoadingModels[providerId] || false
})
// Check if API key is configured
const apiKeyConfigured = computed(() => !!providers.value[providerId]?.apiKey)
// Generate speech with ElevenLabs-specific parameters
// Load models on mount
onMounted(async () => {
await providersStore.loadModelsForConfiguredProviders()
await providersStore.fetchModelsForProvider(providerId)
})
// Generate transcription
async function handleGenerateTranscription(file: File) {
const provider = await providersStore.getProviderInstance<TranscriptionProviderWithExtraOptions<string, any>>(providerId)
if (!provider) {
throw new Error('Failed to initialize speech provider')
throw new Error('Failed to initialize transcription provider')
}
// Get provider configuration
const providerConfig = providersStore.getProviderConfig(providerId)
// Get model from configuration or use default
const model = providerConfig.model as string | undefined || defaultModel
const modelToUse = providerConfig.model as string | undefined || defaultModel
// ElevenLabs doesn't need SSML conversion, but if SSML is provided, use it directly
return await hearingStore.transcription(
providerId,
provider,
model,
modelToUse,
file,
'json',
)
}
watch(model, async () => {
const providerConfig = providersStore.getProviderConfig(providerId)
providerConfig.model = model.value
})
</script>
<template>
@@ -50,6 +80,17 @@ async function handleGenerateTranscription(file: File) {
:provider-id="providerId"
:default-model="defaultModel"
>
<template #basic-settings>
<!-- Model selection -->
<FieldSelect
v-model="model"
label="Model"
description="Select the transcription model to use"
:options="providerModels.map(m => ({ value: m.id, label: m.name }))"
:disabled="isLoadingModels || providerModels.length === 0"
placeholder="Select a model..."
/>
</template>
<template #playground>
<TranscriptionPlayground
:generate-transcription="handleGenerateTranscription"
@@ -15,9 +15,9 @@ import {
import { useProviderValidation } from '@proj-airi/stage-ui/composables/use-provider-validation'
import { useHearingStore } from '@proj-airi/stage-ui/stores/modules/hearing'
import { useProvidersStore } from '@proj-airi/stage-ui/stores/providers'
import { FieldInput } from '@proj-airi/ui'
import { FieldInput, FieldSelect } from '@proj-airi/ui'
import { storeToRefs } from 'pinia'
import { computed } from 'vue'
import { computed, onMounted, watch } from 'vue'
const providerId = 'openai-compatible-audio-transcription'
const hearingStore = useHearingStore()
@@ -35,7 +35,14 @@ const apiKey = computed({
})
const baseUrl = computed({
get: () => providers.value[providerId]?.baseUrl || '',
get: () => {
const stored = providers.value[providerId]?.baseUrl
if (stored)
return stored
// Use default from provider metadata if available
const metadata = providersStore.getProviderMetadata(providerId)
return metadata?.defaultOptions?.().baseUrl as string | undefined || ''
},
set: (value) => {
if (!providers.value[providerId])
providers.value[providerId] = {}
@@ -52,6 +59,15 @@ const model = computed({
},
})
// Load models
const providerModels = computed(() => {
return providersStore.getModelsForProvider(providerId)
})
const isLoadingModels = computed(() => {
return providersStore.isLoadingModels[providerId] || false
})
// Check if API key is configured
const apiKeyConfigured = computed(() => !!providers.value[providerId]?.apiKey)
@@ -61,10 +77,21 @@ async function handleGenerateTranscription(file: File) {
if (!provider)
throw new Error('Failed to initialize transcription provider')
// Get provider configuration
const providerConfig = providersStore.getProviderConfig(providerId)
// Get model from configuration or use the reactive model value
const modelToUse = providerConfig.model as string | undefined || model.value
// Validate model - throw error if no valid model configured
if (!modelToUse || !isValidTranscriptionModel(modelToUse)) {
throw new Error(`Invalid or missing transcription model. Please configure a valid model in the provider settings.`)
}
return await hearingStore.transcription(
providerId,
provider,
model.value,
modelToUse,
file,
'json',
)
@@ -81,6 +108,75 @@ const {
handleResetSettings,
forceValid,
} = useProviderValidation(providerId)
// Expand Advanced section if there's a base URL validation error
const shouldExpandAdvanced = computed(() => {
if (!validationMessage.value)
return false
// Check if validation message mentions base URL
const message = validationMessage.value.toLowerCase()
return message.includes('base url') || message.includes('baseurl')
})
// Valid transcription models (OpenAI doesn't provide an API to list these)
const VALID_TRANSCRIPTION_MODELS = [
'whisper-1',
'gpt-4o-transcribe',
'gpt-4o-mini-transcribe',
'gpt-4o-mini-transcribe-2025-12-15',
'gpt-4o-transcribe-diarize',
]
// Check if a model is a valid transcription model
function isValidTranscriptionModel(modelName: string | undefined | null): boolean {
if (!modelName)
return false
// Check if it's a known transcription model
if (VALID_TRANSCRIPTION_MODELS.includes(modelName))
return true
// Allow custom models that might be transcription-compatible
// But reject obvious chat models
if (modelName.includes('gpt-4') && !modelName.includes('transcribe') && !modelName.includes('whisper'))
return false
return true
}
// Initialize provider settings on mount
onMounted(async () => {
providersStore.initializeProvider(providerId)
// Initialize baseUrl with default if not set
if (!providers.value[providerId]?.baseUrl) {
const metadata = providersStore.getProviderMetadata(providerId)
const defaultBaseUrl = metadata?.defaultOptions?.().baseUrl as string | undefined
if (defaultBaseUrl) {
baseUrl.value = defaultBaseUrl
}
}
// Validate and reset model if it's invalid (e.g., a chat model)
const currentModel = model.value
if (currentModel && !isValidTranscriptionModel(currentModel)) {
console.warn(`Invalid transcription model "${currentModel}" detected. Resetting to default "whisper-1".`)
model.value = 'whisper-1'
}
// Load models if API key and base URL are configured
if (apiKey.value && baseUrl.value) {
await providersStore.loadModelsForConfiguredProviders()
await providersStore.fetchModelsForProvider(providerId)
}
})
// Watch for API key and base URL changes to reload models
watch([apiKey, baseUrl], async ([newApiKey, newBaseUrl]) => {
if (newApiKey && newBaseUrl) {
await providersStore.fetchModelsForProvider(providerId)
}
})
// Watch model changes to save to provider config
watch(model, () => {
const providerConfig = providersStore.getProviderConfig(providerId)
providerConfig.model = model.value
})
</script>
<template>
@@ -89,7 +185,8 @@ const {
:provider-icon-color="providerMetadata?.iconColor"
:on-back="() => router.back()"
>
<ProviderSettingsContainer>
<div flex="~ col md:row gap-6">
<ProviderSettingsContainer class="w-full md:w-[40%]">
<ProviderBasicSettings
:title="t('settings.pages.providers.common.section.basic.title')"
:description="t('settings.pages.providers.common.section.basic.description')"
@@ -100,17 +197,33 @@ const {
:provider-name="providerMetadata?.localizedName"
placeholder="sk-..."
/>
<!-- Model selection: Use dropdown if models are available, otherwise use text input -->
<FieldSelect
v-if="providerModels.length > 0"
v-model="model"
label="Model"
description="Select the transcription model to use"
:options="providerModels.map(m => ({ value: m.id, label: m.name }))"
:disabled="isLoadingModels"
placeholder="Select a model..."
/>
<FieldInput
v-else
v-model="model"
:label="t('settings.pages.modules.consciousness.sections.section.provider-model-selection.manual_model_name')"
:description="apiKey && baseUrl ? 'Enter model name manually, or wait for models to load...' : 'Enter the transcription model name (e.g., whisper-1)'"
:placeholder="t('settings.pages.modules.consciousness.sections.section.provider-model-selection.manual_model_placeholder')"
/>
</ProviderBasicSettings>
<ProviderAdvancedSettings :title="t('settings.pages.providers.common.section.advanced.title')">
<ProviderAdvancedSettings
:title="t('settings.pages.providers.common.section.advanced.title')"
:initial-visible="shouldExpandAdvanced"
>
<ProviderBaseUrlInput
v-model="baseUrl"
placeholder="https://api.openai.com/v1/"
required
/>
</ProviderAdvancedSettings>
@@ -141,10 +254,16 @@ const {
</Alert>
</ProviderSettingsContainer>
<!-- Playground section -->
<div flex="~ col gap-6" class="w-full md:w-[60%]">
<div w-full rounded-xl>
<TranscriptionPlayground
:generate-transcription="handleGenerateTranscription"
:api-key-configured="apiKeyConfigured"
/>
</div>
</div>
</div>
</ProviderSettingsLayout>
</template>
@@ -135,21 +135,24 @@ function updateCustomValue(value: string) {
<!-- Items grid -->
<div class="relative">
<!-- Horizontally scrollable container -->
<!-- Responsive grid container -->
<div
class="grid auto-cols-[350px] grid-flow-col gap-4 overflow-x-auto pb-4 scrollbar-none"
:class="[
isListExpanded ? 'grid-cols-1 md:grid-cols-2 grid-flow-row auto-cols-auto' : '',
isListExpanded
? 'grid grid-cols-1 md:grid-cols-2 gap-4'
: 'grid auto-cols-[min(300px,calc((100vw-5rem)/2))] grid-flow-col gap-4 overflow-x-auto scrollbar-none pb-2',
...(props.listClass
? (typeof props.listClass === 'string'
? [props.listClass]
: props.listClass
)
: ['max-h-[calc(100dvh-7lh)]']
: isListExpanded
? ['max-h-[calc(100dvh-7lh)] overflow-y-auto']
: []
),
]"
transition="all duration-200 ease-in-out"
style="scroll-snap-type: x mandatory;"
:style="isListExpanded ? '' : 'scroll-snap-type: x mandatory;'"
>
<RadioCardDetail
v-for="item in filteredItems"
@@ -338,21 +338,24 @@ const customVoiceName = ref('')
<!-- Voices grid -->
<div class="relative">
<!-- Horizontally scrollable container -->
<!-- Responsive grid container -->
<div
class="grid auto-cols-[350px] grid-flow-col gap-4 overflow-x-auto scrollbar-none"
:class="[
isListExpanded ? 'grid-cols-1 md:grid-cols-2 grid-flow-row auto-cols-auto' : '',
isListExpanded
? 'grid grid-cols-1 md:grid-cols-2 gap-4'
: 'grid auto-cols-[min(300px,calc((100vw-5rem)/2))] grid-flow-col gap-4 overflow-x-auto scrollbar-none pb-2',
...(props.listClass
? (typeof props.listClass === 'string'
? [props.listClass]
: props.listClass
)
: ['max-h-[calc(100dvh-7lh)]']
: isListExpanded
? ['max-h-[calc(100dvh-7lh)] overflow-y-auto']
: []
),
]"
transition="all duration-200 ease-in-out"
style="scroll-snap-type: x mandatory;"
:style="isListExpanded ? '' : 'scroll-snap-type: x mandatory;'"
>
<!-- Not support voices warning -->
<Alert v-if="!searchQuery && filteredVoices.length === 0" type="warning">
@@ -147,12 +147,6 @@ function togglePlayback() {
<div v-if="currentlyPlayingId === voice.id" class="i-solar:pause-circle-bold-duotone text-xl text-neutral-400 dark:text-neutral-500" />
<div v-else class="i-solar:play-circle-bold-duotone text-xl text-neutral-400 dark:text-neutral-500" />
</button>
<div
v-else
class="mt-auto w-full flex items-center justify-center bg-neutral-50 py-3 text-xs text-neutral-400 italic dark:bg-neutral-800/50 dark:text-neutral-600"
>
No preview available. You can select it and test voice on the right experiment.
</div>
<!-- Voice info -->
<div class="flex-1 cursor-pointer">
@@ -124,14 +124,10 @@ defineExpose({
</div>
</h2>
<div flex="~ col gap-4">
<FieldInput
v-model="model"
label="Model ID"
placeholder="tts-1"
/>
<FieldInput
v-model="voice"
label="Voice"
description="Enter the voice ID for your OpenAI-compatible API"
placeholder="alloy"
/>
<FieldCheckbox
@@ -162,10 +158,9 @@ defineExpose({
</template>
<!-- Playground actions -->
<div flex="~ row" gap-4>
<button
border="neutral-800 dark:neutral-200 solid 2" transition="border duration-250 ease-in-out"
rounded-lg px-4 text="neutral-100 dark:neutral-900" py-2 text-sm
rounded-lg px-3 text="neutral-100 dark:neutral-900" py-1.5 text-sm
:disabled="isGenerating || (!testText.trim() && !useSSML) || (useSSML && !ssmlText.trim()) || !apiKeyConfigured"
:class="{ 'opacity-50 cursor-not-allowed': isGenerating || (!testText.trim() && !useSSML) || (useSSML && !ssmlText.trim()) || !apiKeyConfigured }"
bg="neutral-700 dark:neutral-300" @click="handleGenerateTestSpeech"
@@ -175,16 +170,6 @@ defineExpose({
<span>{{ isGenerating ? t('settings.pages.providers.provider.elevenlabs.playground.buttons.button.test-voice.generating') : t('settings.pages.providers.provider.elevenlabs.playground.buttons.button.test-voice.label') }}</span>
</div>
</button>
<button
v-if="audioUrl" border="primary-300 dark:primary-800 solid 2"
transition="border duration-250 ease-in-out" rounded-lg px-4 py-2 text-sm @click="stopTestAudio"
>
<div flex="~ row" items-center gap-2>
<div i-solar:stop-circle-bold-duotone />
<span>{{ t('settings.pages.modules.speech.sections.section.playground.buttons.stop.label') }}</span>
</div>
</button>
</div>
<!-- Error messages -->
<div v-if="!apiKeyConfigured" class="mt-2 text-sm text-red-500">
{{ t('settings.pages.providers.provider.elevenlabs.playground.validation.error-missing-api-key') }}
@@ -164,7 +164,6 @@ defineExpose({
<FieldSelect
v-model="selectedVoice"
class="[&>div]:grid [&>div]:grid-cols-[4fr_2fr]"
:options="voiceOptions"
:label="t('settings.pages.providers.provider.elevenlabs.playground.fields.field.voice.label')"
:description="t('settings.pages.providers.provider.elevenlabs.playground.fields.field.voice.description')"
@@ -172,10 +171,9 @@ defineExpose({
/>
<!-- Playground actions -->
<div flex="~ row" gap-4>
<button
border="neutral-800 dark:neutral-200 solid 2" transition="border duration-250 ease-in-out"
rounded-lg px-4 text="neutral-100 dark:neutral-900" py-2 text-sm
rounded-lg px-3 text="neutral-100 dark:neutral-900" py-1.5 text-sm
:disabled="isGenerating || (!testText.trim() && !useSSML) || (useSSML && !ssmlText.trim()) || !selectedVoice || !apiKeyConfigured"
:class="{ 'opacity-50 cursor-not-allowed': isGenerating || (!testText.trim() && !useSSML) || (useSSML && !ssmlText.trim()) || !selectedVoice || !apiKeyConfigured }"
bg="neutral-700 dark:neutral-300" @click="handleGenerateTestSpeech"
@@ -185,16 +183,6 @@ defineExpose({
<span>{{ isGenerating ? t('settings.pages.providers.provider.elevenlabs.playground.buttons.button.test-voice.generating') : t('settings.pages.providers.provider.elevenlabs.playground.buttons.button.test-voice.label') }}</span>
</div>
</button>
<button
v-if="audioUrl" border="primary-300 dark:primary-800 solid 2"
transition="border duration-250 ease-in-out" rounded-lg px-4 py-2 text-sm @click="stopTestAudio"
>
<div flex="~ row" items-center gap-2>
<div i-solar:stop-circle-bold-duotone />
<span>{{ t('settings.pages.modules.speech.sections.section.playground.buttons.stop.label') }}</span>
</div>
</button>
</div>
<!-- Error messages -->
<div v-if="!apiKeyConfigured" class="mt-2 text-sm text-red-500">
{{ t('settings.pages.providers.provider.elevenlabs.playground.validation.error-missing-api-key') }}
@@ -106,11 +106,15 @@ onStopRecord(async (recording) => {
try {
if (recording && recording.size > 0) {
audios.value.push(recording)
// Clear any previous error message
errorMessage.value = ''
const result = await props.generateTranscription(new File([recording], 'recording.wav'))
const text = result.mode === 'stream'
? await result.text
: result.text
transcriptions.value.push(text)
// Clear error message on success
errorMessage.value = ''
}
}
catch (err) {
@@ -122,6 +126,15 @@ onStopRecord(async (recording) => {
// Monitoring toggle
async function toggleMonitoring() {
if (!isMonitoring.value) {
// Clear previous recordings and transcriptions when starting a new monitoring session
// Clean up previous audio URLs
audioCleanups.value.forEach(cleanup => cleanup())
audioCleanups.value = []
audios.value = []
transcriptions.value = []
// Clear any previous error messages
errorMessage.value = ''
await setupAudioMonitoring()
await startRecord()
isMonitoring.value = true
@@ -178,6 +191,14 @@ onUnmounted(() => {
{{ isMonitoring ? 'Stop Monitoring' : 'Start Monitoring' }}
</Button>
<!-- Error message display -->
<div v-if="errorMessage" class="mb-4 border border-red-200 rounded-lg bg-red-50 p-3 dark:border-red-800 dark:bg-red-900/20">
<div class="flex items-center gap-2 text-red-700 dark:text-red-400">
<div i-solar:warning-circle-line-duotone class="text-lg" />
<span class="text-sm font-medium">{{ errorMessage }}</span>
</div>
</div>
<div>
<div v-for="(audio, index) in audioURLs" :key="index" class="mb-2">
<audio :src="audio" controls class="w-full" />
+134 -28
View File
@@ -162,14 +162,29 @@ function playSpecialToken(special: string) {
}
const lipSyncNode = ref<AudioNode>()
const playbackManager = createPlaybackManager<AudioBuffer>({
play: (item, signal) => {
return new Promise((resolve) => {
async function playFunction(item: Parameters<Parameters<typeof createPlaybackManager<AudioBuffer>>[0]['play']>[0], signal: AbortSignal): Promise<void> {
return new Promise<void>(async (resolve) => {
if (!audioContext) {
resolve()
return
}
if (!item.audio) {
resolve()
return
}
// Ensure audio context is resumed (browsers suspend it by default until user interaction)
if (audioContext.state === 'suspended') {
try {
await audioContext.resume()
}
catch {
resolve()
return
}
}
const source = audioContext.createBufferSource()
currentAudioSource.value = source
source.buffer = item.audio
@@ -202,9 +217,17 @@ const playbackManager = createPlaybackManager<AudioBuffer>({
stopPlayback()
}
try {
source.start(0)
}
catch {
stopPlayback()
}
})
},
}
const playbackManager = createPlaybackManager<AudioBuffer>({
play: playFunction,
maxVoices: 1,
maxVoicesPerOwner: 1,
overflowPolicy: 'queue',
@@ -216,15 +239,8 @@ const speechPipeline = createSpeechPipeline<AudioBuffer>({
if (signal.aborted)
return null
if (!activeSpeechProvider.value) {
console.warn('No active speech provider configured')
if (!activeSpeechProvider.value)
return null
}
if (!activeSpeechVoice.value) {
console.warn('No active speech voice configured')
return null
}
const provider = await providersStore.getProviderInstance(activeSpeechProvider.value) as SpeechProviderWithExtraOptions<string, UnElevenLabsOptions>
if (!provider) {
@@ -236,20 +252,72 @@ const speechPipeline = createSpeechPipeline<AudioBuffer>({
return null
const providerConfig = providersStore.getProviderConfig(activeSpeechProvider.value)
const input = ssmlEnabled.value
? speechStore.generateSSML(request.text, activeSpeechVoice.value, { ...providerConfig, pitch: pitch.value })
: request.text
const res = await generateSpeech({
...provider.speech(activeSpeechModel.value, providerConfig),
input,
voice: activeSpeechVoice.value.id,
})
// For OpenAI Compatible providers, always use provider config for model and voice
// since these are manually configured in provider settings
let model = activeSpeechModel.value
let voice = activeSpeechVoice.value
if (signal.aborted)
if (activeSpeechProvider.value === 'openai-compatible-audio-speech') {
// Always prefer provider config for OpenAI Compatible (user configured it there)
if (providerConfig?.model) {
model = providerConfig.model as string
}
else {
// Fallback to default if not in provider config
model = 'tts-1'
console.warn('[Speech Pipeline] OpenAI Compatible: No model in provider config, using default', { providerConfig })
}
if (providerConfig?.voice) {
voice = {
id: providerConfig.voice as string,
name: providerConfig.voice as string,
description: providerConfig.voice as string,
previewURL: '',
languages: [{ code: 'en', title: 'English' }],
provider: activeSpeechProvider.value,
gender: 'neutral',
}
}
else {
// Fallback to default if not in provider config
voice = {
id: 'alloy',
name: 'alloy',
description: 'alloy',
previewURL: '',
languages: [{ code: 'en', title: 'English' }],
provider: activeSpeechProvider.value,
gender: 'neutral',
}
console.warn('[Speech Pipeline] OpenAI Compatible: No voice in provider config, using default', { providerConfig })
}
}
if (!model || !voice)
return null
return audioContext.decodeAudioData(res)
const input = ssmlEnabled.value
? speechStore.generateSSML(request.text, voice, { ...providerConfig, pitch: pitch.value })
: request.text
try {
const res = await generateSpeech({
...provider.speech(model, providerConfig),
input,
voice: voice.id,
})
if (signal.aborted || !res || res.byteLength === 0)
return null
const audioBuffer = await audioContext.decodeAudioData(res)
return audioBuffer
}
catch {
return null
}
},
playback: playbackManager,
})
@@ -271,15 +339,22 @@ playbackManager.onEnd(({ item }) => {
playbackManager.onStart(({ item }) => {
nowSpeaking.value = true
// NOTICE: currently, postCaption, postPresent from useBroadcastChannel may throw error
// once we navigate away from the page that created the BroadcastChannel,
// as the channel gets closed on unmount, leading to "Failed to execute 'postMessage' on 'BroadcastChannel': The channel is closed."
// error that may block hooks or throw exceptions silently.
//
// TODO: we should consider better way to manage BroadcastChannel lifecycle to avoid such issues.
// NOTICE: postCaption and postPresent may throw errors if the BroadcastChannel is closed
// (e.g., when navigating away from the page). We wrap these in try-catch to prevent
// breaking playback when the channel is unavailable.
assistantCaption.value += ` ${item.text}`
try {
postCaption({ type: 'caption-assistant', text: assistantCaption.value })
}
catch {
// BroadcastChannel may be closed - don't break playback
}
try {
postPresent({ type: 'assistant-append', text: item.text })
}
catch {
// BroadcastChannel may be closed - don't break playback
}
})
function startLipSyncLoop() {
@@ -332,8 +407,20 @@ chatHookCleanups.push(onBeforeMessageComposed(async () => {
await setupLipSync()
// Reset assistant caption for a new message
assistantCaption.value = ''
try {
postCaption({ type: 'caption-assistant', text: '' })
}
catch (error) {
// BroadcastChannel may be closed if user navigated away - don't break flow
console.warn('[Stage] Failed to post caption reset (channel may be closed)', { error })
}
try {
postPresent({ type: 'assistant-reset' })
}
catch (error) {
// BroadcastChannel may be closed if user navigated away - don't break flow
console.warn('[Stage] Failed to post present reset (channel may be closed)', { error })
}
if (currentChatIntent) {
currentChatIntent.cancel('new-message')
@@ -379,6 +466,25 @@ onUnmounted(() => {
lipSyncStarted.value = false
})
// Resume audio context on first user interaction (browser requirement)
let audioContextResumed = false
function resumeAudioContextOnInteraction() {
if (audioContextResumed || !audioContext)
return
audioContextResumed = true
audioContext.resume().catch(() => {
// Ignore errors - audio context will be resumed when needed
})
}
// Add event listeners for user interaction
if (typeof window !== 'undefined') {
const events = ['click', 'touchstart', 'keydown']
events.forEach((event) => {
window.addEventListener(event, resumeAudioContextOnInteraction, { once: true, passive: true })
})
}
onMounted(async () => {
db.value = drizzle({ connection: { bundles: getImportUrlBundles() } })
await db.value.execute(`CREATE TABLE memory_test (vec FLOAT[768]);`)
@@ -100,7 +100,14 @@ export const useHearingStore = defineStore('hearing-store', () => {
return true // Web Speech API is ready if provider is selected and available
}
return !!activeTranscriptionModel.value
// For OpenAI Compatible providers, check provider config as fallback
let hasProviderModel = false
if (activeTranscriptionProvider.value === 'openai-compatible-audio-transcription') {
const providerConfig = providersStore.getProviderConfig(activeTranscriptionProvider.value)
hasProviderModel = !!providerConfig?.model
}
return !!activeTranscriptionModel.value || hasProviderModel
})
function resetState() {
+35 -2
View File
@@ -121,7 +121,27 @@ export const useSpeechStore = defineStore('speech', () => {
watch([activeSpeechVoiceId, availableVoices], ([voiceId, voices]) => {
if (voiceId) {
activeSpeechVoice.value = voices[activeSpeechProvider.value]?.find(voice => voice.id === voiceId)
// For OpenAI Compatible, create a custom voice object (no voices available from API)
if (activeSpeechProvider.value === 'openai-compatible-audio-speech') {
// Always update to match voiceId (in case it changed)
activeSpeechVoice.value = {
id: voiceId,
name: voiceId,
description: voiceId,
previewURL: '',
languages: [{ code: 'en', title: 'English' }],
provider: activeSpeechProvider.value,
gender: 'neutral',
}
}
else {
// For other providers, find voice in available voices
const foundVoice = voices[activeSpeechProvider.value]?.find(voice => voice.id === voiceId)
// Only update if we found a voice, or if activeSpeechVoice is not set
if (foundVoice || !activeSpeechVoice.value) {
activeSpeechVoice.value = foundVoice
}
}
}
}, {
immediate: true,
@@ -201,7 +221,20 @@ export const useSpeechStore = defineStore('speech', () => {
}
const configured = computed(() => {
return !!activeSpeechProvider.value && !!activeSpeechModel.value && !!activeSpeechVoiceId.value
if (!activeSpeechProvider.value)
return false
let hasModel = !!activeSpeechModel.value
let hasVoice = !!activeSpeechVoiceId.value
// For OpenAI Compatible providers, check provider config as fallback
if (activeSpeechProvider.value === 'openai-compatible-audio-speech') {
const providerConfig = providersStore.getProviderConfig(activeSpeechProvider.value)
hasModel ||= !!providerConfig?.model
hasVoice ||= !!providerConfig?.voice
}
return hasModel && hasVoice
})
function resetState() {
+149 -14
View File
@@ -700,94 +700,118 @@ export const useProvidersStore = defineStore('providers', () => {
creator: createOpenAI,
validation: ['health'],
capabilities: {
listVoices: async () => {
// NOTE: OpenAI does not provide an API endpoint to retrieve available voices.
// Voices are hardcoded here - this is a provider limitation, not an application limitation.
// Voice compatibility per https://platform.openai.com/docs/api-reference/audio/createSpeech:
// - tts-1 and tts-1-hd support: alloy, ash, coral, echo, fable, onyx, nova, sage, shimmer (9 voices)
// - gpt-4o-mini-tts supports all 13 voices: alloy, ash, ballad, coral, echo, fable, nova, onyx, sage, shimmer, verse, marin, cedar
listVoices: async (_config: Record<string, unknown>) => {
return [
{
id: 'alloy',
name: 'Alloy',
provider: 'openai-audio-speech',
languages: [],
compatibleModels: ['tts-1', 'tts-1-hd'],
compatibleModels: ['tts-1', 'tts-1-hd', 'gpt-4o-mini-tts', 'gpt-4o-mini-tts-2025-12-15'],
},
{
id: 'ash',
name: 'Ash',
provider: 'openai-audio-speech',
languages: [],
compatibleModels: ['tts-1', 'tts-1-hd'],
compatibleModels: ['tts-1', 'tts-1-hd', 'gpt-4o-mini-tts', 'gpt-4o-mini-tts-2025-12-15'],
},
{
id: 'ballad',
name: 'Ballad',
provider: 'openai-audio-speech',
languages: [],
compatibleModels: ['tts-1', 'tts-1-hd'],
compatibleModels: ['gpt-4o-mini-tts', 'gpt-4o-mini-tts-2025-12-15'],
},
{
id: 'coral',
name: 'Coral',
provider: 'openai-audio-speech',
languages: [],
compatibleModels: ['tts-1', 'tts-1-hd'],
compatibleModels: ['tts-1', 'tts-1-hd', 'gpt-4o-mini-tts', 'gpt-4o-mini-tts-2025-12-15'],
},
{
id: 'echo',
name: 'Echo',
provider: 'openai-audio-speech',
languages: [],
compatibleModels: ['tts-1', 'tts-1-hd'],
compatibleModels: ['tts-1', 'tts-1-hd', 'gpt-4o-mini-tts', 'gpt-4o-mini-tts-2025-12-15'],
},
{
id: 'fable',
name: 'Fable',
provider: 'openai-audio-speech',
languages: [],
compatibleModels: ['tts-1', 'tts-1-hd'],
compatibleModels: ['tts-1', 'tts-1-hd', 'gpt-4o-mini-tts', 'gpt-4o-mini-tts-2025-12-15'],
},
{
id: 'onyx',
name: 'Onyx',
provider: 'openai-audio-speech',
languages: [],
compatibleModels: ['tts-1', 'tts-1-hd'],
compatibleModels: ['tts-1', 'tts-1-hd', 'gpt-4o-mini-tts', 'gpt-4o-mini-tts-2025-12-15'],
},
{
id: 'nova',
name: 'Nova',
provider: 'openai-audio-speech',
languages: [],
compatibleModels: ['tts-1', 'tts-1-hd'],
compatibleModels: ['tts-1', 'tts-1-hd', 'gpt-4o-mini-tts', 'gpt-4o-mini-tts-2025-12-15'],
},
{
id: 'sage',
name: 'Sage',
provider: 'openai-audio-speech',
languages: [],
compatibleModels: ['tts-1', 'tts-1-hd'],
compatibleModels: ['tts-1', 'tts-1-hd', 'gpt-4o-mini-tts', 'gpt-4o-mini-tts-2025-12-15'],
},
{
id: 'shimmer',
name: 'Shimmer',
provider: 'openai-audio-speech',
languages: [],
compatibleModels: ['tts-1', 'tts-1-hd'],
compatibleModels: ['tts-1', 'tts-1-hd', 'gpt-4o-mini-tts', 'gpt-4o-mini-tts-2025-12-15'],
},
{
id: 'verse',
name: 'Verse',
provider: 'openai-audio-speech',
languages: [],
compatibleModels: ['tts-1', 'tts-1-hd'],
compatibleModels: ['gpt-4o-mini-tts', 'gpt-4o-mini-tts-2025-12-15'],
},
{
id: 'marin',
name: 'Marin',
provider: 'openai-audio-speech',
languages: [],
compatibleModels: ['gpt-4o-mini-tts', 'gpt-4o-mini-tts-2025-12-15'],
},
{
id: 'cedar',
name: 'Cedar',
provider: 'openai-audio-speech',
languages: [],
compatibleModels: ['gpt-4o-mini-tts', 'gpt-4o-mini-tts-2025-12-15'],
},
] satisfies VoiceInfo[]
},
listModels: async () => {
// TESTING NOTES: All 4 models tested and confirmed working with fable voice:
// - tts-1: {model: "tts-1", input: "test", voice: "fable"} ✓
// - tts-1-hd: {model: "tts-1-hd", input: "test", voice: "fable"} ✓
// - gpt-4o-mini-tts: {model: "gpt-4o-mini-tts", input: "test", voice: "fable"} ✓
// - gpt-4o-mini-tts-2025-12-15: {model: "gpt-4o-mini-tts-2025-12-15", input: "test", voice: "fable"} ✓
return [
{
id: 'tts-1',
name: 'TTS-1',
provider: 'openai-audio-speech',
description: '',
description: 'Optimized for real-time text-to-speech tasks',
contextLength: 0,
deprecated: false,
},
@@ -795,7 +819,23 @@ export const useProvidersStore = defineStore('providers', () => {
id: 'tts-1-hd',
name: 'TTS-1-HD',
provider: 'openai-audio-speech',
description: '',
description: 'Higher fidelity audio output',
contextLength: 0,
deprecated: false,
},
{
id: 'gpt-4o-mini-tts',
name: 'GPT-4o Mini TTS',
provider: 'openai-audio-speech',
description: 'GPT-4o Mini optimized for text-to-speech',
contextLength: 0,
deprecated: false,
},
{
id: 'gpt-4o-mini-tts-2025-12-15',
name: 'GPT-4o Mini TTS (2025-12-15)',
provider: 'openai-audio-speech',
description: 'GPT-4o Mini TTS snapshot from 2025-12-15',
contextLength: 0,
deprecated: false,
},
@@ -835,6 +875,46 @@ export const useProvidersStore = defineStore('providers', () => {
listVoices: async () => {
return []
},
listModels: async (config: Record<string, unknown>) => {
// Filter models to only include TTS models
const apiKey = typeof config.apiKey === 'string' ? config.apiKey.trim() : ''
let baseUrl = typeof config.baseUrl === 'string' ? config.baseUrl.trim() : ''
if (!baseUrl.endsWith('/'))
baseUrl += '/'
if (!apiKey || !baseUrl) {
return []
}
const provider = await createOpenAI(apiKey, baseUrl)
if (!provider || typeof provider.model !== 'function') {
return []
}
const models = await listModels({
apiKey,
baseURL: baseUrl,
})
// Filter for TTS models - look for models with "tts" in the ID
return models
.filter((model: any) => {
const modelId = model.id.toLowerCase()
// Include models that contain "tts" in their ID
return modelId.includes('tts')
})
.map((model: any) => {
return {
id: model.id,
name: model.name || model.display_name || model.id,
provider: 'openai-compatible-audio-speech',
description: model.description || '',
contextLength: model.context_length || 0,
deprecated: false,
} satisfies ModelInfo
})
},
},
creator: createOpenAI,
}),
@@ -850,6 +930,53 @@ export const useProvidersStore = defineStore('providers', () => {
defaultBaseUrl: 'https://api.openai.com/v1/',
creator: createOpenAI,
validation: ['health'],
capabilities: {
listModels: async () => {
// OpenAI transcription models are hardcoded (no API endpoint to list them)
return [
{
id: 'gpt-4o-transcribe',
name: 'GPT-4o Transcribe',
provider: 'openai-audio-transcription',
description: 'High-quality transcription model',
contextLength: 0,
deprecated: false,
},
{
id: 'gpt-4o-mini-transcribe',
name: 'GPT-4o Mini Transcribe',
provider: 'openai-audio-transcription',
description: 'Faster, cost-effective transcription model',
contextLength: 0,
deprecated: false,
},
{
id: 'gpt-4o-mini-transcribe-2025-12-15',
name: 'GPT-4o Mini Transcribe (2025-12-15)',
provider: 'openai-audio-transcription',
description: 'GPT-4o Mini Transcribe snapshot from 2025-12-15',
contextLength: 0,
deprecated: false,
},
{
id: 'whisper-1',
name: 'Whisper-1',
provider: 'openai-audio-transcription',
description: 'Powered by our open source Whisper V2 model',
contextLength: 0,
deprecated: false,
},
{
id: 'gpt-4o-transcribe-diarize',
name: 'GPT-4o Transcribe Diarize',
provider: 'openai-audio-transcription',
description: 'Transcription with speaker diarization',
contextLength: 0,
deprecated: false,
},
] satisfies ModelInfo[]
},
},
validators: {
validateProviderConfig: (config) => {
const errors = [
@@ -880,6 +1007,14 @@ export const useProvidersStore = defineStore('providers', () => {
category: 'transcription',
tasks: ['speech-to-text', 'automatic-speech-recognition', 'asr', 'stt'],
creator: createOpenAI,
capabilities: {
// Override listModels to return empty array - transcription models cannot be fetched from /v1/models
// Users must manually enter transcription model names (e.g., whisper-1, gpt-4o-transcribe)
// The /v1/models endpoint only returns chat models, not transcription models
listModels: async () => {
return []
},
},
}),
'aliyun-nls-transcription': {
id: 'aliyun-nls-transcription',