feat: different elevenlabs voices

cspell.config.yaml

@@ -4,6 +4,7 @@ dictionaryDefinitions: []
 dictionaries: []
 words:
   - acubismmotion
+  - Aerisita
   - airi
   - airi-vtuber
   - Attributify
@@ -40,12 +41,14 @@ words:
   - hyoban
   - iconify
   - intlify
+  - Kawaii
   - kwaa
   - live2dcubismcore
   - live2dcubismframework
   - Llmmarker
   - Maru
   - micvad
+  - Morioki
   - Myriam
   - Neko
   - nekomeowww

packages/stage/locales/en.yml

@@ -57,8 +57,10 @@ settings:
     label: OpenAI API BaseURL
     placeholder: Input your API base URL
     placeholder_mobile: OpenAI API BaseURL
+  voices: Voice
 stage:
   message: Message
   select-a-audio-input: Select a Audio Input
   select-a-model: Select a model
+  select-a-voice: Choose a voice
   waiting: Waiting

packages/stage/locales/zh-CN.yml

@@ -45,8 +45,10 @@ settings:
     label: OpenAI API BaseURL
     placeholder: 输入您的 API BaseURL
     placeholder_mobile: OpenAI BaseURL
+  voices: 声线
 stage:
   message: 消息
   select-a-audio-input: 选择一个音频输入设备
   select-a-model: 选择一个模型
+  select-a-voice: 选择一个声线
   waiting: 等待中

packages/stage/src/components/Settings.vue

@@ -1,21 +1,23 @@
 <script setup lang="ts">
+import type { Voice } from '../constants/elevenlabs'
 import { useDark } from '@vueuse/core'
 import { storeToRefs } from 'pinia'
 import { onMounted, ref, watch } from 'vue'
-import { useI18n } from 'vue-i18n'
 
+import { useI18n } from 'vue-i18n'
+import { voiceList } from '../constants/elevenlabs'
 import { useLLM } from '../stores/llm'
 import { useSettings } from '../stores/settings'
 import TransitionVertical from './TransitionVertical.vue'
 
-const { t } = useI18n()
+const { t, locale } = useI18n()
 
 const settings = useSettings()
 const show = ref(false)
 const dark = useDark({ disableTransition: false })
 const supportedModels = ref<{ id: string, name?: string }[]>([])
 const { models } = useLLM()
-const { openAiModel, openAiApiBaseURL, openAiApiKey } = storeToRefs(settings)
+const { openAiModel, openAiApiBaseURL, openAiApiKey, elevenlabsVoiceEnglish, elevenlabsVoiceJapanese } = storeToRefs(settings)
 
 function handleModelChange(event: Event) {
   const target = event.target as HTMLSelectElement
@@ -28,6 +30,26 @@ function handleModelChange(event: Event) {
   openAiModel.value = found
 }
 
+function handleVoiceChange(event: Event) {
+  const value = (event.target as HTMLSelectElement).value as Voice
+  switch (locale.value) {
+    case 'en':
+    case 'en-US':
+      elevenlabsVoiceEnglish.value = value
+      break
+    case 'zh':
+    case 'zh-CN':
+    case 'zh-TW':
+    case 'zh-HK':
+      elevenlabsVoiceEnglish.value = value
+      break
+    case 'jp':
+    case 'jp-JP':
+      elevenlabsVoiceJapanese.value = value
+      break
+  }
+}
+
 watch([openAiApiBaseURL, openAiApiKey], async ([baseUrl, apiKey]) => {
   if (!baseUrl || !apiKey) {
     supportedModels.value = []
@@ -192,6 +214,32 @@ onMounted(async () => {
               </option>
             </select>
           </div>
+          <div text-sm>
+            <span>{{ t('settings.voices') }}</span>
+          </div>
+          <div flex="~ row" w-full text="sm">
+            <select
+              bg="zinc-200 dark:zinc-800/50" w-full rounded-md px-2 py-1 font-mono
+              outline-none
+              @change="handleVoiceChange"
+            >
+              <option disabled class="bg-white dark:bg-zinc-800">
+                {{ t('stage.select-a-voice') }}
+              </option>
+              <option v-if="['en', 'en-US'].indexOf(locale) !== -1 && elevenlabsVoiceEnglish" :value="elevenlabsVoiceEnglish">
+                {{ elevenlabsVoiceEnglish }}
+              </option>
+              <option v-if="['zh', 'zh-CN', 'zh-TW', 'zh-HK'].indexOf(locale) !== -1 && elevenlabsVoiceEnglish" :value="elevenlabsVoiceEnglish">
+                {{ elevenlabsVoiceEnglish }}
+              </option>
+              <option v-if="['jp', 'jp-JP'].indexOf(locale) !== -1 && elevenlabsVoiceJapanese" :value="elevenlabsVoiceJapanese">
+                {{ elevenlabsVoiceJapanese }}
+              </option>
+              <option v-for="(m, index) in voiceList[locale]" :key="index" :value="m">
+                {{ m }}
+              </option>
+            </select>
+          </div>
         </div>
       </TransitionVertical>
     </div>

packages/stage/src/components/Widgets/MobileSettings.vue

@@ -1,19 +1,21 @@
 <script setup lang="ts">
+import type { Voice } from '../../constants/elevenlabs'
 import { useDark } from '@vueuse/core'
 import { storeToRefs } from 'pinia'
 import { onMounted, ref, watch } from 'vue'
 import { useI18n } from 'vue-i18n'
 
+import { voiceList } from '../../constants/elevenlabs'
 import { useLLM } from '../../stores/llm'
 import { useSettings } from '../../stores/settings'
 
-const { t } = useI18n()
+const { t, locale } = useI18n()
 
 const settings = useSettings()
 const dark = useDark({ disableTransition: false })
 const supportedModels = ref<{ id: string, name?: string }[]>([])
 const { models } = useLLM()
-const { openAiModel, openAiApiBaseURL, openAiApiKey } = storeToRefs(settings)
+const { openAiModel, openAiApiBaseURL, openAiApiKey, elevenlabsVoiceEnglish, elevenlabsVoiceJapanese } = storeToRefs(settings)
 
 function handleModelChange(event: Event) {
   const target = event.target as HTMLSelectElement
@@ -31,6 +33,26 @@ function handleViewChange(event: Event) {
   settings.stageView = target.value
 }
 
+function handleVoiceChange(event: Event) {
+  const value = (event.target as HTMLSelectElement).value as Voice
+  switch (locale.value) {
+    case 'en':
+    case 'en-US':
+      elevenlabsVoiceEnglish.value = value
+      break
+    case 'zh':
+    case 'zh-CN':
+    case 'zh-TW':
+    case 'zh-HK':
+      elevenlabsVoiceEnglish.value = value
+      break
+    case 'jp':
+    case 'jp-JP':
+      elevenlabsVoiceJapanese.value = value
+      break
+  }
+}
+
 watch([openAiApiBaseURL, openAiApiKey], async ([baseUrl, apiKey]) => {
   if (!baseUrl || !apiKey) {
     supportedModels.value = []
@@ -53,7 +75,7 @@ onMounted(async () => {
     <h2 text="slate-800/80 dark:slate-200/80 xl" font-bold>
       Settings
     </h2>
-    <div flex="~" gap-2>
+    <div>
       <div
         grid="~ cols-[140px_1fr]" my-2 items-center gap-1.5 rounded-lg
         bg="[#fff6fc] dark:[#2c2529]" px-2 py-1 text="pink-400"
@@ -66,7 +88,7 @@ onMounted(async () => {
             v-model="settings.openAiApiBaseURL"
             type="text"
             :placeholder="t('settings.openai-base-url.placeholder_mobile')"
-            w-full rounded-md bg-transparent px-2 py-1 text-right font-mono outline-none
+            h-8 w-full rounded-md bg-transparent px-2 py-1 text-right font-mono outline-none
           >
         </div>
         <div text="sm pink-500">
@@ -77,7 +99,7 @@ onMounted(async () => {
             v-model="settings.openAiApiKey"
             type="text"
             :placeholder="t('settings.openai-api-key.placeholder_mobile')"
-            w-full rounded-md bg-transparent px-2 py-1 text-right font-mono outline-none
+            h-8 w-full rounded-md bg-transparent px-2 py-1 text-right font-mono outline-none
           >
         </div>
         <div text="sm pink-500">
@@ -88,7 +110,7 @@ onMounted(async () => {
             v-model="settings.elevenLabsApiKey"
             type="text"
             :placeholder="t('settings.elevenlabs-api-key.placeholder_mobile')"
-            w-full rounded-md bg-transparent px-2 py-1 text-right font-mono outline-none
+            h-8 w-full rounded-md bg-transparent px-2 py-1 text-right font-mono outline-none
           >
         </div>
         <div text="sm pink-500">
@@ -97,7 +119,7 @@ onMounted(async () => {
         <div flex="~ row" w-full text="sm">
           <select
             v-model="settings.language"
-            w-full rounded-md bg-transparent px-2 py-1 text-right font-mono outline-none
+            h-8 w-full rounded-md bg-transparent px-2 py-1 text-right font-mono outline-none
           >
             <option value="en-US">
               English
@@ -112,7 +134,7 @@ onMounted(async () => {
         </div>
         <div flex="~ row" w-full text="sm">
           <select
-            w-full rounded-md bg-transparent px-2 py-1 font-mono outline-none
+            h-8 w-full rounded-md bg-transparent px-2 py-1 text-right font-mono outline-none
             @change="handleModelChange"
           >
             <option disabled class="bg-white dark:bg-zinc-800">
@@ -126,6 +148,32 @@ onMounted(async () => {
             </option>
           </select>
         </div>
+        <div text="sm pink-500">
+          <span>{{ t('settings.voices') }}</span>
+        </div>
+        <div flex="~ row" w-full text="sm">
+          <select
+            h-8 w-full rounded-md bg-transparent px-2 py-1 text-right font-mono outline-none
+            @change="handleVoiceChange"
+          >
+            <option disabled class="bg-white dark:bg-zinc-800">
+              {{ t('stage.select-a-voice') }}
+            </option>
+            <option v-if="['en', 'en-US'].indexOf(locale) !== -1 && elevenlabsVoiceEnglish" :value="elevenlabsVoiceEnglish">
+              {{ elevenlabsVoiceEnglish }}
+            </option>
+            <!-- TODO -->
+            <option v-if="['zh', 'zh-CN', 'zh-TW', 'zh-HK'].indexOf(locale) !== -1 && elevenlabsVoiceEnglish" :value="elevenlabsVoiceEnglish">
+              {{ elevenlabsVoiceEnglish }}
+            </option>
+            <option v-if="['jp', 'jp-JP'].indexOf(locale) !== -1 && elevenlabsVoiceJapanese" :value="elevenlabsVoiceJapanese">
+              {{ elevenlabsVoiceJapanese }}
+            </option>
+            <option v-for="(m, index) in voiceList[locale]" :key="index" :value="m">
+              {{ m }}
+            </option>
+          </select>
+        </div>
       </div>
     </div>
     <h2 text="slate-800/80 dark:slate-200/80 xl" font-bold>
@@ -140,7 +188,7 @@ onMounted(async () => {
           <span>Viewer</span>
         </div>
         <select
-          w-full rounded-md bg-transparent px-2 py-1 text-right font-mono outline-none
+          h-8 w-full rounded-md bg-transparent px-2 py-1 text-right font-mono outline-none
           @change="handleViewChange"
         >
           <option value="2d">
@@ -153,9 +201,7 @@ onMounted(async () => {
         <div text="sm pink-500">
           <span>Theme</span>
         </div>
-        <label
-          cursor-pointer px-2 py-1
-        >
+        <label h-8 flex cursor-pointer items-center justify-end>
           <input
             v-model="dark"
             :checked="dark"
@@ -164,10 +210,10 @@ onMounted(async () => {
             type="checkbox"
             hidden appearance-none outline-none
           >
-          <div flex select-none justify-end>
+          <div select-none>
             <Transition name="slide-away" mode="out-in">
-              <div v-if="dark" i-solar:sun-fog-bold-duotone text="base hover:pink-600 dark:hover:pink-300" transition="all ease-in-out duration-250" />
-              <div v-else i-solar:moon-stars-bold-duotone text="base hover:pink-600 dark:hover:pink-300" transition="all ease-in-out duration-250" />
+              <div v-if="dark" i-solar:sun-fog-bold-duotone text="lg hover:pink-600 dark:hover:pink-300" transition="all ease-in-out duration-250" />
+              <div v-else i-solar:moon-stars-bold-duotone text="lg hover:pink-600 dark:hover:pink-300" transition="all ease-in-out duration-250" />
             </Transition>
           </div>
         </label>

packages/stage/src/components/Widgets/Stage.vue

@@ -3,14 +3,16 @@ import type { Emotion } from '../../constants/emotions'
 import { storeToRefs } from 'pinia'
 
 import { onUnmounted, ref } from 'vue'
+import { useI18n } from 'vue-i18n'
 import { useMarkdown } from '../../composables/markdown'
 import { useQueue } from '../../composables/queue'
 import { useDelayMessageQueue, useEmotionsMessageQueue, useMessageContentQueue } from '../../composables/queues'
 import { llmInferenceEndToken } from '../../constants'
+import { Voice } from '../../constants/elevenlabs'
 import { EMOTION_EmotionMotionName_value, EMOTION_VRMExpressionName_value, EmotionThinkMotionName } from '../../constants/emotions'
+
 import { useAudioContext, useSpeakingStore } from '../../stores/audio'
 import { useChatStore } from '../../stores/chat'
-
 import { useLLM } from '../../stores/llm'
 import { useSettings } from '../../stores/settings'
 import Live2DScene from '../Scenes/Live2D.vue'
@@ -19,12 +21,13 @@ import VRMScene from '../Scenes/VRM.vue'
 const live2DViewerRef = ref<{ setMotion: (motionName: string) => Promise<void> }>()
 const vrmViewerRef = ref<{ setExpression: (expression: string) => void }>()
 
-const { stageView, elevenLabsApiKey } = storeToRefs(useSettings())
+const { stageView, elevenLabsApiKey, elevenlabsVoiceEnglish, elevenlabsVoiceJapanese } = storeToRefs(useSettings())
 const { mouthOpenSize } = storeToRefs(useSpeakingStore())
 const { audioContext, calculateVolume } = useAudioContext()
 const { streamSpeech } = useLLM()
 const { onBeforeMessageComposed, onBeforeSend, onTokenLiteral, onTokenSpecial, onStreamEnd, streamingMessage } = useChatStore()
 const { process } = useMarkdown()
+const { locale } = useI18n()
 
 const audioAnalyser = ref<AnalyserNode>()
 const nowSpeaking = ref(false)
@@ -58,11 +61,17 @@ const audioQueue = useQueue<{ audioBuffer: AudioBuffer, text: string }>({
 const ttsQueue = useQueue<string>({
   handlers: [
     async (ctx) => {
+      let voice = Voice.Camilla_KM
+      if (locale.value === 'jp' || locale.value === 'jp-JP')
+        voice = elevenlabsVoiceJapanese.value
+      else
+        voice = elevenlabsVoiceEnglish.value
+
       const now = Date.now()
       const res = await streamSpeech('https://airi-api.ayaka.io', elevenLabsApiKey.value, ctx.data, {
         // voice: 'ShanShan',
         // Quite good for English
-        voice: 'Myriam',
+        voice,
         // Beatrice is not 'childish' like the others
         // voice: 'Beatrice',
         model_id: 'eleven_multilingual_v2',

packages/stage/src/constants/elevenlabs.ts

@@ -0,0 +1,43 @@
+export enum Voice {
+  // English
+  Myriam = 'Myriam',
+  Beatrice = 'Beatrice',
+  Camilla_KM = 'Camilla_KM',
+  SallySunshine = 'Sally Sunshine',
+  Annie = 'Annie',
+  KawaiiAerisita = 'Kawaii Aerisita',
+  // Japanese
+  Morioki = 'Morioki',
+}
+
+export const voiceMap: Record<Voice, string> = {
+  // English
+  [Voice.Myriam]: 'lNxY9WuCBCZCISASyJ55',
+  [Voice.Beatrice]: 'KAsXoQDshjF6ehsWa1mF',
+  [Voice.Camilla_KM]: 'dLhSyo03JRp5WkGpUlz1',
+  [Voice.SallySunshine]: 'qswttdunP3b44zVZKMRB',
+  [Voice.Annie]: 'AfA1PA0ldViH0DA6pbml',
+  [Voice.KawaiiAerisita]: 'vGQNBgLaiM3EdZtxIiuY',
+  // Japanese
+  [Voice.Morioki]: '8EkOjt4xTPGMclNlh1pk',
+}
+
+export const enVoiceList = [
+  Voice.Myriam,
+  Voice.Beatrice,
+  Voice.Camilla_KM,
+  Voice.SallySunshine,
+  Voice.Annie,
+  Voice.KawaiiAerisita,
+]
+
+export const jaVoiceList = [
+  Voice.Morioki,
+]
+
+export const voiceList: Record<string, Voice[]> = {
+  'en': enVoiceList,
+  'en-US': enVoiceList,
+  'ja': jaVoiceList,
+  'ja-JP': jaVoiceList,
+}

packages/stage/src/stores/chat.ts

@@ -1,4 +1,4 @@
-import type { AssistantMessage, Message } from '@xsai/shared-chat'
+import { assistant, type AssistantMessage, type Message, user } from '@xsai/shared-chat'
 
 import { defineStore, storeToRefs } from 'pinia'
 import { ref } from 'vue'
@@ -55,6 +55,8 @@ export const useChatStore = defineStore('chat', () => {
       t('prompt.prefix'),
       t('prompt.suffix'),
     ),
+    user('abcd'),
+    assistant('efgh'),
   ])
 
   const streamingMessage = ref<AssistantMessage>({ role: 'assistant', content: '' })

packages/stage/src/stores/settings.ts

@@ -1,22 +1,29 @@
 import { useDevicesList, useLocalStorage } from '@vueuse/core'
 import { defineStore } from 'pinia'
 import { computed, onMounted, ref, watch } from 'vue'
+
+import { Voice } from '../constants/elevenlabs'
 import { i18n } from '../modules/i18n'
 
 export const useSettings = defineStore('settings', () => {
   const selectedAudioDevice = ref<MediaDeviceInfo>()
 
+  const language = useLocalStorage('settings/language', 'en-US')
+  const stageView = useLocalStorage('settings/stage/view/model-renderer', '2d')
+
   const openAiApiKey = useLocalStorage('settings/credentials/openai-api-key', '')
   const openAiApiBaseURL = useLocalStorage('settings/credentials/openai-api-base-url', '')
   const elevenLabsApiKey = useLocalStorage('settings/credentials/elevenlabs-api-key', '')
 
-  const language = useLocalStorage('settings/language', 'en-US')
-  const stageView = useLocalStorage('settings/stage/view/model-renderer', '2d')
   const openAiModel = useLocalStorage<{ id: string, name?: string }>('settings/llm/openai/model', { id: 'openai/gpt-3.5-turbo', name: 'OpenAI GPT3.5 Turbo' })
+
   const isAudioInputOn = useLocalStorage('settings/audio/input', 'true')
   const selectedAudioDeviceId = computed(() => selectedAudioDevice.value?.deviceId)
   const { audioInputs } = useDevicesList({ constraints: { audio: true }, requestPermissions: true })
 
+  const elevenlabsVoiceEnglish = useLocalStorage<Voice>('settings/llm/elevenlabs/voice/en', Voice.Camilla_KM)
+  const elevenlabsVoiceJapanese = useLocalStorage<Voice>('settings/llm/elevenlabs/voice/ja', Voice.Morioki)
+
   watch(isAudioInputOn, (value) => {
     if (value === 'false') {
       selectedAudioDevice.value = undefined
@@ -50,5 +57,7 @@ export const useSettings = defineStore('settings', () => {
     isAudioInputOn,
     selectedAudioDevice,
     selectedAudioDeviceId,
+    elevenlabsVoiceEnglish,
+    elevenlabsVoiceJapanese,
   }
 })

packages/stage/src/typed-router.d.ts

@@ -18,14 +18,5 @@ declare module 'vue-router/auto-routes' {
    * Route name map generated by unplugin-vue-router
    */
   export interface RouteNamedMap {
-    '/': RouteRecordInfo<'/', '/', Record<never, never>, Record<never, never>>,
-    '/[...all]': RouteRecordInfo<'/[...all]', '/:all(.*)', { all: ParamValue<true> }, { all: ParamValue<false> }>,
-    '/audio': RouteRecordInfo<'/audio', '/audio', Record<never, never>, Record<never, never>>,
-    '/devtools/image': RouteRecordInfo<'/devtools/image', '/devtools/image', Record<never, never>, Record<never, never>>,
-    '/queue': RouteRecordInfo<'/queue', '/queue', Record<never, never>, Record<never, never>>,
-    '/test/filter-message': RouteRecordInfo<'/test/filter-message', '/test/filter-message', Record<never, never>, Record<never, never>>,
-    '/test/queues/delays': RouteRecordInfo<'/test/queues/delays', '/test/queues/delays', Record<never, never>, Record<never, never>>,
-    '/test/queues/emotions': RouteRecordInfo<'/test/queues/emotions', '/test/queues/emotions', Record<never, never>, Record<never, never>>,
-    '/test/queues/messages': RouteRecordInfo<'/test/queues/messages', '/test/queues/messages', Record<never, never>, Record<never, never>>,
   }
 }