feat: integrated to @xsai/generate-speech (#9)

* feat: integrated to @xsai/generate-speech
* chore: updated impl
* feat: done integrations
* chore: remove llm-proxy package, remove unused eslint config

package.json

@@ -31,7 +31,7 @@
     "@antfu/eslint-config": "^3.12.1",
     "@antfu/ni": "^0.23.2",
     "@cspell/dict-ru_ru": "^2.2.4",
-    "@types/node": "^22.10.3",
+    "@types/node": "^22.10.5",
     "@unocss/eslint-config": "^0.65.3",
     "@unocss/eslint-plugin": "^0.65.3",
     "bumpp": "^9.9.2",
@@ -48,7 +48,7 @@
     "typescript": "~5.7.2",
     "unbuild": "3.0.0-rc.11",
     "unocss": "^0.65.3",
-    "vite": "^6.0.6",
+    "vite": "^6.0.7",
     "vite-plugin-inspect": "^0.10.6",
     "vitest": "^2.1.8"
   },

packages/hfup/package.json

@@ -39,6 +39,6 @@
   "dependencies": {
     "defu": "^6.1.4",
     "gray-matter": "^4.0.3",
-    "vite": "^6.0.6"
+    "vite": "^6.0.7"
   }
 }

packages/llm-proxy/.gitignore

@@ -1,33 +0,0 @@
-# prod
-dist/
-
-# dev
-.yarn/
-!.yarn/releases
-.vscode/*
-!.vscode/launch.json
-!.vscode/*.code-snippets
-.idea/workspace.xml
-.idea/usage.statistics.xml
-.idea/shelf
-
-# deps
-node_modules/
-.wrangler
-
-# env
-.env
-.env.production
-.dev.vars
-
-# logs
-logs/
-*.log
-npm-debug.log*
-yarn-debug.log*
-yarn-error.log*
-pnpm-debug.log*
-lerna-debug.log*
-
-# misc
-.DS_Store

packages/llm-proxy/README.md

@@ -1,8 +0,0 @@
-```
-npm install
-npm run dev
-```
-
-```
-npm run deploy
-```

packages/llm-proxy/package.json

@@ -1,18 +0,0 @@
-{
-  "name": "@proj-airi/llm-proxy",
-  "type": "module",
-  "private": true,
-  "packageManager": "pnpm@9.15.2",
-  "scripts": {
-    "dev": "wrangler dev",
-    "deploy": "wrangler deploy --minify"
-  },
-  "dependencies": {
-    "elevenlabs": "^1.50.2",
-    "hono": "^4.6.15"
-  },
-  "devDependencies": {
-    "@cloudflare/workers-types": "^4.20241230.0",
-    "wrangler": "^3.99.0"
-  }
-}

packages/llm-proxy/src/index.ts

@@ -1,49 +0,0 @@
-import { ElevenLabsClient } from 'elevenlabs'
-import { Hono } from 'hono'
-import { env } from 'hono/adapter'
-import { cors } from 'hono/cors'
-import { stream } from 'hono/streaming'
-
-const app = new Hono()
-
-app.use(cors({
-  origin: '*',
-  allowHeaders: ['*'],
-  allowMethods: ['POST', 'HEAD'],
-  exposeHeaders: ['Content-Type', 'Transfer-Encoding', 'Content-Length', 'Date', 'Server', 'Connection', 'X-Powered-By', 'X-Request-ID'],
-  maxAge: 60 * 60 * 24 * 30, // 30 days
-}))
-
-app.post('/api/v1/llm/voice/elevenlabs', async (c) => {
-  return stream(c, async (stream) => {
-    const body = await c.req.json()
-    const apiKeyUnknown = c.req.header('Authorization')
-
-    let apiKey = apiKeyUnknown?.trim().substring('Bearer '.length)
-    const { ELEVENLABS_API_KEY } = env<{ ELEVENLABS_API_KEY?: string }>(c)
-    if (!apiKey && !!ELEVENLABS_API_KEY) {
-      apiKey = ELEVENLABS_API_KEY
-    }
-
-    const client = new ElevenLabsClient({ apiKey })
-
-    const res = await client.generate(body)
-
-    // Set headers for streaming
-    c.res.headers.append('Content-Type', 'audio/mpeg')
-    c.res.headers.append('Transfer-Encoding', 'chunked')
-
-    // Stream the response
-    await stream.pipe(new ReadableStream({
-      async pull(controller) {
-        for await (const chunk of res as unknown as Uint8Array) {
-          controller.enqueue(chunk)
-        }
-
-        controller.close()
-      },
-    }))
-  })
-})
-
-export default app

packages/llm-proxy/tsconfig.json

@@ -1,17 +0,0 @@
-{
-  "compilerOptions": {
-    "target": "ESNext",
-    "jsx": "react-jsx",
-    "jsxImportSource": "hono/jsx",
-    "lib": [
-      "ESNext"
-    ],
-    "module": "ESNext",
-    "moduleResolution": "Bundler",
-    "types": [
-      "@cloudflare/workers-types/2023-07-01"
-    ],
-    "strict": true,
-    "skipLibCheck": true
-  }
-}

packages/llm-proxy/wrangler.toml

@@ -1,28 +0,0 @@
-name = "airi-llm-proxy"
-main = "src/index.ts"
-compatibility_date = "2024-12-03"
-
-# compatibility_flags = [ "nodejs_compat" ]
-
-# [vars]
-# MY_VAR = "my-variable"
-
-# [[kv_namespaces]]
-# binding = "MY_KV_NAMESPACE"
-# id = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
-
-# [[r2_buckets]]
-# binding = "MY_BUCKET"
-# bucket_name = "my-bucket"
-
-# [[d1_databases]]
-# binding = "DB"
-# database_name = "my-database"
-# database_id = ""
-
-# [ai]
-# binding = "AI"
-
-# [observability]
-# enabled = true
-# head_sampling_rate = 1

packages/moonshine-web/package.json

@@ -18,9 +18,9 @@
     "typecheck": "vue-tsc --noEmit"
   },
   "dependencies": {
-    "@tresjs/core": "^4.3.1",
+    "@tresjs/core": "^4.3.2",
     "@unocss/reset": "^0.65.3",
-    "@vueuse/core": "^12.2.0",
+    "@vueuse/core": "^12.3.0",
     "@vueuse/motion": "^2.2.6",
     "ofetch": "^1.4.1",
     "three": "^0.172.0",

packages/stage/package.json

@@ -41,20 +41,20 @@
     "@pixiv/three-vrm-core": "^3.3.2",
     "@ricky0123/vad-web": "^0.0.22",
     "@tresjs/cientos": "^4.0.3",
-    "@tresjs/core": "^4.3.1",
+    "@tresjs/core": "^4.3.2",
     "@types/yauzl": "^2.10.3",
     "@typeschema/valibot": "^0.14.0",
     "@unhead/vue": "^1.11.14",
     "@unocss/reset": "^0.65.3",
-    "@vueuse/core": "^12.2.0",
+    "@vueuse/core": "^12.3.0",
     "@vueuse/head": "^2.0.0",
-    "@vueuse/shared": "^12.2.0",
-    "@xsai/generate-speech": "^0.0.22",
-    "@xsai/generate-text": "^0.0.22",
-    "@xsai/model": "^0.0.22",
-    "@xsai/providers": "^0.0.22",
-    "@xsai/shared-chat": "^0.0.22",
-    "@xsai/stream-text": "^0.0.22",
+    "@vueuse/shared": "^12.3.0",
+    "@xsai/generate-speech": "^0.0.23",
+    "@xsai/generate-text": "^0.0.23",
+    "@xsai/model": "^0.0.23",
+    "@xsai/providers": "^0.0.23",
+    "@xsai/shared-chat": "^0.0.23",
+    "@xsai/stream-text": "^0.0.23",
     "defu": "^6.1.4",
     "jszip": "^3.10.1",
     "nprogress": "^0.2.0",
@@ -66,7 +66,7 @@
     "rehype-stringify": "^10.0.1",
     "remark-parse": "^11.0.0",
     "remark-rehype": "^11.1.1",
-    "shiki": "^1.24.4",
+    "shiki": "^1.26.1",
     "three": "^0.172.0",
     "unified": "^11.0.5",
     "valibot": "1.0.0-beta.9",
@@ -81,13 +81,13 @@
   "devDependencies": {
     "@iconify-json/carbon": "^1.2.5",
     "@iconify-json/eos-icons": "^1.2.2",
-    "@iconify-json/lucide": "^1.2.20",
+    "@iconify-json/lucide": "^1.2.21",
     "@iconify-json/mingcute": "^1.2.3",
     "@iconify-json/solar": "^1.2.2",
     "@iconify-json/svg-spinners": "^1.2.2",
     "@intlify/unplugin-vue-i18n": "^6.0.3",
     "@proj-airi/elevenlabs": "workspace:^",
-    "@shikijs/markdown-it": "^1.24.4",
+    "@shikijs/markdown-it": "^1.26.1",
     "@types/markdown-it-link-attributes": "^3.0.5",
     "@types/nprogress": "^0.2.3",
     "@types/three": "^0.171.0",

packages/stage/src/components/Widgets/Stage.vue

@@ -1,22 +1,23 @@
 <script setup lang="ts">
 import type { Emotion } from '../../constants/emotions'
-import { storeToRefs } from 'pinia'
 
+import { generateSpeech } from '@xsai/generate-speech'
+import { createUnElevenLabs } from '@xsai/providers'
+import { storeToRefs } from 'pinia'
 import { onUnmounted, ref } from 'vue'
 import { useI18n } from 'vue-i18n'
+
 import { useMarkdown } from '../../composables/markdown'
 import { useQueue } from '../../composables/queue'
 import { useDelayMessageQueue, useEmotionsMessageQueue, useMessageContentQueue } from '../../composables/queues'
 import { llmInferenceEndToken } from '../../constants'
-import { Voice } from '../../constants/elevenlabs'
-
+import { Voice, voiceMap } from '../../constants/elevenlabs'
 import { EMOTION_EmotionMotionName_value, EMOTION_VRMExpressionName_value, EmotionThinkMotionName } from '../../constants/emotions'
 import { useAudioContext, useSpeakingStore } from '../../stores/audio'
 import { useChatStore } from '../../stores/chat'
-import { useLLM } from '../../stores/llm'
 import { useSettings } from '../../stores/settings'
-import Live2DScene from '../Scenes/Live2D.vue'
 
+import Live2DScene from '../Scenes/Live2D.vue'
 import VRMScene from '../Scenes/VRM.vue'
 
 import '../../utils/live2d-zip-loader'
@@ -27,7 +28,6 @@ const vrmViewerRef = ref<{ setExpression: (expression: string) => void }>()
 const { stageView, elevenLabsApiKey, elevenlabsVoiceEnglish, elevenlabsVoiceJapanese } = storeToRefs(useSettings())
 const { mouthOpenSize } = storeToRefs(useSpeakingStore())
 const { audioContext, calculateVolume } = useAudioContext()
-const { streamSpeech } = useLLM()
 const { onBeforeMessageComposed, onBeforeSend, onTokenLiteral, onTokenSpecial, onStreamEnd, streamingMessage } = useChatStore()
 const { process } = useMarkdown()
 const { locale } = useI18n()
@@ -71,17 +71,21 @@ const ttsQueue = useQueue<string>({
         voice = elevenlabsVoiceEnglish.value
 
       const now = Date.now()
-      const res = await streamSpeech('https://airi-api.ayaka.io', elevenLabsApiKey.value, ctx.data, {
-        // voice: 'ShanShan',
-        // Quite good for English
-        voice,
-        // Beatrice is not 'childish' like the others
-        // voice: 'Beatrice',
-        model_id: 'eleven_multilingual_v2',
-        voice_settings: {
-          stability: 0.4,
-          similarity_boost: 0.5,
-        },
+
+      const elevenlabs = createUnElevenLabs({
+        apiKey: elevenLabsApiKey.value,
+        baseURL: 'https://unspeech.hyp3r.link/v1/',
+      })
+      const res = await generateSpeech({
+        ...elevenlabs.speech({
+          model: 'elevenlabs/eleven_multilingual_v2',
+          voice: voiceMap[voice],
+          voiceSettings: {
+            stability: 0.4,
+            similarityBoost: 0.5,
+          },
+        }),
+        input: ctx.data,
       })
       const elapsed = Date.now() - now
 

packages/stage/src/constants/elevenlabs.ts

@@ -13,6 +13,8 @@ export enum Voice {
 export const voiceMap: Record<Voice, string> = {
   // English
   [Voice.Myriam]: 'lNxY9WuCBCZCISASyJ55',
+  // Beatrice is not 'childish' like the others
+  // voice: 'Beatrice',
   [Voice.Beatrice]: 'KAsXoQDshjF6ehsWa1mF',
   [Voice.Camilla_KM]: 'dLhSyo03JRp5WkGpUlz1',
   [Voice.SallySunshine]: 'qswttdunP3b44zVZKMRB',
@@ -22,6 +24,9 @@ export const voiceMap: Record<Voice, string> = {
   [Voice.Morioki]: '8EkOjt4xTPGMclNlh1pk',
 }
 
+// voice: 'ShanShan',
+// Quite good for English
+
 export const enVoiceList = [
   Voice.Myriam,
   Voice.Beatrice,

packages/stage/src/stores/llm.ts

@@ -1,8 +1,6 @@
-import type { GenerateAudioStream } from '@proj-airi/elevenlabs/types'
 import type { Message } from '@xsai/shared-chat'
 import { listModels } from '@xsai/model'
 import { streamText } from '@xsai/stream-text'
-import { ofetch } from 'ofetch'
 import { defineStore } from 'pinia'
 
 export const useLLM = defineStore('llm', () => {
@@ -38,28 +36,8 @@ export const useLLM = defineStore('llm', () => {
     }
   }
 
-  async function streamSpeech(baseUrl: string, apiKey: string, text: string, options: Omit<Omit<GenerateAudioStream, 'stream'> & { voice: string }, 'text'>) {
-    if (!text || !text.trim())
-      throw new Error('Text is required')
-
-    return await ofetch(`${baseUrl}/api/v1/llm/voice/elevenlabs`, {
-      body: {
-        ...options,
-        stream: true,
-        text,
-      },
-      method: 'POST',
-      cache: 'no-cache',
-      responseType: 'arrayBuffer',
-      headers: {
-        Authorization: `Bearer ${apiKey}`,
-      },
-    })
-  }
-
   return {
     models,
     stream,
-    streamSpeech,
   }
 })

packages/stage/src/stores/settings.ts

@@ -21,7 +21,7 @@ export const useSettings = defineStore('settings', () => {
   const selectedAudioDeviceId = computed(() => selectedAudioDevice.value?.deviceId)
   const { audioInputs } = useDevicesList({ constraints: { audio: true }, requestPermissions: true })
 
-  const elevenlabsVoiceEnglish = useLocalStorage<Voice>('settings/llm/elevenlabs/voice/en', Voice.Camilla_KM)
+  const elevenlabsVoiceEnglish = useLocalStorage<Voice>('settings/llm/elevenlabs/voice/en', Voice.Myriam)
   const elevenlabsVoiceJapanese = useLocalStorage<Voice>('settings/llm/elevenlabs/voice/ja', Voice.Morioki)
 
   watch(isAudioInputOn, (value) => {

packages/tamagotchi/package.json

@@ -19,22 +19,16 @@
     "build:linux": "npm run build:tamagotchi && electron-builder --linux"
   },
   "dependencies": {
-    "@electron-toolkit/preload": "^3.0.0",
+    "@electron-toolkit/preload": "^3.0.1",
     "@electron-toolkit/utils": "^3.0.0"
   },
   "devDependencies": {
-    "@electron-toolkit/eslint-config": "^1.0.2",
-    "@electron-toolkit/eslint-config-ts": "^2.0.0",
     "@electron-toolkit/tsconfig": "^1.0.1",
-    "@rushstack/eslint-patch": "^1.10.3",
-    "@types/node": "^20.14.8",
-    "@vitejs/plugin-vue": "^5.0.5",
-    "electron": "^31.0.2",
+    "@vitejs/plugin-vue": "^5.2.1",
+    "electron": "^31.7.6",
     "electron-builder": "^24.13.3",
     "electron-vite": "^2.3.0",
-    "typescript": "^5.5.2",
-    "vite": "^5.3.1",
-    "vue": "^3.4.30",
-    "vue-tsc": "^2.0.22"
+    "vue": "^3.5.13",
+    "vue-tsc": "^2.2.0"
   }
 }

packages/whisper-webgpu/package.json

@@ -19,7 +19,7 @@
   },
   "dependencies": {
     "@unocss/reset": "^0.65.3",
-    "@vueuse/core": "^12.2.0",
+    "@vueuse/core": "^12.3.0",
     "ofetch": "^1.4.1",
     "vue": "^3.5.13"
   },