feat: dedicated llm proxy serivce for tts

2024-12-03 17:59:56 +08:00
parent 0984c40728
commit f9ddf9af93
17 changed files with 2352 additions and 273 deletions
@@ -4,6 +4,7 @@ dictionaryDefinitions: []
 dictionaries: []
 words:
  - acubismmotion
+  - airi
  - airi-vtuber
  - Attributify
  - Ayaka
@@ -21,9 +22,11 @@ words:
  - cubismmotionqueuemanager
  - cubismusermodel
  - cubismviewmatrix
+  - demi
  - elevenlabs
  - hiyori
  - iconify
+  - intlify
  - kwaa
  - live2dcubismcore
  - live2dcubismframework
@@ -46,6 +49,7 @@ words:
  - taze
  - unhead
  - unocss
+  - unplugin
  - vueuse
 ignoreWords: []
 import: []
@@ -16,6 +16,7 @@
    "dev": "pnpm packages:dev",
    "build": "pnpm packages:build",
    "packages:dev": "pnpm -r --filter=./packages/* run dev",
+    "packages:stub": "pnpm -r --filter=./packages/* run stub",
    "packages:build": "pnpm -r --filter=./packages/* run build",
    "test": "vitest --coverage",
    "test:run": "vitest run",
@@ -42,6 +43,7 @@
    "simple-git-hooks": "^2.11.1",
    "taze": "^0.18.0",
    "typescript": "~5.7.2",
+    "unbuild": "3.0.0-rc.11",
    "unocss": "^0.65.0",
    "vite": "^6.0.2",
    "vite-plugin-inspect": "^0.10.3",
@@ -0,0 +1,37 @@
+{
+  "name": "@airi-proj/elevenlabs",
+  "type": "module",
+  "private": false,
+  "description": "Simple wrapper of types of Elevenlabs's SDK",
+  "author": {
+    "name": "Neko Ayaka",
+    "email": "neko@ayaka.moe",
+    "url": "https://github.com/nekomeowww"
+  },
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/nekomeowww/airi-vtuber.git",
+    "directory": "packages/types-elevenlabs"
+  },
+  "sideEffects": false,
+  "exports": {
+    "./types": {
+      "types": "./dist/types/index.d.ts"
+    }
+  },
+  "types": "./dist/types/index.d.ts",
+  "files": [
+    "README.md",
+    "dist",
+    "package.json"
+  ],
+  "scripts": {
+    "dev": "concurrently \"pnpm run stub\"",
+    "stub": "unbuild --stub",
+    "build": "unbuild"
+  },
+  "devDependencies": {
+    "elevenlabs": "^0.18.1"
+  }
+}
@@ -0,0 +1,209 @@
+export type OptimizeStreamingLatency =
+/**
+ * Default mode (no latency optimizations)
+ */
+  '0'
+/**
+ * Normal latency optimizations (about 50% of possible latency improvement of option 3)
+ */
+  | '1'
+/**
+ * Strong latency optimizations (about 75% of possible latency improvement of option 3)
+ */
+  | '2'
+/**
+ * Max latency optimizations
+ */
+  | '3'
+/**
+ * Max latency optimizations, but also with text normalizer turned off for even more latency savings (best latency, but can mispronounce eg numbers and dates).
+ */
+  | '4'
+
+export type OutputFormat =
+/**
+ * Output format, mp3 with 22.05kHz sample rate at 32kbps
+ */
+  'mp3_22050_32'
+/**
+ * Output format, mp3 with 44.1kHz sample rate at 32kbps
+ */
+  | 'mp3_44100_32'
+/**
+ * Output format, mp3 with 44.1kHz sample rate at 64kbps
+ */
+  | 'mp3_44100_64'
+/**
+ * Output format, mp3 with 44.1kHz sample rate at 96kbps
+ */
+  | 'mp3_44100_96'
+/**
+ * Default output format, mp3 with 44.1kHz sample rate at 128kbps
+ */
+  | 'mp3_44100_128'
+/**
+ * Output format, mp3 with 44.1kHz sample rate at 192kbps.
+ */
+  | 'mp3_44100_192'
+/**
+ * PCM format (S16LE) with 16kHz sample rate.
+ */
+  | 'pcm_16000'
+/**
+ * PCM format (S16LE) with 22.05kHz sample rate.
+ */
+  | 'pcm_22050'
+/**
+ * PCM format (S16LE) with 24kHz sample rate.
+ */
+  | 'pcm_24000'
+/**
+ * PCM format (S16LE) with 44.1kHz sample rate. Requires you to be subscribed to Independent Publisher tier or above.
+ */
+  | 'pcm_44100'
+/**
+ * μ-law format (sometimes written mu-law, often approximated as u-law) with 8kHz sample rate. Note that this format is commonly used for Twilio audio inputs.
+ */
+  | 'ulaw_8000'
+
+export interface VoiceSettings {
+  stability?: number
+  similarity_boost?: number
+  style?: number
+  use_speaker_boost?: boolean
+}
+
+export interface PronunciationDictionaryVersionLocator {
+  pronunciation_dictionary_id: string
+  version_id: string
+}
+
+/**
+ * This parameter controls text normalization with three modes: 'auto', 'on', and 'off'. When set to 'auto', the system will automatically decide whether to apply text normalization (e.g., spelling out numbers). With 'on', text normalization will always be applied, while with 'off', it will be skipped. Cannot be turned on for 'eleven_turbo_v2_5' model.
+ */
+export type BodyTextToSpeechV1TextToSpeechVoiceIdPostApplyTextNormalization = 'auto' | 'on' | 'off'
+
+/**
+ * This parameter controls text normalization with three modes: 'auto', 'on', and 'off'. When set to 'auto', the system will automatically decide whether to apply text normalization (e.g., spelling out numbers). With 'on', text normalization will always be applied, while with 'off', it will be skipped. Cannot be turned on for 'eleven_turbo_v2_5' model.
+ */
+export type BodyTextToSpeechStreamingWithTimestampsV1TextToSpeechVoiceIdStreamWithTimestampsPostApplyTextNormalization = 'auto' | 'on' | 'off'
+
+export interface TextToSpeechRequest {
+  /**
+   * When enable_logging is set to false full privacy mode will be used for the request. This will mean history features are unavailable for this request, including request stitching. Full privacy mode may only be used by enterprise customers.
+   */
+  enable_logging?: boolean
+  /**
+   * You can turn on latency optimizations at some cost of quality. The best possible final latency varies by model.
+   */
+  optimize_streaming_latency?: OptimizeStreamingLatency
+  /**
+   * The output format of the generated audio.
+   */
+  output_format?: OutputFormat
+  /** The text that will get converted into speech. */
+  text: string
+  /** Identifier of the model that will be used, you can query them using GET /v1/models. The model needs to have support for text to speech, you can check this using the can_do_text_to_speech property. */
+  model_id?: string
+  /** Language code (ISO 639-1) used to enforce a language for the model. Currently only Turbo v2.5 supports language enforcement. For other models, an error will be returned if language code is provided. */
+  language_code?: string
+  /** Voice settings overriding stored setttings for the given voice. They are applied only on the given request. */
+  voice_settings?: VoiceSettings
+  /** A list of pronunciation dictionary locators (id, version_id) to be applied to the text. They will be applied in order. You may have up to 3 locators per request */
+  pronunciation_dictionary_locators?: PronunciationDictionaryVersionLocator[]
+  /** If specified, our system will make a best effort to sample deterministically, such that repeated requests with the same seed and parameters should return the same result. Determinism is not guaranteed. Must be integer between 0 and 4294967295. */
+  seed?: number
+  /** The text that came before the text of the current request. Can be used to improve the flow of prosody when concatenating together multiple generations or to influence the prosody in the current generation. */
+  previous_text?: string
+  /** The text that comes after the text of the current request. Can be used to improve the flow of prosody when concatenating together multiple generations or to influence the prosody in the current generation. */
+  next_text?: string
+  /** A list of request_id of the samples that were generated before this generation. Can be used to improve the flow of prosody when splitting up a large task into multiple requests. The results will be best when the same model is used across the generations. In case both previous_text and previous_request_ids is send, previous_text will be ignored. A maximum of 3 request_ids can be send. */
+  previous_request_ids?: string[]
+  /** A list of request_id of the samples that were generated before this generation. Can be used to improve the flow of prosody when splitting up a large task into multiple requests. The results will be best when the same model is used across the generations. In case both next_text and next_request_ids is send, next_text will be ignored. A maximum of 3 request_ids can be send. */
+  next_request_ids?: string[]
+  /** If true, we won't use PVC version of the voice for the generation but the IVC version. This is a temporary workaround for higher latency in PVC versions. */
+  use_pvc_as_ivc?: boolean
+  /** This parameter controls text normalization with three modes: 'auto', 'on', and 'off'. When set to 'auto', the system will automatically decide whether to apply text normalization (e.g., spelling out numbers). With 'on', text normalization will always be applied, while with 'off', it will be skipped. Cannot be turned on for 'eleven_turbo_v2_5' model. */
+  apply_text_normalization?: BodyTextToSpeechV1TextToSpeechVoiceIdPostApplyTextNormalization
+}
+
+/**
+ * @example
+ *     {
+ *         optimize_streaming_latency: "0",
+ *         output_format: "mp3_22050_32",
+ *         text: "It sure does, Jackie\u2026 My mama always said: \u201CIn Carolina, the air's so thick you can wear it!\u201D",
+ *         voice_settings: {
+ *             stability: 0.1,
+ *             similarity_boost: 0.3,
+ *             style: 0.2
+ *         }
+ *     }
+ */
+export interface StreamTextToSpeechRequest {
+  /**
+   * When enable_logging is set to false full privacy mode will be used for the request. This will mean history features are unavailable for this request, including request stitching. Full privacy mode may only be used by enterprise customers.
+   */
+  enable_logging?: boolean
+  /**
+   * You can turn on latency optimizations at some cost of quality. The best possible final latency varies by model.
+   */
+  optimize_streaming_latency?: OptimizeStreamingLatency
+  /**
+   * The output format of the generated audio.
+   */
+  output_format?: OutputFormat
+  /** The text that will get converted into speech. */
+  text: string
+  /** Identifier of the model that will be used, you can query them using GET /v1/models. The model needs to have support for text to speech, you can check this using the can_do_text_to_speech property. */
+  model_id?: string
+  /** Language code (ISO 639-1) used to enforce a language for the model. Currently only Turbo v2.5 supports language enforcement. For other models, an error will be returned if language code is provided. */
+  language_code?: string
+  /** Voice settings overriding stored setttings for the given voice. They are applied only on the given request. */
+  voice_settings?: VoiceSettings
+  /** A list of pronunciation dictionary locators (id, version_id) to be applied to the text. They will be applied in order. You may have up to 3 locators per request */
+  pronunciation_dictionary_locators?: PronunciationDictionaryVersionLocator[]
+  /** If specified, our system will make a best effort to sample deterministically, such that repeated requests with the same seed and parameters should return the same result. Determinism is not guaranteed. Must be integer between 0 and 4294967295. */
+  seed?: number
+  /** The text that came before the text of the current request. Can be used to improve the flow of prosody when concatenating together multiple generations or to influence the prosody in the current generation. */
+  previous_text?: string
+  /** The text that comes after the text of the current request. Can be used to improve the flow of prosody when concatenating together multiple generations or to influence the prosody in the current generation. */
+  next_text?: string
+  /** A list of request_id of the samples that were generated before this generation. Can be used to improve the flow of prosody when splitting up a large task into multiple requests. The results will be best when the same model is used across the generations. In case both previous_text and previous_request_ids is send, previous_text will be ignored. A maximum of 3 request_ids can be send. */
+  previous_request_ids?: string[]
+  /** A list of request_id of the samples that were generated before this generation. Can be used to improve the flow of prosody when splitting up a large task into multiple requests. The results will be best when the same model is used across the generations. In case both next_text and next_request_ids is send, next_text will be ignored. A maximum of 3 request_ids can be send. */
+  next_request_ids?: string[]
+  /** If true, we won't use PVC version of the voice for the generation but the IVC version. This is a temporary workaround for higher latency in PVC versions. */
+  use_pvc_as_ivc?: boolean
+  /** This parameter controls text normalization with three modes: 'auto', 'on', and 'off'. When set to 'auto', the system will automatically decide whether to apply text normalization (e.g., spelling out numbers). With 'on', text normalization will always be applied, while with 'off', it will be skipped. Cannot be turned on for 'eleven_turbo_v2_5' model. */
+  apply_text_normalization?: BodyTextToSpeechStreamingWithTimestampsV1TextToSpeechVoiceIdStreamWithTimestampsPostApplyTextNormalization
+}
+
+export interface GenerateAudioBulk extends TextToSpeechRequest {
+}
+
+export interface GenerateAudioStream extends StreamTextToSpeechRequest {
+  stream: true
+}
+
+/**
+ * Generates audio for a voice.
+ *
+ * @example Generate the entire audio
+ * import { play } from "elevenlabs";
+ *
+ * const audio = eleven.generate({
+ *   voiceId: "George" // defaults to Bella
+ * })
+ * await play(audio);
+ *
+ * @example
+ * import { stream } from "elevenlabs"
+ *
+ * const audioStream = eleven.generate({
+ *   stream: true,
+ *   voice: "Bella"
+ * })
+ * await stream(audioStream);
+ */
+export type GenerateOptions = (GenerateAudioBulk & { voice?: string }) | (GenerateAudioStream & { voice?: string })
@@ -0,0 +1,16 @@
+{
+  "compilerOptions": {
+    "target": "ESNext",
+    "lib": [
+      "ESNext"
+    ],
+    "module": "ESNext",
+    "moduleResolution": "bundler",
+    "esModuleInterop": true,
+    "forceConsistentCasingInFileNames": true,
+    "isolatedModules": true,
+    "verbatimModuleSyntax": true,
+    "skipLibCheck": true
+
+  }
+}
@@ -0,0 +1,33 @@
+# prod
+dist/
+
+# dev
+.yarn/
+!.yarn/releases
+.vscode/*
+!.vscode/launch.json
+!.vscode/*.code-snippets
+.idea/workspace.xml
+.idea/usage.statistics.xml
+.idea/shelf
+
+# deps
+node_modules/
+.wrangler
+
+# env
+.env
+.env.production
+.dev.vars
+
+# logs
+logs/
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+pnpm-debug.log*
+lerna-debug.log*
+
+# misc
+.DS_Store
@@ -0,0 +1,8 @@
+```
+npm install
+npm run dev
+```
+
+```
+npm run deploy
+```
@@ -0,0 +1,18 @@
+{
+  "name": "@airi-proj/llm-proxy",
+  "type": "module",
+  "private": true,
+  "packageManager": "pnpm@9.14.4",
+  "scripts": {
+    "dev": "wrangler dev",
+    "deploy": "wrangler deploy --minify"
+  },
+  "dependencies": {
+    "elevenlabs": "^0.18.1",
+    "hono": "^4.6.12"
+  },
+  "devDependencies": {
+    "@cloudflare/workers-types": "^4.20241112.0",
+    "wrangler": "^3.88.0"
+  }
+}
@@ -0,0 +1,44 @@
+import { ElevenLabsClient } from 'elevenlabs'
+import { Hono } from 'hono'
+import { cors } from 'hono/cors'
+import { stream } from 'hono/streaming'
+
+const app = new Hono()
+
+app.use(cors({
+  origin: '*',
+  allowHeaders: ['Referer', 'Origin', 'Content-Type', 'Accept', 'Authorization', 'X-Requested-With', 'X-HTTP-Method-Override', 'X-Forwarded-For', 'X-Real-IP'],
+  allowMethods: ['POST', 'HEAD'],
+  exposeHeaders: ['Content-Type', 'Transfer-Encoding', 'Content-Length', 'Date', 'Server', 'Connection', 'X-Powered-By', 'X-Request-ID'],
+  maxAge: 60 * 60 * 24 * 30, // 30 days
+}))
+
+app.post('/api/v1/llm/voice/elevenlabs', async (c) => {
+  return stream(c, async (stream) => {
+    const body = await c.req.json()
+    const apiKey = c.req.header('Authorization')
+
+    const client = new ElevenLabsClient({
+      apiKey: apiKey?.trim().substring('Bearer '.length),
+    })
+
+    const res = await client.generate(body)
+
+    // Set headers for streaming
+    c.res.headers.append('Content-Type', 'audio/mpeg')
+    c.res.headers.append('Transfer-Encoding', 'chunked')
+
+    // Stream the response
+    await stream.pipe(new ReadableStream({
+      async pull(controller) {
+        for await (const chunk of res as unknown as Uint8Array) {
+          controller.enqueue(chunk)
+        }
+
+        controller.close()
+      },
+    }))
+  })
+})
+
+export default app
@@ -0,0 +1,17 @@
+{
+  "compilerOptions": {
+    "target": "ESNext",
+    "jsx": "react-jsx",
+    "jsxImportSource": "hono/jsx",
+    "lib": [
+      "ESNext"
+    ],
+    "module": "ESNext",
+    "moduleResolution": "Bundler",
+    "types": [
+      "@cloudflare/workers-types/2023-07-01"
+    ],
+    "strict": true,
+    "skipLibCheck": true
+  }
+}
@@ -0,0 +1,28 @@
+name = "llm-proxy"
+main = "src/index.ts"
+compatibility_date = "2024-12-03"
+
+# compatibility_flags = [ "nodejs_compat" ]
+
+# [vars]
+# MY_VAR = "my-variable"
+
+# [[kv_namespaces]]
+# binding = "MY_KV_NAMESPACE"
+# id = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
+
+# [[r2_buckets]]
+# binding = "MY_BUCKET"
+# bucket_name = "my-bucket"
+
+# [[d1_databases]]
+# binding = "DB"
+# database_name = "my-database"
+# database_id = ""
+
+# [ai]
+# binding = "AI"
+
+# [observability]
+# enabled = true
+# head_sampling_rate = 1
@@ -1,6 +1,6 @@
 [build]
 publish = "packages/stage/dist"
-command = "pnpm run build"
+command = "pnpm run packages:stub && pnpm run build"

 [build.environment]
 NODE_VERSION = "22"
@@ -1,5 +1,5 @@
 {
-  "name": "@airi-vtuber/stage",
+  "name": "@airi-proj/stage",
  "type": "module",
  "private": true,
  "packageManager": "pnpm@9.14.4",
@@ -55,6 +55,7 @@
    "zod": "^3.23.8"
  },
  "devDependencies": {
+    "@airi-proj/elevenlabs": "workspace:^",
    "@iconify-json/carbon": "^1.2.4",
    "@intlify/unplugin-vue-i18n": "^6.0.0",
    "@shikijs/markdown-it": "^1.24.0",
@@ -21,7 +21,7 @@ import {
 } from '../composables/queues'
 import { llmInferenceEndToken } from '../constants'
 import {
-  EMOTION_EmotioMotionName_value,
+  EMOTION_EmotionMotionName_value,
  EmotionThinkMotionName,
 } from '../constants/emotions'
 import SystemPromptV2 from '../constants/prompts/system-v2'
@@ -41,7 +41,7 @@ const elevenLabsApiKey = useLocalStorage('elevenlabs-api-key', '')

 const {
  setupOpenAI,
-  // streamSpeech,
+  streamSpeech,
  stream,
  models,
 } = useLLM()
@@ -79,46 +79,56 @@ function handleModelChange(event: Event) {
  openAIModel.value = found
 }

-// const audioQueue = useQueue<{ audioBuffer: AudioBuffer, text: string }>({
-//   handlers: [
-//     (ctx) => {
-//       return new Promise((resolve) => {
-//         // Create an AudioBufferSourceNode
-//         const source = audioContext.createBufferSource()
-//         source.buffer = ctx.data.audioBuffer
+const audioQueue = useQueue<{ audioBuffer: AudioBuffer, text: string }>({
+  handlers: [
+    (ctx) => {
+      return new Promise((resolve) => {
+        // Create an AudioBufferSourceNode
+        const source = audioContext.createBufferSource()
+        source.buffer = ctx.data.audioBuffer

-//         // Connect the source to the AudioContext's destination (the speakers)
-//         source.connect(audioContext.destination)
-//         // Connect the source to the analyzer
-//         source.connect(audioAnalyser.value!)
+        // Connect the source to the AudioContext's destination (the speakers)
+        source.connect(audioContext.destination)
+        // Connect the source to the analyzer
+        source.connect(audioAnalyser.value!)

-//         // Start playing the audio
-//         nowSpeaking.value = true
-//         source.start(0)
-//         source.onended = () => {
-//           nowSpeaking.value = false
-//           resolve()
-//         }
-//       })
-//     },
-//   ],
-// })
+        // Start playing the audio
+        nowSpeaking.value = true
+        source.start(0)
+        source.onended = () => {
+          nowSpeaking.value = false
+          resolve()
+        }
+      })
+    },
+  ],
+})

 const ttsQueue = useQueue<string>({
  handlers: [
-    async () => {
-      // TODO: migrating to ElevenLabs API, but we need TTS SDK to wrap the API
+    async (ctx) => {
+      const now = Date.now()
+      const res = await streamSpeech('https://airi-api.ayaka.io', elevenLabsApiKey.value, ctx.data, {
+        // voice: 'ShanShan',
+        // Quite good for English
+        voice: 'Myriam',
+        // Beatrice is not 'childish' like the others
+        // voice: 'Beatrice',
+        // text: body.text,
+        model_id: 'eleven_multilingual_v2',
+        voice_settings: {
+          stability: 0.4,
+          similarity_boost: 0.5,
+        },
+      })
+      const elapsed = Date.now() - now

-      // const now = Date.now()
-      // const res = await streamSpeech(ctx.data, elevenLabsApiKey.value)
-      // const elapsed = Date.now() - now
-
-      // // eslint-disable-next-line no-console
-      // console.debug('TTS took', elapsed, 'ms')
+      // eslint-disable-next-line no-console
+      console.debug('TTS took', elapsed, 'ms')

      // Decode the ArrayBuffer into an AudioBuffer
-      // const audioBuffer = await audioContext.decodeAudioData(res)
-      // await audioQueue.add({ audioBuffer, text: ctx.data })
+      const audioBuffer = await audioContext.decodeAudioData(res)
+      await audioQueue.add({ audioBuffer, text: ctx.data })
    },
  ],
 })
@@ -133,7 +143,7 @@ const messageContentQueue = useMessageContentQueue(ttsQueue)
 const emotionsQueue = useQueue<Emotion>({
  handlers: [
    async (ctx) => {
-      await live2DViewerRef.value!.setMotion(EMOTION_EmotioMotionName_value[ctx.data])
+      await live2DViewerRef.value!.setMotion(EMOTION_EmotionMotionName_value[ctx.data])
    },
  ],
 })
@@ -23,7 +23,7 @@ export const EmotionAwkwardMotionName = 'EmotionAwkward'
 export const EmotionThinkMotionName = 'EmotionThink'
 export const EmotionSurpriseMotionName = 'EmotionSurprise'

-export const EMOTION_EmotioMotionName_value = {
+export const EMOTION_EmotionMotionName_value = {
  [Emotion.Happy]: EmotionHappyMotionName,
  [Emotion.Sad]: EmotionSadMotionName,
  [Emotion.Angry]: EmotionAngryMotionName,
@@ -1,3 +1,4 @@
+import type { GenerateAudioStream } from '@airi-proj/elevenlabs/types'
 import type { CoreMessage } from 'ai'
 import { createOpenAI, type OpenAIProvider, type OpenAIProviderSettings } from '@ai-sdk/openai'
 import { streamText } from 'ai'
@@ -35,18 +36,22 @@ export const useLLM = defineStore('llm', () => {
    return await openAI.value.models.list()
  }

-  async function streamSpeech(text: string, apiKey: string) {
+  async function streamSpeech(baseUrl: string, apiKey: string, text: string, options: Omit<Omit<GenerateAudioStream, 'stream'> & { voice: string }, 'text'>) {
    if (!text || !text.trim())
      throw new Error('Text is required')

-    return await ofetch('/api/v1/llm/voice/text-to-speech', {
+    return await ofetch(`${baseUrl}/api/v1/llm/voice/elevenlabs`, {
      body: {
+        ...options,
+        stream: true,
        text,
-        apiKey,
      },
      method: 'POST',
      cache: 'no-cache',
      responseType: 'arrayBuffer',
+      headers: {
+        Authorization: `Bearer ${apiKey}`,
+      },
    })
  }