diff --git a/cspell.config.yaml b/cspell.config.yaml index 89cf277c5..d837a4025 100644 --- a/cspell.config.yaml +++ b/cspell.config.yaml @@ -33,6 +33,7 @@ words: - cientos - cjkfonts - clippy + - clustr - collectblock - colorjs - Comfortaa diff --git a/packages/stage-ui/package.json b/packages/stage-ui/package.json index f59e5ab76..b77cdd9d0 100644 --- a/packages/stage-ui/package.json +++ b/packages/stage-ui/package.json @@ -139,6 +139,7 @@ "@types/three": "^0.178.1", "@unocss/reset": "^66.3.3", "@vitejs/plugin-vue": "^6.0.0", + "clustr": "^0.0.3", "histoire": "1.0.0-alpha.2", "unocss": "^66.3.3", "unplugin-yaml": "^3.0.1", diff --git a/packages/stage-ui/src/components/Scenarios/Providers/SpeechPlayground.vue b/packages/stage-ui/src/components/Scenarios/Providers/SpeechPlayground.vue index eaa8a40dc..3f982bcff 100644 --- a/packages/stage-ui/src/components/Scenarios/Providers/SpeechPlayground.vue +++ b/packages/stage-ui/src/components/Scenarios/Providers/SpeechPlayground.vue @@ -5,6 +5,8 @@ import { FieldCheckbox, FieldSelect } from '@proj-airi/ui' import { computed, onUnmounted, ref, watch } from 'vue' import { useI18n } from 'vue-i18n' +import SpeechStreamingPlayground from './SpeechStreamingPlayground.vue' + import { TestDummyMarker } from '../../Gadgets' const props = defineProps<{ @@ -204,6 +206,12 @@ defineExpose({ {{ errorMessage }} + + diff --git a/packages/stage-ui/src/components/Scenarios/Providers/SpeechStreamingPlayground.vue b/packages/stage-ui/src/components/Scenarios/Providers/SpeechStreamingPlayground.vue new file mode 100644 index 000000000..7be0b8adf --- /dev/null +++ b/packages/stage-ui/src/components/Scenarios/Providers/SpeechStreamingPlayground.vue @@ -0,0 +1,163 @@ + + + + + Streaming Playground + + + + + + Test chunking + + + + + + + Test streaming + + + + + + + + {{ chunk.text }} + + {{ chunk.words }} words, + {{ chunk.reason }} + + + animate(el, { + opacity: [0, 1], + translateX: [10, 0], + duration: 200, + ease: 'inOut', + })" + > + + + Queued + + + + + diff --git a/packages/stage-ui/src/composables/queues.ts b/packages/stage-ui/src/composables/queues.ts index 01e0870f6..84167fd6b 100644 --- a/packages/stage-ui/src/composables/queues.ts +++ b/packages/stage-ui/src/composables/queues.ts @@ -6,6 +6,7 @@ import { ref } from 'vue' import { llmInferenceEndToken } from '../constants' import { EMOTION_VALUES } from '../constants/emotions' +import { chunkTTSInput } from '../utils/tts' import { useQueue } from './queue' export function useEmotionsMessageQueue(emotionsQueue: UseQueueReturn) { @@ -116,19 +117,8 @@ export function useMessageContentQueue(ttsQueue: UseQueueReturn) { return } - const endMarker = /[.?!]/ - processed.value += ctx.data - - while (processed.value) { - const endMarkerExecArray = endMarker.exec(processed.value) - if (!endMarkerExecArray || typeof endMarkerExecArray.index === 'undefined') - break - - const before = processed.value.slice(0, endMarkerExecArray.index + 1) - const after = processed.value.slice(endMarkerExecArray.index + 1) - - await ttsQueue.add(before) - processed.value = after + for await (const chunk of chunkTTSInput(ctx.data)) { + await ttsQueue.add(chunk.text) } }, ], diff --git a/packages/stage-ui/src/utils/tts.ts b/packages/stage-ui/src/utils/tts.ts new file mode 100644 index 000000000..03e1f20d0 --- /dev/null +++ b/packages/stage-ui/src/utils/tts.ts @@ -0,0 +1,142 @@ +import type { ReaderLike } from 'clustr' + +import { readGraphemeClusters } from 'clustr' + +const keptPunctuations = new Set('??!!') +const hardPunctuations = new Set('.。??!!…⋯~~「」\n\t\r') +const softPunctuations = new Set(',,、–—::;;《》') + +export interface TTSInputChunk { + text: string + words: number + reason: 'boost' | 'limit' | 'hard' | 'flush' +} + +export interface TTSInputChunkOptions { + boost?: number + minimumWords?: number + maximumWords?: number +} + +/** + * Processes the input string or UTF-8 byte stream reader into chunks suitable for TTS synthesis. + * + * @param input A string or a ReaderLike object that reads from an underlying UTF-8 byte stream. + * @param options + * @param options.boost Specifies the number of chunks to yield using greedier rules. This may help + * reduce the initial delay when processing long input text. + * @param options.minimumWords Minimum number of words in a chunk. + * @param options.maximumWords Maximum number of words in a chunk. + */ +export async function* chunkTTSInput(input: string | ReaderLike, options?: TTSInputChunkOptions): AsyncGenerator { + const { + boost = 2, + minimumWords = 4, + maximumWords = 12, + } = options ?? {} + + const iterator = readGraphemeClusters( + typeof input === 'string' + ? new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode(input)) + controller.close() + }, + }).getReader() + : input, + ) + + const segmenter = new Intl.Segmenter(undefined, { granularity: 'word' }) // I love Intl.Segmenter + + let yieldCount = 0 + let buffer = '' + let chunk = '' + let chunkWordsCount = 0 + + let previousValue: string | undefined + let current = await iterator.next() + + while (!current.done) { + const value = current.value + + if (value.length > 1) { + previousValue = value + current = await iterator.next() + continue + } + + const hard = hardPunctuations.has(value) + const soft = softPunctuations.has(value) + const kept = keptPunctuations.has(value) + + if (hard || soft) { + switch (value) { + case '.': + case ',': { + if (previousValue !== undefined && /\d/.test(previousValue)) { + const next = await iterator.next() + if (!next.done && next.value && /\d/.test(next.value)) { + // This dot could be a decimal point, so we skip it + previousValue = next.value + current = next + continue + } + } + } + } + + if (buffer.length === 0) { + previousValue = value + current = await iterator.next() + continue + } + + const words = [...segmenter.segment(buffer)].filter(w => w.isWordLike) + + if (chunkWordsCount > minimumWords && chunkWordsCount + words.length > maximumWords) { + const text = kept ? chunk.trim() + value : chunk.trim() + yield { + text, + words: chunkWordsCount, + reason: 'limit', + } + yieldCount++ + chunk = '' + chunkWordsCount = 0 + } + + chunk += buffer + value + chunkWordsCount += words.length + buffer = '' + + if (hard || chunkWordsCount > maximumWords || yieldCount < boost) { + const text = chunk.trim() + yield { + text, + words: chunkWordsCount, + reason: hard ? 'hard' : chunkWordsCount > maximumWords ? 'limit' : 'boost', + } + yieldCount++ + chunk = '' + chunkWordsCount = 0 + } + + previousValue = value + current = await iterator.next() + continue + } + + buffer += value + previousValue = value + current = await iterator.next() + } + + if (chunk.length > 0 || buffer.length > 0) { + const text = (chunk + buffer).trim() + yield { + text, + words: chunkWordsCount + [...segmenter.segment(buffer)].filter(w => w.isWordLike).length, + reason: 'flush', + } + } +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index acf6838fd..65ae02cab 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1419,6 +1419,9 @@ importers: '@vitejs/plugin-vue': specifier: ^6.0.0 version: 6.0.0(vite@6.3.5(@types/node@24.0.13)(jiti@2.4.2)(less@4.3.0)(lightningcss@1.30.1)(terser@5.43.1)(tsx@4.20.3)(yaml@2.8.0))(vue@3.5.17(typescript@5.8.3)) + clustr: + specifier: ^0.0.3 + version: 0.0.3 histoire: specifier: 1.0.0-alpha.2 version: 1.0.0-alpha.2(@types/node@24.0.13)(bufferutil@4.0.9)(less@4.3.0)(lightningcss@1.30.1)(terser@5.43.1)(utf-8-validate@5.0.10)(vite@6.3.5(@types/node@24.0.13)(jiti@2.4.2)(less@4.3.0)(lightningcss@1.30.1)(terser@5.43.1)(tsx@4.20.3)(yaml@2.8.0)) @@ -7047,6 +7050,10 @@ packages: resolution: {integrity: sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA==} engines: {node: '>=6'} + clustr@0.0.3: + resolution: {integrity: sha512-nGJZKUSi2K83migJ7sJj/JYNVbT0L81b7fFNcQ3XIgkXqkeG1tpO1NtISDjr3Unnjqsztzvp/p+8JgBhpAc87g==} + engines: {node: '>=22.10.0'} + color-convert@2.0.1: resolution: {integrity: sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==} engines: {node: '>=7.0.0'} @@ -19426,6 +19433,8 @@ snapshots: clsx@2.1.1: optional: true + clustr@0.0.3: {} + color-convert@2.0.1: dependencies: color-name: 1.1.4