From c13ada508f8010366919ffab30a6fbc4306d8664 Mon Sep 17 00:00:00 2001 From: Lilia_Chen Date: Sun, 16 Nov 2025 19:52:44 +0000 Subject: [PATCH] feat(stage-ui): Synchronised Emotion Expression and Delay Animation to the TTS AIRI Speaking (#741) * feat(stage-ui): Synchronised emotion expression and delay animation with the tts speaking * [autofix.ci] apply automated fixes * Update packages/stage-ui/src/utils/tts.ts Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> * feat(stage-ui): Synchronised emotion expression and delay animation with the tts speaking - code review * feat(stage-ui): Synchronised emotion expression and delay animation with the tts speaking - code review * feat(stage-ui): Synchronised emotion expression and delay animation with the tts speaking - code review * feat(stage-ui): Synchronised emotion expression and delay animation with the tts speaking - code review --------- Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --- .../providers/SpeechStreamingPlayground.vue | 4 +- .../stage-ui/src/components/scenes/Stage.vue | 128 +++-- packages/stage-ui/src/composables/queues.ts | 42 +- packages/stage-ui/src/constants/emotions.ts | 107 ++-- packages/stage-ui/src/utils/tts.ts | 479 ++++++++++-------- 5 files changed, 430 insertions(+), 330 deletions(-) diff --git a/packages/stage-ui/src/components/scenarios/providers/SpeechStreamingPlayground.vue b/packages/stage-ui/src/components/scenarios/providers/SpeechStreamingPlayground.vue index 7071d5351..d6daec6b1 100644 --- a/packages/stage-ui/src/components/scenarios/providers/SpeechStreamingPlayground.vue +++ b/packages/stage-ui/src/components/scenarios/providers/SpeechStreamingPlayground.vue @@ -67,11 +67,11 @@ async function handleSpeechGeneration(ctx: { data: string }) { const ttsQueue = createQueue({ handlers: [handleSpeechGeneration] }) onTextSegmented((chunk) => { - ttsQueue.enqueue(chunk) + ttsQueue.enqueue(chunk.chunk) }) async function testStreaming() { - textSegmentationQueue.value.enqueue(props.text) + textSegmentationQueue.value.enqueue({ type: 'literal', value: props.text }) } async function testChunking() { diff --git a/packages/stage-ui/src/components/scenes/Stage.vue b/packages/stage-ui/src/components/scenes/Stage.vue index d22c1b597..9fb2955c1 100644 --- a/packages/stage-ui/src/components/scenes/Stage.vue +++ b/packages/stage-ui/src/components/scenes/Stage.vue @@ -3,7 +3,9 @@ import type { DuckDBWasmDrizzleDatabase } from '@proj-airi/drizzle-duckdb-wasm' import type { SpeechProviderWithExtraOptions } from '@xsai-ext/shared-providers' import type { UnElevenLabsOptions } from 'unspeech' +import type { TextSegmentationItem } from '../../composables/queues' import type { Emotion } from '../../constants/emotions' +import type { TTSChunkItem } from '../../utils/tts' import { drizzle } from '@proj-airi/drizzle-duckdb-wasm' import { getImportUrlBundles } from '@proj-airi/drizzle-duckdb-wasm/bundles/import-url-browser' @@ -65,7 +67,7 @@ const { textSegmentationQueue } = storeToRefs(textSegmentationStore) clearTextSegmentationHooks() const characterSpeechPlaybackQueue = usePipelineCharacterSpeechPlaybackQueueStore() -const { connectAudioContext, connectAudioAnalyser, clearAll, onPlaybackStarted } = characterSpeechPlaybackQueue +const { connectAudioContext, connectAudioAnalyser, clearAll, onPlaybackStarted, onPlaybackFinished } = characterSpeechPlaybackQueue const { currentAudioSource, playbackQueue } = storeToRefs(characterSpeechPlaybackQueue) const settingsStore = useSettings() @@ -123,60 +125,13 @@ const lipSyncStarted = ref(false) const speechStore = useSpeechStore() const { ssmlEnabled, activeSpeechProvider, activeSpeechModel, activeSpeechVoice, pitch } = storeToRefs(speechStore) -async function handleSpeechGeneration(ctx: { data: string }) { - try { - if (!activeSpeechProvider.value) { - console.warn('No active speech provider configured') - return - } - - if (!activeSpeechVoice.value) { - console.warn('No active speech voice configured') - return - } - - const provider = await providersStore.getProviderInstance(activeSpeechProvider.value) as SpeechProviderWithExtraOptions - if (!provider) { - console.error('Failed to initialize speech provider') - return - } - - const providerConfig = providersStore.getProviderConfig(activeSpeechProvider.value) - - const input = ssmlEnabled.value - ? speechStore.generateSSML(ctx.data, activeSpeechVoice.value, { ...providerConfig, pitch: pitch.value }) - : ctx.data - - const res = await generateSpeech({ - ...provider.speech(activeSpeechModel.value, providerConfig), - input, - voice: activeSpeechVoice.value.id, - }) - - const audioBuffer = await audioContext.decodeAudioData(res) - playbackQueue.value.enqueue({ audioBuffer, text: ctx.data }) - } - catch (error) { - console.error('Speech generation failed:', error) - } -} - -const ttsQueue = createQueue({ - handlers: [ - handleSpeechGeneration, - ], -}) - -onTextSegmented((chunk) => { - ttsQueue.enqueue(chunk) -}) - const { currentMotion } = storeToRefs(useLive2d()) const emotionsQueue = createQueue({ handlers: [ async (ctx) => { if (stageModelRenderer.value === 'vrm') { + // console.debug("VRM emotion anime: ", ctx.data) const value = EMOTION_VRMExpressionName_value[ctx.data] if (!value) return @@ -202,6 +157,73 @@ delaysQueue.onHandlerEvent('delay', (delay) => { console.debug('delay detected', delay) }) +// Play special token: delay or emotion +function playSpecialToken(special: string) { + delaysQueue.enqueue(special) + emotionMessageContentQueue.enqueue(special) +} +onPlaybackFinished(({ special }) => { + playSpecialToken(special) +}) + +async function handleSpeechGeneration(ctx: { data: TTSChunkItem }) { + try { + if (!activeSpeechProvider.value) { + console.warn('No active speech provider configured') + return + } + + if (!activeSpeechVoice.value) { + console.warn('No active speech voice configured') + return + } + + const provider = await providersStore.getProviderInstance(activeSpeechProvider.value) as SpeechProviderWithExtraOptions + if (!provider) { + console.error('Failed to initialize speech provider') + return + } + + // console.debug("ctx.data.chunk is empty? ", ctx.data.chunk === "") + // console.debug("ctx.data.special: ", ctx.data.special) + if (ctx.data.chunk === '' && !ctx.data.special) + return + // If special token only and chunk = "" + if (ctx.data.chunk === '' && ctx.data.special) { + playSpecialToken(ctx.data.special) + return + } + + const providerConfig = providersStore.getProviderConfig(activeSpeechProvider.value) + + const input = ssmlEnabled.value + ? speechStore.generateSSML(ctx.data.chunk, activeSpeechVoice.value, { ...providerConfig, pitch: pitch.value }) + : ctx.data.chunk + + const res = await generateSpeech({ + ...provider.speech(activeSpeechModel.value, providerConfig), + input, + voice: activeSpeechVoice.value.id, + }) + + const audioBuffer = await audioContext.decodeAudioData(res) + playbackQueue.value.enqueue({ audioBuffer, text: ctx.data.chunk, special: ctx.data.special }) + } + catch (error) { + console.error('Speech generation failed:', error) + } +} + +const ttsQueue = createQueue({ + handlers: [ + handleSpeechGeneration, + ], +}) + +onTextSegmented((chunkItem) => { + ttsQueue.enqueue(chunkItem) +}) + function getVolumeWithMinMaxNormalizeWithFrameUpdates() { requestAnimationFrame(getVolumeWithMinMaxNormalizeWithFrameUpdates) if (!nowSpeaking.value) @@ -241,12 +263,14 @@ onBeforeSend(async () => { onTokenLiteral(async (literal) => { // Only push to segmentation; visual presentation happens on playback start - textSegmentationQueue.value.enqueue(literal) + textSegmentationQueue.value.enqueue({ type: 'literal', value: literal } as TextSegmentationItem) }) onTokenSpecial(async (special) => { - delaysQueue.enqueue(special) - emotionMessageContentQueue.enqueue(special) + // delaysQueue.enqueue(special) + // emotionMessageContentQueue.enqueue(special) + // Also push special token to the queue for emotion animation/delay and TTS playback synchronisation + textSegmentationQueue.value.enqueue({ type: 'special', value: special } as TextSegmentationItem) }) onStreamEnd(async () => { diff --git a/packages/stage-ui/src/composables/queues.ts b/packages/stage-ui/src/composables/queues.ts index 8a84ba82f..6205b65dd 100644 --- a/packages/stage-ui/src/composables/queues.ts +++ b/packages/stage-ui/src/composables/queues.ts @@ -1,5 +1,6 @@ import type { Emotion } from '../constants/emotions' import type { UseQueueReturn } from '../utils/queue' +import type { TTSChunkItem } from '../utils/tts' import { sleep } from '@moeru/std' import { invoke } from '@vueuse/core' @@ -9,7 +10,12 @@ import { ref, shallowRef } from 'vue' import { EMOTION_VALUES } from '../constants/emotions' import { createQueue } from '../utils/queue' import { createControllableStream } from '../utils/stream' -import { chunkEmitter } from '../utils/tts' +import { chunkEmitter, TTS_SPECIAL_TOKEN } from '../utils/tts' + +export interface TextSegmentationItem { + type: 'literal' | 'special' + value: string +} export function useEmotionsMessageQueue(emotionsQueue: UseQueueReturn) { function splitEmotion(content: string) { @@ -106,13 +112,13 @@ export function useDelayMessageQueue() { export const usePipelineCharacterSpeechPlaybackQueueStore = defineStore('pipelines:character:speech', () => { // Hooks const onPlaybackStartedHooks = ref Promise | void>>([]) - const onPlaybackFinishedHooks = ref Promise | void>>([]) + const onPlaybackFinishedHooks = ref Promise | void>>([]) // Hooks registers function onPlaybackStarted(hook: (payload: { text: string }) => Promise | void) { onPlaybackStartedHooks.value.push(hook) } - function onPlaybackFinished(hook: (payload: { text: string }) => Promise | void) { + function onPlaybackFinished(hook: (payload: { special: string }) => Promise | void) { onPlaybackFinishedHooks.value.push(hook) } @@ -141,7 +147,7 @@ export const usePipelineCharacterSpeechPlaybackQueueStore = defineStore('pipelin } const playbackQueue = ref(invoke(() => { - return createQueue<{ audioBuffer: AudioBuffer, text: string }>({ + return createQueue<{ audioBuffer: AudioBuffer, text: string, special: string | null }>({ handlers: [ (ctx) => { return new Promise((resolve) => { @@ -167,11 +173,15 @@ export const usePipelineCharacterSpeechPlaybackQueueStore = defineStore('pipelin currentAudioSource.value = source source.start(0) source.onended = () => { - for (const hook of onPlaybackFinishedHooks.value) hook({ text: ctx.data.text }) + // Play special token: delay or emotion + if (ctx.data.special) { + for (const hook of onPlaybackFinishedHooks.value) + hook({ special: ctx.data.special }) + } + if (currentAudioSource.value === source) { currentAudioSource.value = undefined } - resolve() } }) @@ -206,10 +216,10 @@ export const usePipelineCharacterSpeechPlaybackQueueStore = defineStore('pipelin export const usePipelineWorkflowTextSegmentationStore = defineStore('pipelines:workflows:text-segmentation', () => { // Hooks - const onTextSegmentedHooks = ref Promise | void>>([]) + const onTextSegmentedHooks = ref Promise | void>>([]) // Hooks registers - function onTextSegmented(hook: (segment: string) => Promise | void) { + function onTextSegmented(hook: (segment: TTSChunkItem) => Promise | void) { onTextSegmentedHooks.value.push(hook) } @@ -226,17 +236,27 @@ export const usePipelineWorkflowTextSegmentationStore = defineStore('pipelines:w const { stream, controller } = createControllableStream() textSegmentationStream.value = stream textSegmentationStreamController.value = controller + // This is the queue for pending special tokens + const pendingSpecials: string[] = [] - chunkEmitter(stream.getReader(), async (chunk) => { + chunkEmitter(stream.getReader(), pendingSpecials, async (chunk) => { for (const hook of onTextSegmentedHooks.value) { await hook(chunk) } }) - return createQueue({ + return createQueue({ handlers: [ async (ctx) => { - controller.enqueue(encoder.encode(ctx.data)) + if (ctx.data.type === 'literal') { + controller.enqueue(encoder.encode(ctx.data.value)) + } + else { + // Special literal, need to be flushed in tts rechunking + // console.debug("TextSegmentationQueue: Special enqueue", encoder.encode(TTS_SPECIAL_TOKEN)) + pendingSpecials.push(ctx.data.value) + controller.enqueue(encoder.encode(TTS_SPECIAL_TOKEN)) + } }, ], }) diff --git a/packages/stage-ui/src/constants/emotions.ts b/packages/stage-ui/src/constants/emotions.ts index 392395c45..6e781c1cd 100644 --- a/packages/stage-ui/src/constants/emotions.ts +++ b/packages/stage-ui/src/constants/emotions.ts @@ -1,51 +1,56 @@ -export const EMOTION_HAPPY = '<|EMOTE_HAPPY|>' -export const EMOTION_SAD = '<|EMOTE_SAD|>' -export const EMOTION_ANGRY = '<|EMOTE_ANGRY|>' -export const EMOTION_THINK = '<|EMOTE_THINK|>' -export const EMOTION_SURPRISE = '<|EMOTE_SURPRISE|>' -export const EMOTION_AWKWARD = '<|EMOTE_AWKWARD|>' -export const EMOTION_QUESTION = '<|EMOTE_QUESTION|>' - -export enum Emotion { - Idle = '<|EMOTE_NEUTRAL|>', - Happy = '<|EMOTE_HAPPY|>', - Sad = '<|EMOTE_SAD|>', - Angry = '<|EMOTE_ANGRY|>', - Think = '<|EMOTE_THINK|>', - Surprise = '<|EMOTE_SURPRISE|>', - Awkward = '<|EMOTE_AWKWARD|>', - Question = '<|EMOTE_QUESTION|>', -} - -export const EMOTION_VALUES = Object.values(Emotion) - -export const EmotionHappyMotionName = 'Happy' -export const EmotionSadMotionName = 'Sad' -export const EmotionAngryMotionName = 'Angry' -export const EmotionAwkwardMotionName = 'Awkward' -export const EmotionThinkMotionName = 'Think' -export const EmotionSurpriseMotionName = 'Surprise' -export const EmotionQuestionMotionName = 'Question' -export const EmotionNeutralMotionName = 'Idle' - -export const EMOTION_EmotionMotionName_value = { - [Emotion.Happy]: EmotionHappyMotionName, - [Emotion.Sad]: EmotionSadMotionName, - [Emotion.Angry]: EmotionAngryMotionName, - [Emotion.Think]: EmotionThinkMotionName, - [Emotion.Surprise]: EmotionSurpriseMotionName, - [Emotion.Awkward]: EmotionAwkwardMotionName, - [Emotion.Question]: EmotionQuestionMotionName, - [Emotion.Idle]: EmotionNeutralMotionName, -} - -export const EMOTION_VRMExpressionName_value = { - [Emotion.Happy]: 'happy', - [Emotion.Sad]: 'sad', - [Emotion.Angry]: 'angry', - [Emotion.Think]: undefined, - [Emotion.Surprise]: 'surprised', - [Emotion.Awkward]: undefined, - [Emotion.Question]: undefined, - [Emotion.Idle]: undefined, -} satisfies Record +export const EMOTION_HAPPY = '<|EMOTE_HAPPY|>' +export const EMOTION_SAD = '<|EMOTE_SAD|>' +export const EMOTION_ANGRY = '<|EMOTE_ANGRY|>' +export const EMOTION_THINK = '<|EMOTE_THINK|>' +export const EMOTION_SURPRISE = '<|EMOTE_SURPRISED|>' +export const EMOTION_AWKWARD = '<|EMOTE_AWKWARD|>' +export const EMOTION_QUESTION = '<|EMOTE_QUESTION|>' +export const EMOTION_CURIOUS = '<|EMOTE_CURIOUS|>' + +export enum Emotion { + Idle = '<|EMOTE_NEUTRAL|>', + Happy = '<|EMOTE_HAPPY|>', + Sad = '<|EMOTE_SAD|>', + Angry = '<|EMOTE_ANGRY|>', + Think = '<|EMOTE_THINK|>', + Surprise = '<|EMOTE_SURPRISED|>', + Awkward = '<|EMOTE_AWKWARD|>', + Question = '<|EMOTE_QUESTION|>', + Curious = '<|EMOTE_CURIOUS|>', +} + +export const EMOTION_VALUES = Object.values(Emotion) + +export const EmotionHappyMotionName = 'Happy' +export const EmotionSadMotionName = 'Sad' +export const EmotionAngryMotionName = 'Angry' +export const EmotionAwkwardMotionName = 'Awkward' +export const EmotionThinkMotionName = 'Think' +export const EmotionSurpriseMotionName = 'Surprise' +export const EmotionQuestionMotionName = 'Question' +export const EmotionNeutralMotionName = 'Idle' +export const EmotionCuriousMotionName = 'Curious' + +export const EMOTION_EmotionMotionName_value = { + [Emotion.Happy]: EmotionHappyMotionName, + [Emotion.Sad]: EmotionSadMotionName, + [Emotion.Angry]: EmotionAngryMotionName, + [Emotion.Think]: EmotionThinkMotionName, + [Emotion.Surprise]: EmotionSurpriseMotionName, + [Emotion.Awkward]: EmotionAwkwardMotionName, + [Emotion.Question]: EmotionQuestionMotionName, + [Emotion.Idle]: EmotionNeutralMotionName, + [Emotion.Curious]: EmotionCuriousMotionName, +} + +export const EMOTION_VRMExpressionName_value = { + [Emotion.Happy]: 'happy', + [Emotion.Sad]: 'sad', + [Emotion.Angry]: 'angry', + [Emotion.Think]: undefined, + [Emotion.Surprise]: 'surprised', + [Emotion.Awkward]: undefined, + [Emotion.Question]: undefined, + [Emotion.Idle]: undefined, + [Emotion.Curious]: 'surprised', +} satisfies Record diff --git a/packages/stage-ui/src/utils/tts.ts b/packages/stage-ui/src/utils/tts.ts index af8823510..84cac8374 100644 --- a/packages/stage-ui/src/utils/tts.ts +++ b/packages/stage-ui/src/utils/tts.ts @@ -1,214 +1,265 @@ -import type { ReaderLike } from 'clustr' - -import { readGraphemeClusters } from 'clustr' - -// A special character to instruct the TTS pipeline to flush -export const TTS_FLUSH_INSTRUCTION = '\u200B' - -const keptPunctuations = new Set('?？!！') -const hardPunctuations = new Set('.。?？!！…⋯～~\n\t\r') -const softPunctuations = new Set(',，、–—:：;；《》「」') - -export interface TTSInputChunk { - text: string - words: number - reason: 'boost' | 'limit' | 'hard' | 'flush' -} - -export interface TTSInputChunkOptions { - boost?: number - minimumWords?: number - maximumWords?: number -} - -/** - * Processes the input string or UTF-8 byte stream reader into chunks suitable for TTS synthesis. - * - * @param input A string or a ReaderLike object that reads from an underlying UTF-8 byte stream. - * @param options - * @param options.boost Specifies the number of chunks to yield using greedier rules. This may help - * reduce the initial delay when processing long input text. - * @param options.minimumWords Minimum number of words in a chunk. - * @param options.maximumWords Maximum number of words in a chunk. - */ -export async function* chunkTTSInput(input: string | ReaderLike, options?: TTSInputChunkOptions): AsyncGenerator { - const { - boost = 2, - minimumWords = 4, - maximumWords = 12, - } = options ?? {} - - const iterator = readGraphemeClusters( - typeof input === 'string' - ? new ReadableStream({ - start(controller) { - controller.enqueue(new TextEncoder().encode(input)) - controller.close() - }, - }).getReader() - : input, - ) - - const segmenter = new Intl.Segmenter(undefined, { granularity: 'word' }) // I love Intl.Segmenter - - let yieldCount = 0 - let buffer = '' - let chunk = '' - let chunkWordsCount = 0 - - let previousValue: string | undefined - let current = await iterator.next() - - while (!current.done) { - let value = current.value - - if (value.length > 1) { - previousValue = value - current = await iterator.next() - continue - } - - const flush = value === TTS_FLUSH_INSTRUCTION - const hard = hardPunctuations.has(value) - const soft = softPunctuations.has(value) - const kept = keptPunctuations.has(value) - let next: IteratorResult | undefined - let afterNext: IteratorResult | undefined - - if (flush || hard || soft) { - switch (value) { - case '.': - case ',': { - if (previousValue !== undefined && /\d/.test(previousValue)) { - next = await iterator.next() - if (!next.done && next.value && /\d/.test(next.value)) { - // This dot could be a decimal point, so we skip it (don't fully skip! keep in tts input!) - - // REVIEW: @Lilia-Chen I think we need to remove the below line - // 1. Not sensible to let the previousValue to be the value of the next value - // 2. after the continue (jump to the bottom of the while loop), the previousValue will be reset to value, so this value assignment doesn't work at all - // previousValue = next.value - - // If we don't append value ("." or ","), we will lose the decimal point, and 2.5 will become 25, which hugely impacted the tts... - buffer += value - current = next - next = undefined - continue - } - } - else if (value === '.') { - // trying catch '...' and turn it into U+2026 - next = await iterator.next() - if (!next.done && next.value && next.value === '.') { - afterNext = await iterator.next() - // If this is a '...' repalce the current value - if (!afterNext.done && afterNext.value && afterNext.value === '.') { - value = '…' - next = undefined - afterNext = undefined - } - } - } - } - } - - if (buffer.length === 0) { - previousValue = value - current = await iterator.next() - continue - } - - const words = [...segmenter.segment(buffer)].filter(w => w.isWordLike) - - if (chunkWordsCount > minimumWords && chunkWordsCount + words.length > maximumWords) { - const text = kept ? chunk.trim() + value : chunk.trim() - yield { - text, - words: chunkWordsCount, - reason: 'limit', - } - yieldCount++ - chunk = '' - chunkWordsCount = 0 - } - - chunk += buffer + value - chunkWordsCount += words.length - buffer = '' - - if (flush || hard || chunkWordsCount > maximumWords || yieldCount < boost) { - const text = chunk.trim() - yield { - text, - words: chunkWordsCount, - reason: flush ? 'flush' : hard ? 'hard' : chunkWordsCount > maximumWords ? 'limit' : 'boost', - } - yieldCount++ - chunk = '' - chunkWordsCount = 0 - } - - previousValue = value - // If next had been read during decimal recognition or "..." recognition - if (next !== undefined) { - // If afterNext had also been read - if (afterNext !== undefined) { - // The only case that the program will come to this place is: - // "x..y" and y is not a "." so "..." is not recognised - // ".." is not a legal punctuation so ignored. - // The first "." had been consumed during hard flush - // next.value = second '.' - // afterNext.value = 'y', so we just need to care about 'y' - // In case 'y' is not just a useless blank. It's OK if 'y' is blank since we have `text = chunk.trim()` - current = afterNext - next = undefined - afterNext = undefined - } - else { - // Only next had been read - // This is the case where "x.y" and x is a number but y is not - // In case 'y' is not just a useless blank. It's OK if 'y' is blank since we have `text = chunk.trim()` - current = next - next = undefined - } - } - else { - // No next nor afterNext, so run `iterator.next()` - current = await iterator.next() - } - // No need to do anything with buffer, just jump to the next loop - continue - } - // If normal character enters, add it to the buffer and move to the next - buffer += value - previousValue = value - next = await iterator.next() - current = next - } - - // TODO: remove later - // eslint-disable-next-line no-console - console.debug('while loop ends, chunk/buffer:', chunk, buffer) - if (chunk.length > 0 || buffer.length > 0) { - const text = (chunk + buffer).trim() - yield { - text, - words: chunkWordsCount + [...segmenter.segment(buffer)].filter(w => w.isWordLike).length, - reason: 'flush', - } - } -} - -export async function chunkEmitter(reader: ReaderLike, handler: (chunk: string) => Promise | void) { - try { - for await (const chunk of chunkTTSInput(reader)) { - // TODO: remove later - // eslint-disable-next-line no-console - console.debug('chunk to be pushed: ', chunk) - await handler(chunk.text) - } - } - catch (e) { - console.error('Error chunking stream to TTS queue:', e) - } -} +import type { ReaderLike } from 'clustr' + +import { readGraphemeClusters } from 'clustr' + +// A special character to instruct the TTS pipeline to flush +export const TTS_FLUSH_INSTRUCTION = '\u200B' +// This is for special literals +export const TTS_SPECIAL_TOKEN = '\u2063' + +const keptPunctuations = new Set('?？!！') +const hardPunctuations = new Set('.。?？!！…⋯～~\n\t\r') +const softPunctuations = new Set(',，、–—:：;；《》「」') + +export interface TTSInputChunk { + text: string + words: number + reason: 'boost' | 'limit' | 'hard' | 'flush' | 'special' +} + +export interface TTSInputChunkOptions { + boost?: number + minimumWords?: number + maximumWords?: number +} + +// New output type for tts, metaData (special token) contained +export interface TTSChunkItem { + chunk: string + special: string | null +} + +/** + * Processes the input string or UTF-8 byte stream reader into chunks suitable for TTS synthesis. + * + * @param input A string or a ReaderLike object that reads from an underlying UTF-8 byte stream. + * @param options + * @param options.boost Specifies the number of chunks to yield using greedier rules. This may help + * reduce the initial delay when processing long input text. + * @param options.minimumWords Minimum number of words in a chunk. + * @param options.maximumWords Maximum number of words in a chunk. + */ +export async function* chunkTTSInput( + input: string | ReaderLike, + options?: TTSInputChunkOptions, +): AsyncGenerator { + const { + boost = 2, + minimumWords = 4, + maximumWords = 12, + } = options ?? {} + + const iterator = readGraphemeClusters( + typeof input === 'string' + ? new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode(input)) + controller.close() + }, + }).getReader() + : input, + ) + + const segmenter = new Intl.Segmenter(undefined, { granularity: 'word' }) // I love Intl.Segmenter + + let yieldCount = 0 + let buffer = '' + let chunk = '' + let chunkWordsCount = 0 + + let previousValue: string | undefined + let current = await iterator.next() + + while (!current.done) { + let value = current.value + + if (value.length > 1) { + previousValue = value + current = await iterator.next() + continue + } + + const flush = value === TTS_FLUSH_INSTRUCTION + const special = value === TTS_SPECIAL_TOKEN + const hard = hardPunctuations.has(value) + const soft = softPunctuations.has(value) + const kept = keptPunctuations.has(value) + let next: IteratorResult | undefined + let afterNext: IteratorResult | undefined + + if (flush || special || hard || soft) { + switch (value) { + case '.': + case ',': { + if (previousValue !== undefined && /\d/.test(previousValue)) { + next = await iterator.next() + if (!next.done && next.value && /\d/.test(next.value)) { + // This dot could be a decimal point, so we skip it (don't fully skip! keep in tts input!) + + // REVIEW: @Lilia-Chen I think we need to remove the below line + // 1. Not sensible to let the previousValue to be the value of the next value + // 2. after the continue (jump to the bottom of the while loop), the previousValue will be reset to value, so this value assignment doesn't work at all + // previousValue = next.value + + // If we don't append value ("." or ","), we will lose the decimal point, and 2.5 will become 25, which hugely impacted the tts... + buffer += value + current = next + next = undefined + continue + } + } + else if (value === '.') { + // trying catch '...' and turn it into U+2026 + next = await iterator.next() + if (!next.done && next.value && next.value === '.') { + afterNext = await iterator.next() + // If this is a '...' repalce the current value + if (!afterNext.done && afterNext.value && afterNext.value === '.') { + value = '…' + next = undefined + afterNext = undefined + } + } + } + } + } + + if (buffer.length === 0) { + if (special) { + // Special token without buffered text still needs to be surfaced so + // downstream queues can process delay/emotion markers. + yield { + text: '', + words: 0, + reason: 'special', + } + yieldCount++ + chunkWordsCount = 0 + } + + previousValue = value + current = await iterator.next() + continue + } + + const words = [...segmenter.segment(buffer)].filter(w => w.isWordLike) + + if (chunkWordsCount > minimumWords && chunkWordsCount + words.length > maximumWords) { + const text = kept ? chunk.trim() + value : chunk.trim() + yield { + text, + words: chunkWordsCount, + reason: 'limit', + } + yieldCount++ + chunk = '' + chunkWordsCount = 0 + } + + chunk += buffer + value + chunkWordsCount += words.length + buffer = '' + + if (special) { + const text = chunk.slice(0, -1).trim() + yield { + text, + words: chunkWordsCount, + reason: 'special', + } + yieldCount++ + chunk = '' + chunkWordsCount = 0 + } + else if (flush || hard || chunkWordsCount > maximumWords || yieldCount < boost) { + const text = chunk.trim() + yield { + text, + words: chunkWordsCount, + reason: flush ? 'flush' : hard ? 'hard' : chunkWordsCount > maximumWords ? 'limit' : 'boost', + } + yieldCount++ + chunk = '' + chunkWordsCount = 0 + } + + previousValue = value + // If next had been read during decimal recognition or "..." recognition + if (next !== undefined) { + // If afterNext had also been read + if (afterNext !== undefined) { + // The only case that the program will come to this place is: + // "x..y" and y is not a "." so "..." is not recognised + // ".." is not a legal punctuation so ignored. + // The first "." had been consumed during hard flush + // next.value = second '.' + // afterNext.value = 'y', so we just need to care about 'y' + // In case 'y' is not just a useless blank. It's OK if 'y' is blank since we have `text = chunk.trim()` + current = afterNext + next = undefined + afterNext = undefined + } + else { + // Only next had been read + // This is the case where "x.y" and x is a number but y is not + // In case 'y' is not just a useless blank. It's OK if 'y' is blank since we have `text = chunk.trim()` + current = next + next = undefined + } + } + else { + // No next nor afterNext, so run `iterator.next()` + current = await iterator.next() + } + // No need to do anything with buffer, just jump to the next loop + continue + } + // If normal character enters, add it to the buffer and move to the next + buffer += value + previousValue = value + next = await iterator.next() + current = next + } + + // TODO: remove later + // eslint-disable-next-line no-console + console.debug('while loop ends, chunk/buffer:', chunk, buffer) + if (chunk.length > 0 || buffer.length > 0) { + const text = (chunk + buffer).trim() + yield { + text, + words: chunkWordsCount + [...segmenter.segment(buffer)].filter(w => w.isWordLike).length, + reason: 'flush', + } + } +} + +export async function chunkEmitter( + reader: ReaderLike, + pendingSpecials: string[], + handler: (ttsSegment: TTSChunkItem) => Promise | void, +) { + const sanitizeChunk = (text: string) => + text + .replaceAll(TTS_SPECIAL_TOKEN, '') + .replaceAll(TTS_FLUSH_INSTRUCTION, '') + .trim() + + try { + for await (const chunk of chunkTTSInput(reader)) { + // TODO: remove later + + if (chunk.reason === 'special') { + const specialToken = pendingSpecials.shift() + // console.debug("special yield:", specialToken) + await handler({ chunk: sanitizeChunk(chunk.text), special: specialToken ?? null }) + } + else { + await handler({ chunk: sanitizeChunk(chunk.text), special: null } as TTSChunkItem) + } + } + } + catch (e) { + console.error('Error chunking stream to TTS queue:', e) + } +}