feat(stage-ui): Synchronised Emotion Expression and Delay Animation to the TTS AIRI Speaking (#741)
* feat(stage-ui): Synchronised emotion expression and delay animation with the tts speaking * [autofix.ci] apply automated fixes * Update packages/stage-ui/src/utils/tts.ts Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> * feat(stage-ui): Synchronised emotion expression and delay animation with the tts speaking - code review * feat(stage-ui): Synchronised emotion expression and delay animation with the tts speaking - code review * feat(stage-ui): Synchronised emotion expression and delay animation with the tts speaking - code review * feat(stage-ui): Synchronised emotion expression and delay animation with the tts speaking - code review --------- Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
This commit is contained in:
Lilia_Chen
co-authored by
gemini-code-assist[bot]
autofix-ci[bot]
gemini-code-assist[bot]
autofix-ci[bot]
parent
3240276e40
commit
c13ada508f
@@ -67,11 +67,11 @@ async function handleSpeechGeneration(ctx: { data: string }) {
|
||||
const ttsQueue = createQueue<string>({ handlers: [handleSpeechGeneration] })
|
||||
|
||||
onTextSegmented((chunk) => {
|
||||
ttsQueue.enqueue(chunk)
|
||||
ttsQueue.enqueue(chunk.chunk)
|
||||
})
|
||||
|
||||
async function testStreaming() {
|
||||
textSegmentationQueue.value.enqueue(props.text)
|
||||
textSegmentationQueue.value.enqueue({ type: 'literal', value: props.text })
|
||||
}
|
||||
|
||||
async function testChunking() {
|
||||
|
||||
@@ -3,7 +3,9 @@ import type { DuckDBWasmDrizzleDatabase } from '@proj-airi/drizzle-duckdb-wasm'
|
||||
import type { SpeechProviderWithExtraOptions } from '@xsai-ext/shared-providers'
|
||||
import type { UnElevenLabsOptions } from 'unspeech'
|
||||
|
||||
import type { TextSegmentationItem } from '../../composables/queues'
|
||||
import type { Emotion } from '../../constants/emotions'
|
||||
import type { TTSChunkItem } from '../../utils/tts'
|
||||
|
||||
import { drizzle } from '@proj-airi/drizzle-duckdb-wasm'
|
||||
import { getImportUrlBundles } from '@proj-airi/drizzle-duckdb-wasm/bundles/import-url-browser'
|
||||
@@ -65,7 +67,7 @@ const { textSegmentationQueue } = storeToRefs(textSegmentationStore)
|
||||
clearTextSegmentationHooks()
|
||||
|
||||
const characterSpeechPlaybackQueue = usePipelineCharacterSpeechPlaybackQueueStore()
|
||||
const { connectAudioContext, connectAudioAnalyser, clearAll, onPlaybackStarted } = characterSpeechPlaybackQueue
|
||||
const { connectAudioContext, connectAudioAnalyser, clearAll, onPlaybackStarted, onPlaybackFinished } = characterSpeechPlaybackQueue
|
||||
const { currentAudioSource, playbackQueue } = storeToRefs(characterSpeechPlaybackQueue)
|
||||
|
||||
const settingsStore = useSettings()
|
||||
@@ -123,60 +125,13 @@ const lipSyncStarted = ref(false)
|
||||
const speechStore = useSpeechStore()
|
||||
const { ssmlEnabled, activeSpeechProvider, activeSpeechModel, activeSpeechVoice, pitch } = storeToRefs(speechStore)
|
||||
|
||||
async function handleSpeechGeneration(ctx: { data: string }) {
|
||||
try {
|
||||
if (!activeSpeechProvider.value) {
|
||||
console.warn('No active speech provider configured')
|
||||
return
|
||||
}
|
||||
|
||||
if (!activeSpeechVoice.value) {
|
||||
console.warn('No active speech voice configured')
|
||||
return
|
||||
}
|
||||
|
||||
const provider = await providersStore.getProviderInstance(activeSpeechProvider.value) as SpeechProviderWithExtraOptions<string, UnElevenLabsOptions>
|
||||
if (!provider) {
|
||||
console.error('Failed to initialize speech provider')
|
||||
return
|
||||
}
|
||||
|
||||
const providerConfig = providersStore.getProviderConfig(activeSpeechProvider.value)
|
||||
|
||||
const input = ssmlEnabled.value
|
||||
? speechStore.generateSSML(ctx.data, activeSpeechVoice.value, { ...providerConfig, pitch: pitch.value })
|
||||
: ctx.data
|
||||
|
||||
const res = await generateSpeech({
|
||||
...provider.speech(activeSpeechModel.value, providerConfig),
|
||||
input,
|
||||
voice: activeSpeechVoice.value.id,
|
||||
})
|
||||
|
||||
const audioBuffer = await audioContext.decodeAudioData(res)
|
||||
playbackQueue.value.enqueue({ audioBuffer, text: ctx.data })
|
||||
}
|
||||
catch (error) {
|
||||
console.error('Speech generation failed:', error)
|
||||
}
|
||||
}
|
||||
|
||||
const ttsQueue = createQueue<string>({
|
||||
handlers: [
|
||||
handleSpeechGeneration,
|
||||
],
|
||||
})
|
||||
|
||||
onTextSegmented((chunk) => {
|
||||
ttsQueue.enqueue(chunk)
|
||||
})
|
||||
|
||||
const { currentMotion } = storeToRefs(useLive2d())
|
||||
|
||||
const emotionsQueue = createQueue<Emotion>({
|
||||
handlers: [
|
||||
async (ctx) => {
|
||||
if (stageModelRenderer.value === 'vrm') {
|
||||
// console.debug("VRM emotion anime: ", ctx.data)
|
||||
const value = EMOTION_VRMExpressionName_value[ctx.data]
|
||||
if (!value)
|
||||
return
|
||||
@@ -202,6 +157,73 @@ delaysQueue.onHandlerEvent('delay', (delay) => {
|
||||
console.debug('delay detected', delay)
|
||||
})
|
||||
|
||||
// Play special token: delay or emotion
|
||||
function playSpecialToken(special: string) {
|
||||
delaysQueue.enqueue(special)
|
||||
emotionMessageContentQueue.enqueue(special)
|
||||
}
|
||||
onPlaybackFinished(({ special }) => {
|
||||
playSpecialToken(special)
|
||||
})
|
||||
|
||||
async function handleSpeechGeneration(ctx: { data: TTSChunkItem }) {
|
||||
try {
|
||||
if (!activeSpeechProvider.value) {
|
||||
console.warn('No active speech provider configured')
|
||||
return
|
||||
}
|
||||
|
||||
if (!activeSpeechVoice.value) {
|
||||
console.warn('No active speech voice configured')
|
||||
return
|
||||
}
|
||||
|
||||
const provider = await providersStore.getProviderInstance(activeSpeechProvider.value) as SpeechProviderWithExtraOptions<string, UnElevenLabsOptions>
|
||||
if (!provider) {
|
||||
console.error('Failed to initialize speech provider')
|
||||
return
|
||||
}
|
||||
|
||||
// console.debug("ctx.data.chunk is empty? ", ctx.data.chunk === "")
|
||||
// console.debug("ctx.data.special: ", ctx.data.special)
|
||||
if (ctx.data.chunk === '' && !ctx.data.special)
|
||||
return
|
||||
// If special token only and chunk = ""
|
||||
if (ctx.data.chunk === '' && ctx.data.special) {
|
||||
playSpecialToken(ctx.data.special)
|
||||
return
|
||||
}
|
||||
|
||||
const providerConfig = providersStore.getProviderConfig(activeSpeechProvider.value)
|
||||
|
||||
const input = ssmlEnabled.value
|
||||
? speechStore.generateSSML(ctx.data.chunk, activeSpeechVoice.value, { ...providerConfig, pitch: pitch.value })
|
||||
: ctx.data.chunk
|
||||
|
||||
const res = await generateSpeech({
|
||||
...provider.speech(activeSpeechModel.value, providerConfig),
|
||||
input,
|
||||
voice: activeSpeechVoice.value.id,
|
||||
})
|
||||
|
||||
const audioBuffer = await audioContext.decodeAudioData(res)
|
||||
playbackQueue.value.enqueue({ audioBuffer, text: ctx.data.chunk, special: ctx.data.special })
|
||||
}
|
||||
catch (error) {
|
||||
console.error('Speech generation failed:', error)
|
||||
}
|
||||
}
|
||||
|
||||
const ttsQueue = createQueue<TTSChunkItem>({
|
||||
handlers: [
|
||||
handleSpeechGeneration,
|
||||
],
|
||||
})
|
||||
|
||||
onTextSegmented((chunkItem) => {
|
||||
ttsQueue.enqueue(chunkItem)
|
||||
})
|
||||
|
||||
function getVolumeWithMinMaxNormalizeWithFrameUpdates() {
|
||||
requestAnimationFrame(getVolumeWithMinMaxNormalizeWithFrameUpdates)
|
||||
if (!nowSpeaking.value)
|
||||
@@ -241,12 +263,14 @@ onBeforeSend(async () => {
|
||||
|
||||
onTokenLiteral(async (literal) => {
|
||||
// Only push to segmentation; visual presentation happens on playback start
|
||||
textSegmentationQueue.value.enqueue(literal)
|
||||
textSegmentationQueue.value.enqueue({ type: 'literal', value: literal } as TextSegmentationItem)
|
||||
})
|
||||
|
||||
onTokenSpecial(async (special) => {
|
||||
delaysQueue.enqueue(special)
|
||||
emotionMessageContentQueue.enqueue(special)
|
||||
// delaysQueue.enqueue(special)
|
||||
// emotionMessageContentQueue.enqueue(special)
|
||||
// Also push special token to the queue for emotion animation/delay and TTS playback synchronisation
|
||||
textSegmentationQueue.value.enqueue({ type: 'special', value: special } as TextSegmentationItem)
|
||||
})
|
||||
|
||||
onStreamEnd(async () => {
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import type { Emotion } from '../constants/emotions'
|
||||
import type { UseQueueReturn } from '../utils/queue'
|
||||
import type { TTSChunkItem } from '../utils/tts'
|
||||
|
||||
import { sleep } from '@moeru/std'
|
||||
import { invoke } from '@vueuse/core'
|
||||
@@ -9,7 +10,12 @@ import { ref, shallowRef } from 'vue'
|
||||
import { EMOTION_VALUES } from '../constants/emotions'
|
||||
import { createQueue } from '../utils/queue'
|
||||
import { createControllableStream } from '../utils/stream'
|
||||
import { chunkEmitter } from '../utils/tts'
|
||||
import { chunkEmitter, TTS_SPECIAL_TOKEN } from '../utils/tts'
|
||||
|
||||
export interface TextSegmentationItem {
|
||||
type: 'literal' | 'special'
|
||||
value: string
|
||||
}
|
||||
|
||||
export function useEmotionsMessageQueue(emotionsQueue: UseQueueReturn<Emotion>) {
|
||||
function splitEmotion(content: string) {
|
||||
@@ -106,13 +112,13 @@ export function useDelayMessageQueue() {
|
||||
export const usePipelineCharacterSpeechPlaybackQueueStore = defineStore('pipelines:character:speech', () => {
|
||||
// Hooks
|
||||
const onPlaybackStartedHooks = ref<Array<(payload: { text: string }) => Promise<void> | void>>([])
|
||||
const onPlaybackFinishedHooks = ref<Array<(payload: { text: string }) => Promise<void> | void>>([])
|
||||
const onPlaybackFinishedHooks = ref<Array<(payload: { special: string }) => Promise<void> | void>>([])
|
||||
|
||||
// Hooks registers
|
||||
function onPlaybackStarted(hook: (payload: { text: string }) => Promise<void> | void) {
|
||||
onPlaybackStartedHooks.value.push(hook)
|
||||
}
|
||||
function onPlaybackFinished(hook: (payload: { text: string }) => Promise<void> | void) {
|
||||
function onPlaybackFinished(hook: (payload: { special: string }) => Promise<void> | void) {
|
||||
onPlaybackFinishedHooks.value.push(hook)
|
||||
}
|
||||
|
||||
@@ -141,7 +147,7 @@ export const usePipelineCharacterSpeechPlaybackQueueStore = defineStore('pipelin
|
||||
}
|
||||
|
||||
const playbackQueue = ref(invoke(() => {
|
||||
return createQueue<{ audioBuffer: AudioBuffer, text: string }>({
|
||||
return createQueue<{ audioBuffer: AudioBuffer, text: string, special: string | null }>({
|
||||
handlers: [
|
||||
(ctx) => {
|
||||
return new Promise((resolve) => {
|
||||
@@ -167,11 +173,15 @@ export const usePipelineCharacterSpeechPlaybackQueueStore = defineStore('pipelin
|
||||
currentAudioSource.value = source
|
||||
source.start(0)
|
||||
source.onended = () => {
|
||||
for (const hook of onPlaybackFinishedHooks.value) hook({ text: ctx.data.text })
|
||||
// Play special token: delay or emotion
|
||||
if (ctx.data.special) {
|
||||
for (const hook of onPlaybackFinishedHooks.value)
|
||||
hook({ special: ctx.data.special })
|
||||
}
|
||||
|
||||
if (currentAudioSource.value === source) {
|
||||
currentAudioSource.value = undefined
|
||||
}
|
||||
|
||||
resolve()
|
||||
}
|
||||
})
|
||||
@@ -206,10 +216,10 @@ export const usePipelineCharacterSpeechPlaybackQueueStore = defineStore('pipelin
|
||||
|
||||
export const usePipelineWorkflowTextSegmentationStore = defineStore('pipelines:workflows:text-segmentation', () => {
|
||||
// Hooks
|
||||
const onTextSegmentedHooks = ref<Array<(segment: string) => Promise<void> | void>>([])
|
||||
const onTextSegmentedHooks = ref<Array<(segment: TTSChunkItem) => Promise<void> | void>>([])
|
||||
|
||||
// Hooks registers
|
||||
function onTextSegmented(hook: (segment: string) => Promise<void> | void) {
|
||||
function onTextSegmented(hook: (segment: TTSChunkItem) => Promise<void> | void) {
|
||||
onTextSegmentedHooks.value.push(hook)
|
||||
}
|
||||
|
||||
@@ -226,17 +236,27 @@ export const usePipelineWorkflowTextSegmentationStore = defineStore('pipelines:w
|
||||
const { stream, controller } = createControllableStream<Uint8Array>()
|
||||
textSegmentationStream.value = stream
|
||||
textSegmentationStreamController.value = controller
|
||||
// This is the queue for pending special tokens
|
||||
const pendingSpecials: string[] = []
|
||||
|
||||
chunkEmitter(stream.getReader(), async (chunk) => {
|
||||
chunkEmitter(stream.getReader(), pendingSpecials, async (chunk) => {
|
||||
for (const hook of onTextSegmentedHooks.value) {
|
||||
await hook(chunk)
|
||||
}
|
||||
})
|
||||
|
||||
return createQueue<string>({
|
||||
return createQueue<TextSegmentationItem>({
|
||||
handlers: [
|
||||
async (ctx) => {
|
||||
controller.enqueue(encoder.encode(ctx.data))
|
||||
if (ctx.data.type === 'literal') {
|
||||
controller.enqueue(encoder.encode(ctx.data.value))
|
||||
}
|
||||
else {
|
||||
// Special literal, need to be flushed in tts rechunking
|
||||
// console.debug("TextSegmentationQueue: Special enqueue", encoder.encode(TTS_SPECIAL_TOKEN))
|
||||
pendingSpecials.push(ctx.data.value)
|
||||
controller.enqueue(encoder.encode(TTS_SPECIAL_TOKEN))
|
||||
}
|
||||
},
|
||||
],
|
||||
})
|
||||
|
||||
@@ -1,51 +1,56 @@
|
||||
export const EMOTION_HAPPY = '<|EMOTE_HAPPY|>'
|
||||
export const EMOTION_SAD = '<|EMOTE_SAD|>'
|
||||
export const EMOTION_ANGRY = '<|EMOTE_ANGRY|>'
|
||||
export const EMOTION_THINK = '<|EMOTE_THINK|>'
|
||||
export const EMOTION_SURPRISE = '<|EMOTE_SURPRISE|>'
|
||||
export const EMOTION_AWKWARD = '<|EMOTE_AWKWARD|>'
|
||||
export const EMOTION_QUESTION = '<|EMOTE_QUESTION|>'
|
||||
|
||||
export enum Emotion {
|
||||
Idle = '<|EMOTE_NEUTRAL|>',
|
||||
Happy = '<|EMOTE_HAPPY|>',
|
||||
Sad = '<|EMOTE_SAD|>',
|
||||
Angry = '<|EMOTE_ANGRY|>',
|
||||
Think = '<|EMOTE_THINK|>',
|
||||
Surprise = '<|EMOTE_SURPRISE|>',
|
||||
Awkward = '<|EMOTE_AWKWARD|>',
|
||||
Question = '<|EMOTE_QUESTION|>',
|
||||
}
|
||||
|
||||
export const EMOTION_VALUES = Object.values(Emotion)
|
||||
|
||||
export const EmotionHappyMotionName = 'Happy'
|
||||
export const EmotionSadMotionName = 'Sad'
|
||||
export const EmotionAngryMotionName = 'Angry'
|
||||
export const EmotionAwkwardMotionName = 'Awkward'
|
||||
export const EmotionThinkMotionName = 'Think'
|
||||
export const EmotionSurpriseMotionName = 'Surprise'
|
||||
export const EmotionQuestionMotionName = 'Question'
|
||||
export const EmotionNeutralMotionName = 'Idle'
|
||||
|
||||
export const EMOTION_EmotionMotionName_value = {
|
||||
[Emotion.Happy]: EmotionHappyMotionName,
|
||||
[Emotion.Sad]: EmotionSadMotionName,
|
||||
[Emotion.Angry]: EmotionAngryMotionName,
|
||||
[Emotion.Think]: EmotionThinkMotionName,
|
||||
[Emotion.Surprise]: EmotionSurpriseMotionName,
|
||||
[Emotion.Awkward]: EmotionAwkwardMotionName,
|
||||
[Emotion.Question]: EmotionQuestionMotionName,
|
||||
[Emotion.Idle]: EmotionNeutralMotionName,
|
||||
}
|
||||
|
||||
export const EMOTION_VRMExpressionName_value = {
|
||||
[Emotion.Happy]: 'happy',
|
||||
[Emotion.Sad]: 'sad',
|
||||
[Emotion.Angry]: 'angry',
|
||||
[Emotion.Think]: undefined,
|
||||
[Emotion.Surprise]: 'surprised',
|
||||
[Emotion.Awkward]: undefined,
|
||||
[Emotion.Question]: undefined,
|
||||
[Emotion.Idle]: undefined,
|
||||
} satisfies Record<Emotion, string | undefined>
|
||||
export const EMOTION_HAPPY = '<|EMOTE_HAPPY|>'
|
||||
export const EMOTION_SAD = '<|EMOTE_SAD|>'
|
||||
export const EMOTION_ANGRY = '<|EMOTE_ANGRY|>'
|
||||
export const EMOTION_THINK = '<|EMOTE_THINK|>'
|
||||
export const EMOTION_SURPRISE = '<|EMOTE_SURPRISED|>'
|
||||
export const EMOTION_AWKWARD = '<|EMOTE_AWKWARD|>'
|
||||
export const EMOTION_QUESTION = '<|EMOTE_QUESTION|>'
|
||||
export const EMOTION_CURIOUS = '<|EMOTE_CURIOUS|>'
|
||||
|
||||
export enum Emotion {
|
||||
Idle = '<|EMOTE_NEUTRAL|>',
|
||||
Happy = '<|EMOTE_HAPPY|>',
|
||||
Sad = '<|EMOTE_SAD|>',
|
||||
Angry = '<|EMOTE_ANGRY|>',
|
||||
Think = '<|EMOTE_THINK|>',
|
||||
Surprise = '<|EMOTE_SURPRISED|>',
|
||||
Awkward = '<|EMOTE_AWKWARD|>',
|
||||
Question = '<|EMOTE_QUESTION|>',
|
||||
Curious = '<|EMOTE_CURIOUS|>',
|
||||
}
|
||||
|
||||
export const EMOTION_VALUES = Object.values(Emotion)
|
||||
|
||||
export const EmotionHappyMotionName = 'Happy'
|
||||
export const EmotionSadMotionName = 'Sad'
|
||||
export const EmotionAngryMotionName = 'Angry'
|
||||
export const EmotionAwkwardMotionName = 'Awkward'
|
||||
export const EmotionThinkMotionName = 'Think'
|
||||
export const EmotionSurpriseMotionName = 'Surprise'
|
||||
export const EmotionQuestionMotionName = 'Question'
|
||||
export const EmotionNeutralMotionName = 'Idle'
|
||||
export const EmotionCuriousMotionName = 'Curious'
|
||||
|
||||
export const EMOTION_EmotionMotionName_value = {
|
||||
[Emotion.Happy]: EmotionHappyMotionName,
|
||||
[Emotion.Sad]: EmotionSadMotionName,
|
||||
[Emotion.Angry]: EmotionAngryMotionName,
|
||||
[Emotion.Think]: EmotionThinkMotionName,
|
||||
[Emotion.Surprise]: EmotionSurpriseMotionName,
|
||||
[Emotion.Awkward]: EmotionAwkwardMotionName,
|
||||
[Emotion.Question]: EmotionQuestionMotionName,
|
||||
[Emotion.Idle]: EmotionNeutralMotionName,
|
||||
[Emotion.Curious]: EmotionCuriousMotionName,
|
||||
}
|
||||
|
||||
export const EMOTION_VRMExpressionName_value = {
|
||||
[Emotion.Happy]: 'happy',
|
||||
[Emotion.Sad]: 'sad',
|
||||
[Emotion.Angry]: 'angry',
|
||||
[Emotion.Think]: undefined,
|
||||
[Emotion.Surprise]: 'surprised',
|
||||
[Emotion.Awkward]: undefined,
|
||||
[Emotion.Question]: undefined,
|
||||
[Emotion.Idle]: undefined,
|
||||
[Emotion.Curious]: 'surprised',
|
||||
} satisfies Record<Emotion, string | undefined>
|
||||
|
||||
+265
-214
@@ -1,214 +1,265 @@
|
||||
import type { ReaderLike } from 'clustr'
|
||||
|
||||
import { readGraphemeClusters } from 'clustr'
|
||||
|
||||
// A special character to instruct the TTS pipeline to flush
|
||||
export const TTS_FLUSH_INSTRUCTION = '\u200B'
|
||||
|
||||
const keptPunctuations = new Set('?？!！')
|
||||
const hardPunctuations = new Set('.。?？!！…⋯～~\n\t\r')
|
||||
const softPunctuations = new Set(',，、–—:：;；《》「」')
|
||||
|
||||
export interface TTSInputChunk {
|
||||
text: string
|
||||
words: number
|
||||
reason: 'boost' | 'limit' | 'hard' | 'flush'
|
||||
}
|
||||
|
||||
export interface TTSInputChunkOptions {
|
||||
boost?: number
|
||||
minimumWords?: number
|
||||
maximumWords?: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Processes the input string or UTF-8 byte stream reader into chunks suitable for TTS synthesis.
|
||||
*
|
||||
* @param input A string or a ReaderLike object that reads from an underlying UTF-8 byte stream.
|
||||
* @param options
|
||||
* @param options.boost Specifies the number of chunks to yield using greedier rules. This may help
|
||||
* reduce the initial delay when processing long input text.
|
||||
* @param options.minimumWords Minimum number of words in a chunk.
|
||||
* @param options.maximumWords Maximum number of words in a chunk.
|
||||
*/
|
||||
export async function* chunkTTSInput(input: string | ReaderLike, options?: TTSInputChunkOptions): AsyncGenerator<TTSInputChunk, void, unknown> {
|
||||
const {
|
||||
boost = 2,
|
||||
minimumWords = 4,
|
||||
maximumWords = 12,
|
||||
} = options ?? {}
|
||||
|
||||
const iterator = readGraphemeClusters(
|
||||
typeof input === 'string'
|
||||
? new ReadableStream({
|
||||
start(controller) {
|
||||
controller.enqueue(new TextEncoder().encode(input))
|
||||
controller.close()
|
||||
},
|
||||
}).getReader()
|
||||
: input,
|
||||
)
|
||||
|
||||
const segmenter = new Intl.Segmenter(undefined, { granularity: 'word' }) // I love Intl.Segmenter
|
||||
|
||||
let yieldCount = 0
|
||||
let buffer = ''
|
||||
let chunk = ''
|
||||
let chunkWordsCount = 0
|
||||
|
||||
let previousValue: string | undefined
|
||||
let current = await iterator.next()
|
||||
|
||||
while (!current.done) {
|
||||
let value = current.value
|
||||
|
||||
if (value.length > 1) {
|
||||
previousValue = value
|
||||
current = await iterator.next()
|
||||
continue
|
||||
}
|
||||
|
||||
const flush = value === TTS_FLUSH_INSTRUCTION
|
||||
const hard = hardPunctuations.has(value)
|
||||
const soft = softPunctuations.has(value)
|
||||
const kept = keptPunctuations.has(value)
|
||||
let next: IteratorResult<string, any> | undefined
|
||||
let afterNext: IteratorResult<string, any> | undefined
|
||||
|
||||
if (flush || hard || soft) {
|
||||
switch (value) {
|
||||
case '.':
|
||||
case ',': {
|
||||
if (previousValue !== undefined && /\d/.test(previousValue)) {
|
||||
next = await iterator.next()
|
||||
if (!next.done && next.value && /\d/.test(next.value)) {
|
||||
// This dot could be a decimal point, so we skip it (don't fully skip! keep in tts input!)
|
||||
|
||||
// REVIEW: @Lilia-Chen I think we need to remove the below line
|
||||
// 1. Not sensible to let the previousValue to be the value of the next value
|
||||
// 2. after the continue (jump to the bottom of the while loop), the previousValue will be reset to value, so this value assignment doesn't work at all
|
||||
// previousValue = next.value
|
||||
|
||||
// If we don't append value ("." or ","), we will lose the decimal point, and 2.5 will become 25, which hugely impacted the tts...
|
||||
buffer += value
|
||||
current = next
|
||||
next = undefined
|
||||
continue
|
||||
}
|
||||
}
|
||||
else if (value === '.') {
|
||||
// trying catch '...' and turn it into U+2026
|
||||
next = await iterator.next()
|
||||
if (!next.done && next.value && next.value === '.') {
|
||||
afterNext = await iterator.next()
|
||||
// If this is a '...' repalce the current value
|
||||
if (!afterNext.done && afterNext.value && afterNext.value === '.') {
|
||||
value = '…'
|
||||
next = undefined
|
||||
afterNext = undefined
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (buffer.length === 0) {
|
||||
previousValue = value
|
||||
current = await iterator.next()
|
||||
continue
|
||||
}
|
||||
|
||||
const words = [...segmenter.segment(buffer)].filter(w => w.isWordLike)
|
||||
|
||||
if (chunkWordsCount > minimumWords && chunkWordsCount + words.length > maximumWords) {
|
||||
const text = kept ? chunk.trim() + value : chunk.trim()
|
||||
yield {
|
||||
text,
|
||||
words: chunkWordsCount,
|
||||
reason: 'limit',
|
||||
}
|
||||
yieldCount++
|
||||
chunk = ''
|
||||
chunkWordsCount = 0
|
||||
}
|
||||
|
||||
chunk += buffer + value
|
||||
chunkWordsCount += words.length
|
||||
buffer = ''
|
||||
|
||||
if (flush || hard || chunkWordsCount > maximumWords || yieldCount < boost) {
|
||||
const text = chunk.trim()
|
||||
yield {
|
||||
text,
|
||||
words: chunkWordsCount,
|
||||
reason: flush ? 'flush' : hard ? 'hard' : chunkWordsCount > maximumWords ? 'limit' : 'boost',
|
||||
}
|
||||
yieldCount++
|
||||
chunk = ''
|
||||
chunkWordsCount = 0
|
||||
}
|
||||
|
||||
previousValue = value
|
||||
// If next had been read during decimal recognition or "..." recognition
|
||||
if (next !== undefined) {
|
||||
// If afterNext had also been read
|
||||
if (afterNext !== undefined) {
|
||||
// The only case that the program will come to this place is:
|
||||
// "x..y" and y is not a "." so "..." is not recognised
|
||||
// ".." is not a legal punctuation so ignored.
|
||||
// The first "." had been consumed during hard flush
|
||||
// next.value = second '.'
|
||||
// afterNext.value = 'y', so we just need to care about 'y'
|
||||
// In case 'y' is not just a useless blank. It's OK if 'y' is blank since we have `text = chunk.trim()`
|
||||
current = afterNext
|
||||
next = undefined
|
||||
afterNext = undefined
|
||||
}
|
||||
else {
|
||||
// Only next had been read
|
||||
// This is the case where "x.y" and x is a number but y is not
|
||||
// In case 'y' is not just a useless blank. It's OK if 'y' is blank since we have `text = chunk.trim()`
|
||||
current = next
|
||||
next = undefined
|
||||
}
|
||||
}
|
||||
else {
|
||||
// No next nor afterNext, so run `iterator.next()`
|
||||
current = await iterator.next()
|
||||
}
|
||||
// No need to do anything with buffer, just jump to the next loop
|
||||
continue
|
||||
}
|
||||
// If normal character enters, add it to the buffer and move to the next
|
||||
buffer += value
|
||||
previousValue = value
|
||||
next = await iterator.next()
|
||||
current = next
|
||||
}
|
||||
|
||||
// TODO: remove later
|
||||
// eslint-disable-next-line no-console
|
||||
console.debug('while loop ends, chunk/buffer:', chunk, buffer)
|
||||
if (chunk.length > 0 || buffer.length > 0) {
|
||||
const text = (chunk + buffer).trim()
|
||||
yield {
|
||||
text,
|
||||
words: chunkWordsCount + [...segmenter.segment(buffer)].filter(w => w.isWordLike).length,
|
||||
reason: 'flush',
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function chunkEmitter(reader: ReaderLike, handler: (chunk: string) => Promise<void> | void) {
|
||||
try {
|
||||
for await (const chunk of chunkTTSInput(reader)) {
|
||||
// TODO: remove later
|
||||
// eslint-disable-next-line no-console
|
||||
console.debug('chunk to be pushed: ', chunk)
|
||||
await handler(chunk.text)
|
||||
}
|
||||
}
|
||||
catch (e) {
|
||||
console.error('Error chunking stream to TTS queue:', e)
|
||||
}
|
||||
}
|
||||
import type { ReaderLike } from 'clustr'
|
||||
|
||||
import { readGraphemeClusters } from 'clustr'
|
||||
|
||||
// A special character to instruct the TTS pipeline to flush
|
||||
export const TTS_FLUSH_INSTRUCTION = '\u200B'
|
||||
// This is for special literals
|
||||
export const TTS_SPECIAL_TOKEN = '\u2063'
|
||||
|
||||
const keptPunctuations = new Set('?？!！')
|
||||
const hardPunctuations = new Set('.。?？!！…⋯～~\n\t\r')
|
||||
const softPunctuations = new Set(',，、–—:：;；《》「」')
|
||||
|
||||
export interface TTSInputChunk {
|
||||
text: string
|
||||
words: number
|
||||
reason: 'boost' | 'limit' | 'hard' | 'flush' | 'special'
|
||||
}
|
||||
|
||||
export interface TTSInputChunkOptions {
|
||||
boost?: number
|
||||
minimumWords?: number
|
||||
maximumWords?: number
|
||||
}
|
||||
|
||||
// New output type for tts, metaData (special token) contained
|
||||
export interface TTSChunkItem {
|
||||
chunk: string
|
||||
special: string | null
|
||||
}
|
||||
|
||||
/**
|
||||
* Processes the input string or UTF-8 byte stream reader into chunks suitable for TTS synthesis.
|
||||
*
|
||||
* @param input A string or a ReaderLike object that reads from an underlying UTF-8 byte stream.
|
||||
* @param options
|
||||
* @param options.boost Specifies the number of chunks to yield using greedier rules. This may help
|
||||
* reduce the initial delay when processing long input text.
|
||||
* @param options.minimumWords Minimum number of words in a chunk.
|
||||
* @param options.maximumWords Maximum number of words in a chunk.
|
||||
*/
|
||||
export async function* chunkTTSInput(
|
||||
input: string | ReaderLike,
|
||||
options?: TTSInputChunkOptions,
|
||||
): AsyncGenerator<TTSInputChunk, void, unknown> {
|
||||
const {
|
||||
boost = 2,
|
||||
minimumWords = 4,
|
||||
maximumWords = 12,
|
||||
} = options ?? {}
|
||||
|
||||
const iterator = readGraphemeClusters(
|
||||
typeof input === 'string'
|
||||
? new ReadableStream({
|
||||
start(controller) {
|
||||
controller.enqueue(new TextEncoder().encode(input))
|
||||
controller.close()
|
||||
},
|
||||
}).getReader()
|
||||
: input,
|
||||
)
|
||||
|
||||
const segmenter = new Intl.Segmenter(undefined, { granularity: 'word' }) // I love Intl.Segmenter
|
||||
|
||||
let yieldCount = 0
|
||||
let buffer = ''
|
||||
let chunk = ''
|
||||
let chunkWordsCount = 0
|
||||
|
||||
let previousValue: string | undefined
|
||||
let current = await iterator.next()
|
||||
|
||||
while (!current.done) {
|
||||
let value = current.value
|
||||
|
||||
if (value.length > 1) {
|
||||
previousValue = value
|
||||
current = await iterator.next()
|
||||
continue
|
||||
}
|
||||
|
||||
const flush = value === TTS_FLUSH_INSTRUCTION
|
||||
const special = value === TTS_SPECIAL_TOKEN
|
||||
const hard = hardPunctuations.has(value)
|
||||
const soft = softPunctuations.has(value)
|
||||
const kept = keptPunctuations.has(value)
|
||||
let next: IteratorResult<string, any> | undefined
|
||||
let afterNext: IteratorResult<string, any> | undefined
|
||||
|
||||
if (flush || special || hard || soft) {
|
||||
switch (value) {
|
||||
case '.':
|
||||
case ',': {
|
||||
if (previousValue !== undefined && /\d/.test(previousValue)) {
|
||||
next = await iterator.next()
|
||||
if (!next.done && next.value && /\d/.test(next.value)) {
|
||||
// This dot could be a decimal point, so we skip it (don't fully skip! keep in tts input!)
|
||||
|
||||
// REVIEW: @Lilia-Chen I think we need to remove the below line
|
||||
// 1. Not sensible to let the previousValue to be the value of the next value
|
||||
// 2. after the continue (jump to the bottom of the while loop), the previousValue will be reset to value, so this value assignment doesn't work at all
|
||||
// previousValue = next.value
|
||||
|
||||
// If we don't append value ("." or ","), we will lose the decimal point, and 2.5 will become 25, which hugely impacted the tts...
|
||||
buffer += value
|
||||
current = next
|
||||
next = undefined
|
||||
continue
|
||||
}
|
||||
}
|
||||
else if (value === '.') {
|
||||
// trying catch '...' and turn it into U+2026
|
||||
next = await iterator.next()
|
||||
if (!next.done && next.value && next.value === '.') {
|
||||
afterNext = await iterator.next()
|
||||
// If this is a '...' repalce the current value
|
||||
if (!afterNext.done && afterNext.value && afterNext.value === '.') {
|
||||
value = '…'
|
||||
next = undefined
|
||||
afterNext = undefined
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (buffer.length === 0) {
|
||||
if (special) {
|
||||
// Special token without buffered text still needs to be surfaced so
|
||||
// downstream queues can process delay/emotion markers.
|
||||
yield {
|
||||
text: '',
|
||||
words: 0,
|
||||
reason: 'special',
|
||||
}
|
||||
yieldCount++
|
||||
chunkWordsCount = 0
|
||||
}
|
||||
|
||||
previousValue = value
|
||||
current = await iterator.next()
|
||||
continue
|
||||
}
|
||||
|
||||
const words = [...segmenter.segment(buffer)].filter(w => w.isWordLike)
|
||||
|
||||
if (chunkWordsCount > minimumWords && chunkWordsCount + words.length > maximumWords) {
|
||||
const text = kept ? chunk.trim() + value : chunk.trim()
|
||||
yield {
|
||||
text,
|
||||
words: chunkWordsCount,
|
||||
reason: 'limit',
|
||||
}
|
||||
yieldCount++
|
||||
chunk = ''
|
||||
chunkWordsCount = 0
|
||||
}
|
||||
|
||||
chunk += buffer + value
|
||||
chunkWordsCount += words.length
|
||||
buffer = ''
|
||||
|
||||
if (special) {
|
||||
const text = chunk.slice(0, -1).trim()
|
||||
yield {
|
||||
text,
|
||||
words: chunkWordsCount,
|
||||
reason: 'special',
|
||||
}
|
||||
yieldCount++
|
||||
chunk = ''
|
||||
chunkWordsCount = 0
|
||||
}
|
||||
else if (flush || hard || chunkWordsCount > maximumWords || yieldCount < boost) {
|
||||
const text = chunk.trim()
|
||||
yield {
|
||||
text,
|
||||
words: chunkWordsCount,
|
||||
reason: flush ? 'flush' : hard ? 'hard' : chunkWordsCount > maximumWords ? 'limit' : 'boost',
|
||||
}
|
||||
yieldCount++
|
||||
chunk = ''
|
||||
chunkWordsCount = 0
|
||||
}
|
||||
|
||||
previousValue = value
|
||||
// If next had been read during decimal recognition or "..." recognition
|
||||
if (next !== undefined) {
|
||||
// If afterNext had also been read
|
||||
if (afterNext !== undefined) {
|
||||
// The only case that the program will come to this place is:
|
||||
// "x..y" and y is not a "." so "..." is not recognised
|
||||
// ".." is not a legal punctuation so ignored.
|
||||
// The first "." had been consumed during hard flush
|
||||
// next.value = second '.'
|
||||
// afterNext.value = 'y', so we just need to care about 'y'
|
||||
// In case 'y' is not just a useless blank. It's OK if 'y' is blank since we have `text = chunk.trim()`
|
||||
current = afterNext
|
||||
next = undefined
|
||||
afterNext = undefined
|
||||
}
|
||||
else {
|
||||
// Only next had been read
|
||||
// This is the case where "x.y" and x is a number but y is not
|
||||
// In case 'y' is not just a useless blank. It's OK if 'y' is blank since we have `text = chunk.trim()`
|
||||
current = next
|
||||
next = undefined
|
||||
}
|
||||
}
|
||||
else {
|
||||
// No next nor afterNext, so run `iterator.next()`
|
||||
current = await iterator.next()
|
||||
}
|
||||
// No need to do anything with buffer, just jump to the next loop
|
||||
continue
|
||||
}
|
||||
// If normal character enters, add it to the buffer and move to the next
|
||||
buffer += value
|
||||
previousValue = value
|
||||
next = await iterator.next()
|
||||
current = next
|
||||
}
|
||||
|
||||
// TODO: remove later
|
||||
// eslint-disable-next-line no-console
|
||||
console.debug('while loop ends, chunk/buffer:', chunk, buffer)
|
||||
if (chunk.length > 0 || buffer.length > 0) {
|
||||
const text = (chunk + buffer).trim()
|
||||
yield {
|
||||
text,
|
||||
words: chunkWordsCount + [...segmenter.segment(buffer)].filter(w => w.isWordLike).length,
|
||||
reason: 'flush',
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function chunkEmitter(
|
||||
reader: ReaderLike,
|
||||
pendingSpecials: string[],
|
||||
handler: (ttsSegment: TTSChunkItem) => Promise<void> | void,
|
||||
) {
|
||||
const sanitizeChunk = (text: string) =>
|
||||
text
|
||||
.replaceAll(TTS_SPECIAL_TOKEN, '')
|
||||
.replaceAll(TTS_FLUSH_INSTRUCTION, '')
|
||||
.trim()
|
||||
|
||||
try {
|
||||
for await (const chunk of chunkTTSInput(reader)) {
|
||||
// TODO: remove later
|
||||
|
||||
if (chunk.reason === 'special') {
|
||||
const specialToken = pendingSpecials.shift()
|
||||
// console.debug("special yield:", specialToken)
|
||||
await handler({ chunk: sanitizeChunk(chunk.text), special: specialToken ?? null })
|
||||
}
|
||||
else {
|
||||
await handler({ chunk: sanitizeChunk(chunk.text), special: null } as TTSChunkItem)
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (e) {
|
||||
console.error('Error chunking stream to TTS queue:', e)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user