nanakura/moeka-project
1
0
Fork 0
Code Issues Pull Requests 1 Actions 2 Packages Projects Releases Wiki Activity

fix: parser issue, and audio dispatching issue

Browse Source
This commit is contained in:
Neko Ayaka
2024-12-09 23:43:48 +08:00
parent 26c37943bf
commit 3735157b42
9 changed files with 266 additions and 153 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cspell.config.yaml
+1
View File
@@ -32,6 +32,7 @@ words:
- kwaa
- live2dcubismcore
- live2dcubismframework
- Llmmarker
- Myriam
- Neko
- nekomeowww
package.json
-6
View File
@@ -53,12 +53,6 @@
"packages/*",
"docs"
],
"pnpm": {
"override": {
"@babel/preset-env": "7.26.0",
"workbox-build": "7.3.0"
}
},
"simple-git-hooks": {
"pre-commit": "pnpm lint-staged"
},
packages/stage/src/auto-imports.d.ts
Vendored
+2
View File
@@ -196,6 +196,7 @@ declare global {
const useLLM: typeof import('./stores/llm')['useLLM']
const useLastChanged: typeof import('@vueuse/core')['useLastChanged']
const useLink: typeof import('vue-router/auto')['useLink']
const useLlmmarkerParser: typeof import('./composables/llmmarkerParser')['useLlmmarkerParser']
const useLocalStorage: typeof import('@vueuse/core')['useLocalStorage']
const useMagicKeys: typeof import('@vueuse/core')['useMagicKeys']
const useManualRefHistory: typeof import('@vueuse/core')['useManualRefHistory']
@@ -503,6 +504,7 @@ declare module 'vue' {
readonly useLLM: UnwrapRef<typeof import('./stores/llm')['useLLM']>
readonly useLastChanged: UnwrapRef<typeof import('@vueuse/core')['useLastChanged']>
readonly useLink: UnwrapRef<typeof import('vue-router/auto')['useLink']>
readonly useLlmmarkerParser: UnwrapRef<typeof import('./composables/llmmarkerParser')['useLlmmarkerParser']>
readonly useLocalStorage: UnwrapRef<typeof import('@vueuse/core')['useLocalStorage']>
readonly useMagicKeys: UnwrapRef<typeof import('@vueuse/core')['useMagicKeys']>
readonly useManualRefHistory: UnwrapRef<typeof import('@vueuse/core')['useManualRefHistory']>
packages/stage/src/components/MainStage.vue
+19 -39
View File
@@ -17,7 +17,6 @@ import { useLLM } from '../stores/llm'
import { useSettings } from '../stores/settings'
import BasicTextarea from './BasicTextarea.vue'
// import AudioWaveform from './AudioWaveform.vue'
import Live2DViewer from './Live2DViewer.vue'
import Settings from './Settings.vue'
import ThreeDScene from './ThreeDScene.vue'
@@ -107,7 +106,6 @@ const ttsQueue = useQueue<string>({
voice: 'Myriam',
// Beatrice is not 'childish' like the others
// voice: 'Beatrice',
// text: body.text,
model_id: 'eleven_multilingual_v2',
voice_settings: {
stability: 0.4,
@@ -150,13 +148,13 @@ const emotionsQueue = useQueue<Emotion>({
],
})
const emotionMessageContentQueue = useEmotionsMessageQueue(emotionsQueue, messageContentQueue)
const emotionMessageContentQueue = useEmotionsMessageQueue(emotionsQueue)
emotionMessageContentQueue.onHandlerEvent('emotion', (emotion) => {
// eslint-disable-next-line no-console
console.debug('emotion detected', emotion)
})
const delaysQueue = useDelayMessageQueue(emotionMessageContentQueue)
const delaysQueue = useDelayMessageQueue()
delaysQueue.onHandlerEvent('delay', (delay) => {
// eslint-disable-next-line no-console
console.debug('delay detected', delay)
@@ -216,49 +214,31 @@ async function onSendMessage(sendingMessage: string) {
live2DViewerRef.value?.setMotion(EmotionThinkMotionName)
const res = await stream(openAiApiBaseURL.value, openAiApiKey.value, openAIModel.value.id, messages.value.slice(0, messages.value.length - 1))
let fullText = ''
enum States {
Literal = 'literal',
Special = 'special',
}
let state = States.Literal
let buffer = ''
const parser = useLlmmarkerParser({
onLiteral: async (literal) => {
await messageContentQueue.add(literal)
streamingMessage.value.content += literal
},
onSpecial: async (special) => {
await delaysQueue.add(special)
await emotionMessageContentQueue.add(special)
},
})
for await (const textPart of asyncIteratorFromReadableStream(res.textStream, async v => v)) {
for (const textSingleChar of textPart) {
let newState: States = state
if (textSingleChar === '<')
newState = States.Special
else if (textSingleChar === '>')
newState = States.Literal
if (state === States.Literal && newState === States.Special) {
streamingMessage.value.content += buffer
buffer = ''
}
if (state === States.Special && newState === States.Literal)
buffer = '' // Clear buffer when exiting Special state
if (state === States.Literal && newState === States.Literal) {
streamingMessage.value.content += textSingleChar
buffer = ''
}
await delaysQueue.add(textSingleChar)
state = newState
buffer += textSingleChar
}
fullText += textPart
await parser.consume(textPart)
}
if (buffer)
streamingMessage.value.content += buffer
await parser.end()
await delaysQueue.add(llmInferenceEndToken)
messageInput.value = ''
// eslint-disable-next-line no-console
console.debug('Full text:', fullText)
}
watch([openAiApiBaseURL, openAiApiKey], async ([baseUrl, apiKey]) => {
packages/stage/src/composables/llmmarkerParser.test.ts
+144
View File
@@ -0,0 +1,144 @@
import { describe, expect, it } from 'vitest'
import { useLlmmarkerParser } from './llmmarkerParser'
describe('useLlmmarkerParser', async () => {
it('should parse pure literals', async () => {
const fullText = 'Hello, world!'
const collectedLiterals: string[] = []
const collectedSpecials: string[] = []
const parser = useLlmmarkerParser({
onLiteral(literal) {
collectedLiterals.push(literal)
},
onSpecial(special) {
collectedSpecials.push(special)
},
})
for (const char of fullText) {
await parser.consume(char)
}
await parser.end()
expect(collectedLiterals).toEqual('Hello, world!'.split(''))
expect(collectedSpecials).toEqual([])
})
it('should parse pure specials', async () => {
const fullText = '<|Hello, world!|>'
const collectedLiterals: string[] = []
const collectedSpecials: string[] = []
const parser = useLlmmarkerParser({
onLiteral(literal) {
collectedLiterals.push(literal)
},
onSpecial(special) {
collectedSpecials.push(special)
},
})
for (const char of fullText) {
await parser.consume(char)
}
await parser.end()
expect(collectedLiterals).toEqual([])
expect(collectedSpecials).toEqual(['<|Hello, world!|>'])
})
it('should not include unfinished special', async () => {
const fullText = '<|Hello, world'
const collectedLiterals: string[] = []
const collectedSpecials: string[] = []
const parser = useLlmmarkerParser({
onLiteral(literal) {
collectedLiterals.push(literal)
},
onSpecial(special) {
collectedSpecials.push(special)
},
})
for (const char of fullText) {
await parser.consume(char)
}
await parser.end()
expect(collectedLiterals).toEqual([])
expect(collectedSpecials).toEqual([])
})
it('should parse with mixed input, ends with special', async () => {
const fullText = 'This is sentence 1, <|HELLO|> and this is sentence 2.<|WORLD|>'
const collectedLiterals: string[] = []
const collectedSpecials: string[] = []
const parser = useLlmmarkerParser({
onLiteral(literal) {
collectedLiterals.push(literal)
},
onSpecial(special) {
collectedSpecials.push(special)
},
})
for (const char of fullText) {
await parser.consume(char)
}
await parser.end()
expect(collectedLiterals).toEqual([...'This is sentence 1, '.split(''), ...' and this is sentence 2.'.split('')])
expect(collectedSpecials).toEqual(['<|HELLO|>', '<|WORLD|>'])
})
it('should parse correctly', async () => {
const testCases: { input: string, expectedLiterals: string[], expectedSpecials: string[] }[] = [
{
input: `<|A|> Wow, hello there!`,
expectedLiterals: ' Wow, hello there!'.split(''),
expectedSpecials: ['<|A|>'],
},
{
input: `<|A|> Hello!`,
expectedLiterals: ' Hello!'.split(''),
expectedSpecials: ['<|A|>'],
},
{
input: `<|A|> Hello! <|B|>`,
expectedLiterals: ' Hello! '.split(''),
expectedSpecials: ['<|A|>', '<|B|>'],
},
]
for (const tc of testCases) {
const { input, expectedLiterals, expectedSpecials } = tc
const collectedLiterals: string[] = []
const collectedSpecials: string[] = []
const parser = useLlmmarkerParser({
onLiteral(literal) {
collectedLiterals.push(literal)
},
onSpecial(special) {
collectedSpecials.push(special)
},
})
for (const char of input) {
await parser.consume(char)
}
await parser.end()
expect(collectedLiterals).toEqual(expectedLiterals)
expect(collectedSpecials).toEqual(expectedSpecials)
}
})
})
packages/stage/src/composables/llmmarkerParser.ts
+86
View File
@@ -0,0 +1,86 @@
enum States {
Literal = 'literal',
Special = 'special',
}
function peek(array: string, index: number, offset: number): string | undefined {
if (index + offset < 0 || index + offset >= (array.length - 1))
return ''
return array[index + offset]
}
export function useLlmmarkerParser(options: {
onLiteral?: (literal: string) => void | Promise<void>
onSpecial?: (special: string) => void | Promise<void>
}) {
let state = States.Literal
let buffer = ''
return {
async consume(textPart: string) {
for (let i = 0; i < textPart.length; i++) {
let current = textPart[i]
let newState: States = state
// read
if (current === '<' && peek(textPart, i, 1) === '|') {
current += peek(textPart, i, 1)
newState = States.Special
i++
}
else if (current === '|' && peek(textPart, i, 1) === '>') {
current += peek(textPart, i, 1)
newState = States.Literal
i++
}
else if (current === '<') {
newState = States.Special
}
else if (current === '>') {
newState = States.Literal
}
// handle
if (state === States.Literal && newState === States.Special) {
if (buffer !== '') {
await options.onLiteral?.(buffer)
buffer = ''
}
}
else if (state === States.Special && newState === States.Literal) {
if (buffer !== '') {
buffer += current
await options.onSpecial?.(buffer)
buffer = '' // Clear buffer when exiting Special state
}
}
if (state === States.Literal && newState === States.Literal) {
await options.onLiteral?.(current)
buffer = ''
}
else if (state === States.Special && newState === States.Literal) {
buffer = ''
}
else {
buffer += current
}
state = newState
}
},
async end() {
if (buffer !== '') {
if (state === States.Literal) {
await options.onLiteral?.(buffer)
}
else {
if (buffer.endsWith('|>')) {
await options.onSpecial?.(buffer)
}
}
}
},
}
}
packages/stage/src/composables/queues.ts
+8 -107
View File
@@ -5,56 +5,30 @@ import { llmInferenceEndToken } from '../constants'
import { EMOTION_VALUES } from '../constants/emotions'
import { useQueue } from './queue'
export function useEmotionsMessageQueue(emotionsQueue: ReturnType<typeof useQueue<Emotion>>, messageContentQueue: ReturnType<typeof useQueue<string>>) {
export function useEmotionsMessageQueue(emotionsQueue: ReturnType<typeof useQueue<Emotion>>) {
function splitEmotion(content: string) {
for (const emotion of EMOTION_VALUES) {
// doesn't include the emotion, continue
if (!content.includes(emotion))
continue
// find the emotion and push the content before the emotion to the queue
const emotionIndex = content.indexOf(emotion)
const beforeEmotion = content.slice(0, emotionIndex)
const afterEmotion = content.slice(emotionIndex + emotion.length)
return {
ok: true,
emotion: emotion as Emotion,
before: beforeEmotion,
after: afterEmotion,
}
}
return {
ok: false,
emotion: '' as Emotion,
before: content,
after: '',
}
}
const processed = ref<string>('')
return useQueue<string>({
handlers: [
async (ctx) => {
// inference ended, push the last content to the message queue
if (ctx.data.includes(llmInferenceEndToken)) {
const content = processed.value.trim()
if (content)
await messageContentQueue.add(content)
processed.value = ''
return
}
// if the message is an emotion, push the last content to the message queue
if (EMOTION_VALUES.includes(ctx.data as Emotion)) {
const content = processed.value.trim()
if (content)
await messageContentQueue.add(content)
processed.value = ''
ctx.emit('emotion', ctx.data as Emotion)
await emotionsQueue.add(ctx.data as Emotion)
@@ -62,34 +36,12 @@ export function useEmotionsMessageQueue(emotionsQueue: ReturnType<typeof useQueu
}
// otherwise we should process the message to find the emotions
{
// iterate through the message to find the emotions
const { ok, before, emotion, after } = splitEmotion(ctx.data)
const { ok, emotion } = splitEmotion(ctx.data)
if (ok) {
await messageContentQueue.add(before)
ctx.emit('emotion', emotion)
await emotionsQueue.add(emotion)
await messageContentQueue.add(after)
processed.value = ''
return
}
else {
// if none of the emotions are found, push the content to the temp queue
processed.value += ctx.data
}
}
// iterate through the message to find the emotions
{
const { ok, before, emotion, after } = splitEmotion(processed.value)
if (ok) {
await messageContentQueue.add(before)
ctx.emit('emotion', emotion)
await emotionsQueue.add(emotion)
await messageContentQueue.add(after)
processed.value = ''
}
}
},
@@ -97,15 +49,13 @@ export function useEmotionsMessageQueue(emotionsQueue: ReturnType<typeof useQueu
})
}
export function useDelayMessageQueue(useEmotionsMessageQueue: ReturnType<typeof useQueue<string>>) {
export function useDelayMessageQueue() {
function splitDelays(content: string) {
// doesn't include the emotion, continue
// doesn't include the delay, continue
if (!(/<\|DELAY:\d+\|>/i.test(content))) {
return {
ok: false,
delay: 0,
before: content,
after: '',
}
}
@@ -116,29 +66,21 @@ export function useDelayMessageQueue(useEmotionsMessageQueue: ReturnType<typeof
return {
ok: false,
delay: 0,
before: content,
after: '',
}
}
const delaySeconds = Number.parseFloat(delay)
const before = content.split(delayExecArray[0])[0]
const after = content.split(delayExecArray[0])[1]
if (delaySeconds <= 0 || Number.isNaN(delaySeconds)) {
return {
ok: true,
delay: 0,
before,
after,
}
}
return {
ok: true,
delay: delaySeconds,
before,
after,
}
}
@@ -146,55 +88,14 @@ export function useDelayMessageQueue(useEmotionsMessageQueue: ReturnType<typeof
return new Promise(resolve => setTimeout(resolve, ms))
}
const delaysQueueProcessedTemp = ref<string>('')
return useQueue<string>({
handlers: [
async (ctx) => {
// inference ended, push the last content to the message queue
if (ctx.data.includes(llmInferenceEndToken)) {
const content = delaysQueueProcessedTemp.value.trim()
if (content)
await useEmotionsMessageQueue.add(content)
delaysQueueProcessedTemp.value = ''
return
}
{
// iterate through the message to find the emotions
const { ok, before, delay, after } = splitDelays(ctx.data)
if (ok && before) {
await useEmotionsMessageQueue.add(before)
if (delay) {
ctx.emit('delay', delay)
await sleep(delay * 1000)
}
if (after)
await useEmotionsMessageQueue.add(after)
}
else {
// if none of the emotions are found, push the content to the temp queue
delaysQueueProcessedTemp.value += ctx.data
}
}
// iterate through the message to find the emotions
{
const { ok, before, delay, after } = splitDelays(delaysQueueProcessedTemp.value)
if (ok && before) {
await useEmotionsMessageQueue.add(before)
if (delay) {
ctx.emit('delay', delay)
await sleep(delay * 1000)
}
if (after)
await useEmotionsMessageQueue.add(after)
delaysQueueProcessedTemp.value = ''
}
const { ok, delay } = splitDelays(ctx.data)
if (ok) {
ctx.emit('delay', delay)
await sleep(delay * 1000)
}
},
],
packages/stage/vite.config.ts
+1 -1
View File
@@ -62,7 +62,7 @@ export default defineConfig({
// https://github.com/posva/unplugin-vue-router
VueRouter({
extensions: ['.vue', '.md'],
dts: 'src/typed-router.d.ts',
dts: path.resolve(__dirname, 'src/typed-router.d.ts'),
}),
// https://github.com/JohnCampionJr/vite-plugin-vue-layouts
vitest.workspace.ts
+5
View File
@@ -0,0 +1,5 @@
import { defineWorkspace } from 'vitest/config'
export default defineWorkspace([
'packages/*',
])