diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 000000000..9d08a1a82 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,9 @@ +root = true + +[*] +charset = utf-8 +indent_style = space +indent_size = 2 +end_of_line = lf +insert_final_newline = true +trim_trailing_whitespace = true diff --git a/components/MainStage.vue b/components/MainStage.vue index a766d2445..0209c6eea 100644 --- a/components/MainStage.vue +++ b/components/MainStage.vue @@ -12,6 +12,7 @@ import Avatar from '../assets/live2d/models/hiyori_free_zh/avatar.png' import Live2DViewer from '../components/Live2DViewer.vue' import BasicTextarea from '../components/BasicTextarea.vue' import { useLLM } from '../stores/llm' +import { useQueue } from '../composables/queue' import AudioWaveform from './AudioWaveform.vue' interface Message { @@ -22,8 +23,8 @@ interface Message { const llm = useLLM() const { audioContext } = useAudioContext() -const openAIAPIKey = useLocalStorage('openai-api-key', '') -const openAIAPIBaseURL = useLocalStorage('openai-api-base-url', 'https://api.openai.com/v1') +const openAiApiKey = useLocalStorage('openai-api-key', '') +const openAiApiBaseURL = useLocalStorage('openai-api-base-url', 'https://api.openai.com/v1') const openAIModel = useLocalStorage('openai-model', '') const mouthOpenSize = ref(0) @@ -52,6 +53,81 @@ const model = computed({ }, }) +const temp = ref('') + +const audioQueue = useQueue<{ audioBuffer: AudioBuffer, text: string }>({ + handlers: [ + (ctx) => { + return new Promise((resolve) => { + // Create an AudioBufferSourceNode + const source = audioContext.createBufferSource() + source.buffer = ctx.data.audioBuffer + + // Connect the source to the AudioContext's destination (the speakers) + source.connect(audioContext.destination) + // Connect the source to the analyzer + source.connect(audioWaveformRef.value!.analyser()) + + // Start playing the audio + speaking.value = true + source.start(0) + source.onended = () => { + speaking.value = false + resolve() + } + }) + }, + ], +}) + +const ttsQueue = useQueue({ + handlers: [ + async (ctx) => { + const audioBuffer = await streamSpeech(ctx.data) + audioQueue.add({ audioBuffer, text: ctx.data }) + }, + ], +}) + +const messageContentQueue = useQueue({ + handlers: [ + async (ctx) => { + if (ctx.data === '||') { + const content = temp.value.trim() + if (content) + ttsQueue.add(content) + + temp.value = '' + return + } + + const endMarker = ['.', '?', '!'] + + let newEndPartDiscovered = false + + for (const marker of endMarker) { + if (!ctx.data.includes(marker)) + continue + + // find the end of the sentence and push it to the queue with temp + const periodIndex = ctx.data.indexOf(marker) + // split + const beforePeriod = ctx.data.slice(0, periodIndex + 1) + const afterPeriod = ctx.data.slice(periodIndex + 1) + + temp.value += beforePeriod + ttsQueue.add(temp.value.trim()) + temp.value = afterPeriod + + newEndPartDiscovered = true + } + + if (!newEndPartDiscovered) + temp.value += ctx.data + }, + ], +}) + async function streamSpeech(text: string) { const res = await ofetch('/api/v1/llm/voice/text-to-speech', { body: { @@ -63,23 +139,7 @@ async function streamSpeech(text: string) { }) // Decode the ArrayBuffer into an AudioBuffer - const audioBuffer = await audioContext.decodeAudioData(res) - - // Create an AudioBufferSourceNode - const source = audioContext.createBufferSource() - source.buffer = audioBuffer - - // Connect the source to the AudioContext's destination (the speakers) - source.connect(audioContext.destination) - // Connect the source to the analyzer - source.connect(audioWaveformRef.value!.analyser()) - - // Start playing the audio - speaking.value = true - source.start(0) - source.onended = () => { - speaking.value = false - } + return await audioContext.decodeAudioData(res) } function getVolumeWithLinearNormalize() { @@ -155,12 +215,16 @@ function onSendMessage(sendingMessage: string) { messages.value.push({ role: 'user', content: sendingMessage }) messages.value.push(message) const index = messages.value.length - 1 + const textParts: string[] = [] llm.stream(model.value, sendingMessage).then(async (res) => { - for await (const textPart of res.textStream) + for await (const textPart of res.textStream) { messages.value[index].content += textPart + messageContentQueue.add(textPart) + textParts.push(textPart) + } - await streamSpeech(messages.value[index].content) + messageContentQueue.add('||') }) input.value = '' @@ -175,20 +239,20 @@ function fromMarkdownToHTML(markdown: string) { .toString() } -watch(openAIAPIKey, (value) => { +watch(openAiApiKey, (value) => { llm.setupOpenAI({ apiKey: value, - baseURL: openAIAPIBaseURL.value, + baseURL: openAiApiBaseURL.value, }) }) onMounted(async () => { - if (!openAIAPIKey.value) + if (!openAiApiKey.value) return llm.setupOpenAI({ - apiKey: openAIAPIKey.value, - baseURL: openAIAPIBaseURL.value, + apiKey: openAiApiKey.value, + baseURL: openAiApiBaseURL.value, }) const fetchedModels = await llm.models() @@ -205,14 +269,14 @@ onUnmounted(() => {
@@ -220,11 +284,11 @@ onUnmounted(() => {
+
+ {{ mouthOpenSize }}
- -
diff --git a/composables/count.ts b/composables/count.ts deleted file mode 100644 index 6122453f4..000000000 --- a/composables/count.ts +++ /dev/null @@ -1,16 +0,0 @@ -export function useCount() { - const count = useState('count', () => Math.round(Math.random() * 20)) - - function inc() { - count.value += 1 - } - function dec() { - count.value -= 1 - } - - return { - count, - inc, - dec, - } -} diff --git a/composables/queue.ts b/composables/queue.ts new file mode 100644 index 000000000..b292fb5c6 --- /dev/null +++ b/composables/queue.ts @@ -0,0 +1,94 @@ +import { ref } from 'vue' +import type { Ref } from 'vue' + +export interface HandlerContext { + data: T + itemsToBeProcessed: () => number +} + +interface Events { + add: Array<(payload: T) => void> + pick: Array<(payload: T) => void> + processing: Array<(payload: T, handler: (param: HandlerContext) => Promise) => void> + error: Array<(payload: T, error: Error, handler: (param: HandlerContext) => Promise) => void> + processed: Array<(payload: T, result: R, handler: (param: HandlerContext) => Promise) => void> + done: Array<(payload: T) => void> +} + +export function useQueue(options: { + handlers: Array<(param: { + data: T + itemsToBeProcessed: () => number + }) => Promise> +}) { + const queue = ref([]) as Ref + const isProcessing = ref(false) + const internalEventHandler: Events = { + add: [], + pick: [], + processing: [], + error: [], + processed: [], + done: [], + } + + function on>(eventName: E, handler: Events[E][number]) { + internalEventHandler[eventName].push(handler as any) + } + + function emit>(eventName: E, ...params: Parameters[E][number]>) { + const handlers = internalEventHandler[eventName] as Events[E] + handlers.forEach((handler) => { + (handler as any)(...params) + }) + } + + function add(payload: T) { + queue.value.push(payload) + emit('add', payload) + } + + function pick() { + const payload = queue.value.shift() + if (!payload) + return + + emit('pick', payload) + return payload + } + + async function handleItem() { + if (isProcessing.value) + return + + const payload = pick() + if (!payload) + return + + isProcessing.value = true + + for (const handler of options.handlers) { + emit('processing', payload, handler) + try { + const result = await handler({ data: payload, itemsToBeProcessed: () => queue.value.length }) + emit('processed', payload, result, handler) + } + catch (err) { + emit('error', payload, err as Error, handler) + continue + } + } + + isProcessing.value = false + emit('done', payload) + } + + on('add', handleItem) + on('done', handleItem) + + return { + add, + on, + queue, + } +} diff --git a/composables/user.ts b/composables/user.ts deleted file mode 100644 index c4440cd93..000000000 --- a/composables/user.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { acceptHMRUpdate, defineStore } from 'pinia' - -export const useUserStore = defineStore('user', () => { - /** - * Current named of the user. - */ - const savedName = ref('') - const previousNames = ref(new Set()) - - const usedNames = computed(() => Array.from(previousNames.value)) - const otherNames = computed(() => usedNames.value.filter(name => name !== savedName.value)) - - /** - * Changes the current name of the user and saves the one that was used - * before. - * - * @param name - new name to set - */ - function setNewName(name: string) { - if (savedName.value) - previousNames.value.add(savedName.value) - - savedName.value = name - } - - return { - setNewName, - otherNames, - savedName, - } -}) - -if (import.meta.hot) - import.meta.hot.accept(acceptHMRUpdate(useUserStore, import.meta.hot)) diff --git a/cspell.config.yaml b/cspell.config.yaml new file mode 100644 index 000000000..8e4c17304 --- /dev/null +++ b/cspell.config.yaml @@ -0,0 +1,15 @@ +version: "0.2" +ignorePaths: [] +dictionaryDefinitions: [] +dictionaries: [] +words: + - composables + - hiyori + - Neuro + - ofetch + - openai + - pixi + - rehype + - vueuse +ignoreWords: [] +import: [] diff --git a/pages/queue.vue b/pages/queue.vue new file mode 100644 index 000000000..6497e9c10 --- /dev/null +++ b/pages/queue.vue @@ -0,0 +1,161 @@ + + + diff --git a/server/api/v1/llm/voice/text-to-speech.ts b/server/api/v1/llm/voice/text-to-speech.ts index f11b99ac5..95512b8c2 100644 --- a/server/api/v1/llm/voice/text-to-speech.ts +++ b/server/api/v1/llm/voice/text-to-speech.ts @@ -7,7 +7,9 @@ export default defineEventHandler(async (event) => { }) const res = await client.generate({ - voice: 'Beatrice', + voice: 'Myriam', + // Beatrice is not 'childish' like the others + // voice: 'Beatrice', text: body.text, stream: true, model_id: 'eleven_multilingual_v2',