refactor: split into components
This commit is contained in:
Neko Ayaka
Vendored
+6
@@ -138,6 +138,8 @@ declare global {
|
||||
const useBroadcastChannel: typeof import('@vueuse/core')['useBroadcastChannel']
|
||||
const useBrowserLocation: typeof import('@vueuse/core')['useBrowserLocation']
|
||||
const useCached: typeof import('@vueuse/core')['useCached']
|
||||
const useChat: typeof import('./composables/chat')['useChat']
|
||||
const useChatStore: typeof import('./stores/chat')['useChatStore']
|
||||
const useClipboard: typeof import('@vueuse/core')['useClipboard']
|
||||
const useClipboardItems: typeof import('@vueuse/core')['useClipboardItems']
|
||||
const useCloned: typeof import('@vueuse/core')['useCloned']
|
||||
@@ -250,6 +252,8 @@ declare global {
|
||||
const useShare: typeof import('@vueuse/core')['useShare']
|
||||
const useSlots: typeof import('vue')['useSlots']
|
||||
const useSorted: typeof import('@vueuse/core')['useSorted']
|
||||
const useSpeak: typeof import('./stores/audio')['useSpeak']
|
||||
const useSpeakingStore: typeof import('./stores/audio')['useSpeakingStore']
|
||||
const useSpeechRecognition: typeof import('@vueuse/core')['useSpeechRecognition']
|
||||
const useSpeechSynthesis: typeof import('@vueuse/core')['useSpeechSynthesis']
|
||||
const useStepper: typeof import('@vueuse/core')['useStepper']
|
||||
@@ -448,6 +452,7 @@ declare module 'vue' {
|
||||
readonly useBroadcastChannel: UnwrapRef<typeof import('@vueuse/core')['useBroadcastChannel']>
|
||||
readonly useBrowserLocation: UnwrapRef<typeof import('@vueuse/core')['useBrowserLocation']>
|
||||
readonly useCached: UnwrapRef<typeof import('@vueuse/core')['useCached']>
|
||||
readonly useChatStore: UnwrapRef<typeof import('./stores/chat')['useChatStore']>
|
||||
readonly useClipboard: UnwrapRef<typeof import('@vueuse/core')['useClipboard']>
|
||||
readonly useClipboardItems: UnwrapRef<typeof import('@vueuse/core')['useClipboardItems']>
|
||||
readonly useCloned: UnwrapRef<typeof import('@vueuse/core')['useCloned']>
|
||||
@@ -560,6 +565,7 @@ declare module 'vue' {
|
||||
readonly useShare: UnwrapRef<typeof import('@vueuse/core')['useShare']>
|
||||
readonly useSlots: UnwrapRef<typeof import('vue')['useSlots']>
|
||||
readonly useSorted: UnwrapRef<typeof import('@vueuse/core')['useSorted']>
|
||||
readonly useSpeakingStore: UnwrapRef<typeof import('./stores/audio')['useSpeakingStore']>
|
||||
readonly useSpeechRecognition: UnwrapRef<typeof import('@vueuse/core')['useSpeechRecognition']>
|
||||
readonly useSpeechSynthesis: UnwrapRef<typeof import('@vueuse/core')['useSpeechSynthesis']>
|
||||
readonly useStepper: UnwrapRef<typeof import('@vueuse/core')['useStepper']>
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
<script setup lang="ts">
|
||||
import Settings from '../Settings.vue'
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<header flex="~" mb-1 w-full gap-2>
|
||||
<div flex="~ 1" w-full items-center gap-2 text-nowrap text-2xl>
|
||||
<div i-solar:cat-outline text="[#ed869d]" />
|
||||
<div font-cute>
|
||||
<span>アイリ</span>
|
||||
</div>
|
||||
</div>
|
||||
<Settings />
|
||||
</header>
|
||||
</template>
|
||||
@@ -1,578 +0,0 @@
|
||||
<script setup lang="ts">
|
||||
import type { AssistantMessage, Message } from '@xsai/shared-chat-completion'
|
||||
import type { Emotion } from '../constants/emotions'
|
||||
import { useDevicesList, useElementBounding, useLocalStorage, useScroll } from '@vueuse/core'
|
||||
|
||||
import { storeToRefs } from 'pinia'
|
||||
|
||||
import { computed, onMounted, ref, watch } from 'vue'
|
||||
import { useI18n } from 'vue-i18n'
|
||||
|
||||
import { useWhisper } from '~/composables/whisper'
|
||||
import Avatar from '../assets/live2d/models/hiyori_free_zh/avatar.png'
|
||||
import { useMarkdown } from '../composables/markdown'
|
||||
import { useMicVAD } from '../composables/micvad'
|
||||
import { useQueue } from '../composables/queue'
|
||||
import { useDelayMessageQueue, useEmotionsMessageQueue, useMessageContentQueue } from '../composables/queues'
|
||||
import { llmInferenceEndToken } from '../constants'
|
||||
import { EMOTION_EmotionMotionName_value, EMOTION_VRMExpressionName_value, EmotionThinkMotionName } from '../constants/emotions'
|
||||
import SystemPromptV2 from '../constants/prompts/system-v2'
|
||||
import WhisperWorker from '../libs/workers/worker?worker&url'
|
||||
import { useLLM } from '../stores/llm'
|
||||
|
||||
import { useSettings } from '../stores/settings'
|
||||
import { encodeWAVToBase64 } from '../utils/binary'
|
||||
import { asyncIteratorFromReadableStream } from '../utils/iterator'
|
||||
import BasicTextarea from './BasicTextarea.vue'
|
||||
import Live2DScene from './Live2DScene.vue'
|
||||
import Settings from './Settings.vue'
|
||||
import ThreeDScene from './ThreeDScene.vue'
|
||||
|
||||
const { t } = useI18n()
|
||||
|
||||
const nowSpeakingAvatarBorderOpacityMin = 30
|
||||
const nowSpeakingAvatarBorderOpacityMax = 100
|
||||
|
||||
const {
|
||||
elevenLabsApiKey,
|
||||
openAiApiBaseURL,
|
||||
openAiApiKey,
|
||||
stageView,
|
||||
} = storeToRefs(useSettings())
|
||||
const openAIModel = useLocalStorage<{ id: string, name?: string }>('settings/llm/openai/model', { id: 'openai/gpt-3.5-turbo', name: 'OpenAI GPT3.5 Turbo' })
|
||||
|
||||
const { streamSpeech, stream, models } = useLLM()
|
||||
const { audioContext, calculateVolume } = useAudioContext()
|
||||
const { process } = useMarkdown()
|
||||
const { audioInputs } = useDevicesList({ constraints: { audio: true }, requestPermissions: true })
|
||||
|
||||
const isAudioInputOn = ref('true')
|
||||
const chatHistoryRef = ref<HTMLDivElement>()
|
||||
const listening = ref(false)
|
||||
const live2DViewerRef = ref<{ setMotion: (motionName: string) => Promise<void> }>()
|
||||
const vrmViewerRef = ref<{ setExpression: (expression: string) => void }>()
|
||||
const supportedModels = ref<{ id: string, name?: string }[]>([])
|
||||
const messageInput = ref<string>('')
|
||||
const messages = ref<Array<Message>>([SystemPromptV2(
|
||||
t('prompt.prefix'),
|
||||
t('prompt.suffix'),
|
||||
)])
|
||||
const streamingMessage = ref<AssistantMessage>({ role: 'assistant', content: '' })
|
||||
const audioAnalyser = ref<AnalyserNode>()
|
||||
const mouthOpenSize = ref(0)
|
||||
const nowSpeaking = ref(false)
|
||||
const lipSyncStarted = ref(false)
|
||||
const selectedAudioDevice = ref<MediaDeviceInfo>()
|
||||
|
||||
const bounding = useElementBounding(chatHistoryRef, { immediate: true, windowScroll: true, windowResize: true })
|
||||
const { y: chatHistoryContainerY } = useScroll(chatHistoryRef)
|
||||
|
||||
const selectedAudioDeviceId = computed(() => selectedAudioDevice.value?.deviceId)
|
||||
const nowSpeakingAvatarBorderOpacity = computed<number>(() => {
|
||||
if (!nowSpeaking.value)
|
||||
return nowSpeakingAvatarBorderOpacityMin
|
||||
|
||||
return ((nowSpeakingAvatarBorderOpacityMin
|
||||
+ (nowSpeakingAvatarBorderOpacityMax - nowSpeakingAvatarBorderOpacityMin) * mouthOpenSize.value) / 100)
|
||||
})
|
||||
|
||||
function handleModelChange(event: Event) {
|
||||
const target = event.target as HTMLSelectElement
|
||||
const found = supportedModels.value.find(m => m.id === target.value)
|
||||
if (!found) {
|
||||
openAIModel.value = undefined
|
||||
return
|
||||
}
|
||||
|
||||
openAIModel.value = found
|
||||
}
|
||||
|
||||
async function handleAudioInputChange(event: Event) {
|
||||
const target = event.target as HTMLSelectElement
|
||||
const found = audioInputs.value.find(d => d.deviceId === target.value)
|
||||
if (!found) {
|
||||
selectedAudioDevice.value = undefined
|
||||
return
|
||||
}
|
||||
|
||||
selectedAudioDevice.value = found
|
||||
}
|
||||
|
||||
const audioQueue = useQueue<{ audioBuffer: AudioBuffer, text: string }>({
|
||||
handlers: [
|
||||
(ctx) => {
|
||||
return new Promise((resolve) => {
|
||||
// Create an AudioBufferSourceNode
|
||||
const source = audioContext.createBufferSource()
|
||||
source.buffer = ctx.data.audioBuffer
|
||||
|
||||
// Connect the source to the AudioContext's destination (the speakers)
|
||||
source.connect(audioContext.destination)
|
||||
// Connect the source to the analyzer
|
||||
source.connect(audioAnalyser.value!)
|
||||
|
||||
// Start playing the audio
|
||||
nowSpeaking.value = true
|
||||
source.start(0)
|
||||
source.onended = () => {
|
||||
nowSpeaking.value = false
|
||||
resolve()
|
||||
}
|
||||
})
|
||||
},
|
||||
],
|
||||
})
|
||||
|
||||
const ttsQueue = useQueue<string>({
|
||||
handlers: [
|
||||
async (ctx) => {
|
||||
const now = Date.now()
|
||||
const res = await streamSpeech('https://airi-api.ayaka.io', elevenLabsApiKey.value, ctx.data, {
|
||||
// voice: 'ShanShan',
|
||||
// Quite good for English
|
||||
voice: 'Myriam',
|
||||
// Beatrice is not 'childish' like the others
|
||||
// voice: 'Beatrice',
|
||||
model_id: 'eleven_multilingual_v2',
|
||||
voice_settings: {
|
||||
stability: 0.4,
|
||||
similarity_boost: 0.5,
|
||||
},
|
||||
})
|
||||
const elapsed = Date.now() - now
|
||||
|
||||
// eslint-disable-next-line no-console
|
||||
console.debug('TTS took', elapsed, 'ms')
|
||||
|
||||
// Decode the ArrayBuffer into an AudioBuffer
|
||||
const audioBuffer = await audioContext.decodeAudioData(res)
|
||||
await audioQueue.add({ audioBuffer, text: ctx.data })
|
||||
},
|
||||
],
|
||||
})
|
||||
|
||||
ttsQueue.on('add', (content) => {
|
||||
// eslint-disable-next-line no-console
|
||||
console.debug('ttsQueue added', content)
|
||||
})
|
||||
|
||||
const messageContentQueue = useMessageContentQueue(ttsQueue)
|
||||
|
||||
const emotionsQueue = useQueue<Emotion>({
|
||||
handlers: [
|
||||
async (ctx) => {
|
||||
if (stageView.value === '3d') {
|
||||
const value = EMOTION_VRMExpressionName_value[ctx.data]
|
||||
if (!value)
|
||||
return
|
||||
|
||||
await vrmViewerRef.value!.setExpression(value)
|
||||
}
|
||||
else if (stageView.value === '2d') {
|
||||
await live2DViewerRef.value!.setMotion(EMOTION_EmotionMotionName_value[ctx.data])
|
||||
}
|
||||
},
|
||||
],
|
||||
})
|
||||
|
||||
const emotionMessageContentQueue = useEmotionsMessageQueue(emotionsQueue)
|
||||
emotionMessageContentQueue.onHandlerEvent('emotion', (emotion) => {
|
||||
// eslint-disable-next-line no-console
|
||||
console.debug('emotion detected', emotion)
|
||||
})
|
||||
|
||||
const delaysQueue = useDelayMessageQueue()
|
||||
delaysQueue.onHandlerEvent('delay', (delay) => {
|
||||
// eslint-disable-next-line no-console
|
||||
console.debug('delay detected', delay)
|
||||
})
|
||||
|
||||
function getVolumeWithMinMaxNormalizeWithFrameUpdates() {
|
||||
requestAnimationFrame(getVolumeWithMinMaxNormalizeWithFrameUpdates)
|
||||
if (!nowSpeaking.value)
|
||||
return
|
||||
|
||||
mouthOpenSize.value = calculateVolume(audioAnalyser.value!, 'linear')
|
||||
}
|
||||
|
||||
function setupLipSync() {
|
||||
if (!lipSyncStarted.value) {
|
||||
getVolumeWithMinMaxNormalizeWithFrameUpdates()
|
||||
audioContext.resume()
|
||||
lipSyncStarted.value = true
|
||||
}
|
||||
}
|
||||
|
||||
function setupAnalyser() {
|
||||
if (!audioAnalyser.value)
|
||||
audioAnalyser.value = audioContext.createAnalyser()
|
||||
}
|
||||
|
||||
async function onSendMessage(sendingMessage: string) {
|
||||
if (!sendingMessage)
|
||||
return
|
||||
|
||||
setupLipSync()
|
||||
setupAnalyser()
|
||||
|
||||
streamingMessage.value = { role: 'assistant', content: '' }
|
||||
messages.value.push({ role: 'user', content: sendingMessage })
|
||||
|
||||
// Scroll down to the new sent message
|
||||
nextTick().then(() => {
|
||||
bounding.update()
|
||||
chatHistoryContainerY.value = bounding.height.value
|
||||
})
|
||||
|
||||
messages.value.push(streamingMessage.value)
|
||||
// const index = messages.value.length - 1
|
||||
live2DViewerRef.value?.setMotion(EmotionThinkMotionName)
|
||||
|
||||
const res = await stream(openAiApiBaseURL.value, openAiApiKey.value, openAIModel.value.id, messages.value.slice(0, messages.value.length - 1))
|
||||
let fullText = ''
|
||||
|
||||
const parser = useLlmmarkerParser({
|
||||
onLiteral: async (literal) => {
|
||||
await messageContentQueue.add(literal)
|
||||
streamingMessage.value.content += literal
|
||||
|
||||
// Scroll down to the new responding message
|
||||
nextTick(() => {
|
||||
bounding.update()
|
||||
chatHistoryContainerY.value = bounding.height.value
|
||||
})
|
||||
},
|
||||
onSpecial: async (special) => {
|
||||
await delaysQueue.add(special)
|
||||
await emotionMessageContentQueue.add(special)
|
||||
},
|
||||
})
|
||||
|
||||
for await (const textPart of asyncIteratorFromReadableStream(res.textStream, async v => v)) {
|
||||
fullText += textPart
|
||||
await parser.consume(textPart)
|
||||
}
|
||||
|
||||
await parser.end()
|
||||
await delaysQueue.add(llmInferenceEndToken)
|
||||
|
||||
messageInput.value = ''
|
||||
|
||||
// eslint-disable-next-line no-console
|
||||
console.debug('Full text:', fullText)
|
||||
}
|
||||
|
||||
const { transcribe: generate, load: loadWhisper, status: whisperStatus, terminate } = useWhisper(WhisperWorker, {
|
||||
onComplete: async (res) => {
|
||||
await onSendMessage(res)
|
||||
},
|
||||
})
|
||||
|
||||
function handleLoadWhisper() {
|
||||
if (whisperStatus.value === 'loading')
|
||||
return
|
||||
|
||||
loadWhisper()
|
||||
}
|
||||
|
||||
async function handleTranscription(buffer: Float32Array) {
|
||||
await audioContext.resume()
|
||||
|
||||
// Convert Float32Array to WAV format
|
||||
const audioBase64 = await encodeWAVToBase64(buffer, audioContext.sampleRate)
|
||||
generate({ type: 'generate', data: { audio: audioBase64, language: 'en' } })
|
||||
}
|
||||
|
||||
const { destroy } = useMicVAD(selectedAudioDeviceId, {
|
||||
onSpeechStart: () => {
|
||||
// TODO: interrupt the playback
|
||||
// TODO: interrupt any of the ongoing TTS
|
||||
// TODO: interrupt any of the ongoing LLM requests
|
||||
// TODO: interrupt any of the ongoing animation of Live2D or VRM
|
||||
// TODO: once interrupted, we should somehow switch to listen or thinking
|
||||
// emotion / expression?
|
||||
listening.value = true
|
||||
},
|
||||
// VAD misfire means while speech end is detected but
|
||||
// the frames of the segment of the audio buffer
|
||||
// is not enough to be considered as a speech segment
|
||||
// which controlled by the `minSpeechFrames` parameter
|
||||
onVADMisfire: () => {
|
||||
// TODO: do audio buffer send to whisper
|
||||
listening.value = false
|
||||
},
|
||||
onSpeechEnd: (buffer) => {
|
||||
// TODO: do audio buffer send to whisper
|
||||
listening.value = false
|
||||
handleTranscription(buffer)
|
||||
},
|
||||
})
|
||||
|
||||
watch(isAudioInputOn, async (value) => {
|
||||
if (value === 'false') {
|
||||
selectedAudioDevice.value = undefined
|
||||
destroy()
|
||||
terminate()
|
||||
}
|
||||
if (value === 'true') {
|
||||
selectedAudioDevice.value = audioInputs.value[0]
|
||||
}
|
||||
})
|
||||
|
||||
watch([openAiApiBaseURL, openAiApiKey], async ([baseUrl, apiKey]) => {
|
||||
if (!baseUrl || !apiKey) {
|
||||
supportedModels.value = []
|
||||
return
|
||||
}
|
||||
|
||||
supportedModels.value = await models(baseUrl, apiKey)
|
||||
})
|
||||
|
||||
onMounted(async () => {
|
||||
if (!openAiApiBaseURL.value || !openAiApiKey.value)
|
||||
return
|
||||
|
||||
supportedModels.value = await models(openAiApiBaseURL.value, openAiApiKey.value)
|
||||
})
|
||||
|
||||
onUnmounted(() => {
|
||||
lipSyncStarted.value = false
|
||||
})
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<div h-full max-h="[100vh]" max-w="[100vw]" p="2" flex="~ col" overflow-hidden>
|
||||
<header flex="~" mb-1 w-full gap-2>
|
||||
<div flex="~ 1" w-full items-center gap-2 text-nowrap text-2xl>
|
||||
<div i-solar:cat-outline text="[#ed869d]" />
|
||||
<div font-cute>
|
||||
<span>アイリ</span>
|
||||
</div>
|
||||
</div>
|
||||
<Settings />
|
||||
</header>
|
||||
<div flex="~ row 1" max-h="[calc(100vh-270px)] <sm:[calc(100vh-280px)]" relative h-full w-full items-end gap-2>
|
||||
<Live2DScene
|
||||
v-if="stageView === '2d'"
|
||||
ref="live2DViewerRef"
|
||||
:mouth-open-size="mouthOpenSize"
|
||||
model="/assets/live2d/models/hiyori_pro_zh/runtime/hiyori_pro_t11.model3.json"
|
||||
w="50%" min-w="50% <lg:full" min-h="100 sm:100" h-full flex-1
|
||||
/>
|
||||
<ThreeDScene
|
||||
v-else-if="stageView === '3d'"
|
||||
ref="vrmViewerRef"
|
||||
model="/assets/vrm/models/AvatarSample-B/AvatarSample_B.vrm"
|
||||
idle-animation="/assets/vrm/animations/idle_loop.vrma"
|
||||
w="50%" min-w="50% <lg:full" min-h="100 sm:100" h-full flex-1
|
||||
@error="console.error"
|
||||
/>
|
||||
<div
|
||||
class="relative <lg:(absolute bottom-0 from-zinc-100/80 to-zinc-800/0 bg-gradient-to-t p-2 dark:from-zinc-800/80)"
|
||||
px="<sm:2" py="<sm:2" rounded="lg"
|
||||
w="50% <lg:full" flex="~ col 1" overflow-hidden max-h="[calc(100vh-280px)] <sm:[calc(100vh-96%)]"
|
||||
>
|
||||
<div ref="chatHistoryRef" h-full w-full overflow-scroll>
|
||||
<div v-for="(message, index) in messages" :key="index" mb-2>
|
||||
<div v-if="message.role === 'assistant'" flex mr="12">
|
||||
<div
|
||||
class="block <sm:hidden"
|
||||
mr-2 h-10
|
||||
min-h-10 min-w-10 w-10
|
||||
overflow-hidden rounded-full
|
||||
border="solid 3"
|
||||
transition="all ease-in-out" duration-100
|
||||
:style="{
|
||||
borderColor: `rgba(236, 72, 153, ${nowSpeakingAvatarBorderOpacity.toFixed(2)})`,
|
||||
}"
|
||||
>
|
||||
<img :src="Avatar">
|
||||
</div>
|
||||
<div
|
||||
flex="~ col"
|
||||
bg="pink-50 dark:pink-900"
|
||||
border="2 solid pink dark:pink-700"
|
||||
min-w-20 rounded-lg px-2 py-1
|
||||
h="unset <sm:fit"
|
||||
>
|
||||
<div>
|
||||
<span text-xs text="black/50 dark:white/50" font-semibold class="inline <sm:hidden">Airi</span>
|
||||
</div>
|
||||
<div v-if="message.content" class="markdown-content" text="base <sm:xs" v-html="process(message.content as string)" />
|
||||
<div v-else i-eos-icons:three-dots-loading />
|
||||
</div>
|
||||
</div>
|
||||
<div v-else-if="message.role === 'user'" flex="~ row-reverse" ml="12">
|
||||
<div
|
||||
class="block <sm:hidden"
|
||||
border="purple solid 3"
|
||||
ml="2"
|
||||
h-10 min-h-10 min-w-10 w-10
|
||||
overflow-hidden rounded-full
|
||||
>
|
||||
<div i-carbon:user-avatar-filled text="purple" h-full w-full p="0" m="0" />
|
||||
</div>
|
||||
<div
|
||||
flex="~ col"
|
||||
bg="purple-50 dark:purple-900"
|
||||
px="2"
|
||||
border="2 solid purple dark:purple-700"
|
||||
h="unset <sm:fit" min-w-20 rounded-lg px-2 py-1
|
||||
>
|
||||
<div>
|
||||
<span text-xs text="black/50 dark:white/50" font-semibold class="inline <sm:hidden">You</span>
|
||||
</div>
|
||||
<div v-if="message.content" class="markdown-content" text="base <sm:xs" whitespace-nowrap v-html="process(message.content as string)" />
|
||||
<div v-else />
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div flex="~ row" my="2" space-x="2" w-full self-end>
|
||||
<div flex="~ col" w="100%" space-y="2">
|
||||
<select
|
||||
p="2"
|
||||
bg="zinc-100 dark:zinc-700" w-full rounded-lg
|
||||
outline-none
|
||||
@change="handleAudioInputChange"
|
||||
>
|
||||
<option disabled>
|
||||
{{ t('stage.select-a-audio-input') }}
|
||||
</option>
|
||||
<option v-if="selectedAudioDevice" :value="selectedAudioDevice.deviceId">
|
||||
{{ selectedAudioDevice.label }}
|
||||
</option>
|
||||
<option v-for="m in audioInputs" :key="m.deviceId" :value="m.deviceId">
|
||||
{{ m.label }}
|
||||
</option>
|
||||
</select>
|
||||
<select
|
||||
p="2"
|
||||
bg="zinc-100 dark:zinc-700" w-full rounded-lg
|
||||
outline-none
|
||||
@change="handleModelChange"
|
||||
>
|
||||
<option disabled>
|
||||
{{ t('stage.select-a-model') }}
|
||||
</option>
|
||||
<option v-if="openAIModel" :value="openAIModel.id">
|
||||
{{ 'name' in openAIModel ? `${openAIModel.name} (${openAIModel.id})` : openAIModel.id }}
|
||||
</option>
|
||||
<option v-for="m in supportedModels" :key="m.id" :value="m.id">
|
||||
{{ 'name' in m ? `${m.name} (${m.id})` : m.id }}
|
||||
</option>
|
||||
</select>
|
||||
<div flex gap-2>
|
||||
<BasicTextarea
|
||||
v-model="messageInput"
|
||||
:placeholder="t('stage.message')"
|
||||
p="2" bg="zinc-100 dark:zinc-700"
|
||||
w-full rounded-lg outline-none min-h="[100px]"
|
||||
@submit="onSendMessage"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
<div h-full w="[16%] <sm:[50%]" flex="~ col" gap-2>
|
||||
<fieldset
|
||||
flex="~ row"
|
||||
bg="zinc-100 dark:zinc-700"
|
||||
text="sm zinc-400 dark:zinc-500"
|
||||
appearance-none gap-1 rounded-lg rounded-md border-none p-1
|
||||
>
|
||||
<label
|
||||
:class="[isAudioInputOn === 'true' ? 'bg-zinc-300 text-zinc-900 dark:bg-zinc-200 dark:text-zinc-800' : '']"
|
||||
min-h="7.75" flex="~" w-full cursor-pointer items-center justify-center rounded-md
|
||||
>
|
||||
<input
|
||||
v-model="isAudioInputOn"
|
||||
:checked="isAudioInputOn === 'true'"
|
||||
:aria-checked="isAudioInputOn === 'true'"
|
||||
name="isAudioInputOn"
|
||||
type="radio"
|
||||
role="radio"
|
||||
value="true"
|
||||
hidden appearance-none outline-none
|
||||
>
|
||||
<div select-none>ON</div>
|
||||
</label>
|
||||
<label
|
||||
:class="[isAudioInputOn === 'false' ? 'bg-zinc-300 text-zinc-900 dark:bg-zinc-200 dark:text-zinc-800' : '']"
|
||||
min-h="7.75" flex="~" w-full cursor-pointer items-center justify-center rounded-md
|
||||
>
|
||||
<input
|
||||
v-model="isAudioInputOn"
|
||||
:checked="isAudioInputOn === 'false'"
|
||||
:aria-checked="isAudioInputOn === 'false'"
|
||||
name="stageView"
|
||||
type="radio"
|
||||
role="radio"
|
||||
value="false"
|
||||
hidden appearance-none outline-none
|
||||
>
|
||||
<div select-none>OFF</div>
|
||||
</label>
|
||||
</fieldset>
|
||||
<button
|
||||
flex="~ row"
|
||||
p="2" bg="zinc-100 dark:zinc-700" min-h="9.75"
|
||||
min-w-20 w-full items-center justify-center rounded-lg outline-none
|
||||
transition="all ease-in-out"
|
||||
@click="handleLoadWhisper"
|
||||
>
|
||||
<Transition mode="out-in">
|
||||
<div v-if="whisperStatus === null" flex="~ row" items-center justify-center space-x-1>
|
||||
Load
|
||||
</div>
|
||||
<div v-else-if="whisperStatus === 'loading'" flex="~ row" items-center justify-center space-x-1>
|
||||
<div i-svg-spinners:bouncing-ball text-pink />
|
||||
<span>Loading</span>
|
||||
</div>
|
||||
<div v-else-if="whisperStatus === 'ready'" flex="~ row" items-center justify-center space-x-1>
|
||||
<div i-lucide:check text-green />
|
||||
<span>Ready</span>
|
||||
</div>
|
||||
</Transition>
|
||||
</button>
|
||||
<button
|
||||
flex="~ row" h-full
|
||||
p="2" bg="zinc-100 dark:zinc-700"
|
||||
min-w-20 w-full items-center justify-center rounded-lg outline-none
|
||||
transition="all ease-in-out"
|
||||
>
|
||||
<Transition mode="out-in">
|
||||
<div v-if="listening" flex="~ row" items-center justify-center space-x-1>
|
||||
<div i-carbon:microphone-filled text-red />
|
||||
</div>
|
||||
<div v-else flex="~ row" items-center justify-center space-x-1>
|
||||
<div i-carbon:microphone text-inherit />
|
||||
{{ t('stage.waiting') }}
|
||||
</div>
|
||||
</Transition>
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<style>
|
||||
.v-enter-active,
|
||||
.v-leave-active {
|
||||
transition: opacity 0.5s ease;
|
||||
}
|
||||
|
||||
.v-enter-from,
|
||||
.v-leave-to {
|
||||
opacity: 0;
|
||||
}
|
||||
|
||||
.markdown-content p {
|
||||
word-wrap: break-word;
|
||||
word-break: normal;
|
||||
text-overflow: ellipsis;
|
||||
text-wrap: auto;
|
||||
white-space: break-spaces;
|
||||
overflow-wrap: anywhere;
|
||||
}
|
||||
</style>
|
||||
+14
-3
@@ -1,5 +1,16 @@
|
||||
<script setup lang="ts">
|
||||
import { EmotionAngryMotionName, EmotionAwkwardMotionName, EmotionHappyMotionName, EmotionQuestionMotionName, EmotionSadMotionName, EmotionSurpriseMotionName, EmotionThinkMotionName } from '~/constants/emotions'
|
||||
import {
|
||||
EmotionAngryMotionName,
|
||||
EmotionAwkwardMotionName,
|
||||
EmotionHappyMotionName,
|
||||
EmotionQuestionMotionName,
|
||||
EmotionSadMotionName,
|
||||
EmotionSurpriseMotionName,
|
||||
EmotionThinkMotionName,
|
||||
} from '../../constants/emotions'
|
||||
|
||||
import Live2DViewer from '../Live2D/Viewer.vue'
|
||||
import Screen from '../Screen.vue'
|
||||
|
||||
withDefaults(defineProps<{
|
||||
model: string
|
||||
@@ -20,7 +31,7 @@ defineExpose({
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<Screen v-slot="{ canvasHeight, canvasWidth }" relative>
|
||||
<Screen v-slot="{ width, height }" relative>
|
||||
<div z="10" top="2" absolute w-full flex="~ col" gap-2>
|
||||
<div flex="~ row" w-full flex-wrap gap-2>
|
||||
<button
|
||||
@@ -67,6 +78,6 @@ defineExpose({
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
<Live2DViewer ref="modelRef" :canvas-width="canvasWidth" :canvas-height="canvasHeight" :model="model" :mouth-open-size="mouthOpenSize" />
|
||||
<Live2DViewer ref="modelRef" :canvas-width="width" :canvas-height="height" :model="model" :mouth-open-size="mouthOpenSize" />
|
||||
</Screen>
|
||||
</template>
|
||||
+3
-3
@@ -2,9 +2,9 @@
|
||||
import { OrbitControls } from '@tresjs/cientos'
|
||||
import { TresCanvas } from '@tresjs/core'
|
||||
|
||||
import Collapsable from './Collapsable.vue'
|
||||
import DataGuiRange from './DataGui/Range.vue'
|
||||
import VRMModel from './VRMModel.vue'
|
||||
import Collapsable from '../Collapsable.vue'
|
||||
import DataGuiRange from '../DataGui/Range.vue'
|
||||
import VRMModel from '../VRM/Model.vue'
|
||||
|
||||
const props = defineProps<{
|
||||
model: string
|
||||
@@ -51,6 +51,6 @@ onMounted(async () => {
|
||||
|
||||
<template>
|
||||
<div ref="containerRef" h-full w-full>
|
||||
<slot :width="canvasWidth" height="canvasHeight" />
|
||||
<slot :width="canvasWidth" :height="canvasHeight" />
|
||||
</div>
|
||||
</template>
|
||||
|
||||
+4
-3
@@ -2,9 +2,10 @@
|
||||
import type { VRMCore } from '@pixiv/three-vrm-core'
|
||||
import { useLoop, useTresContext } from '@tresjs/core'
|
||||
import { AnimationMixer } from 'three'
|
||||
import { clipFromVRMAnimation, loadVRMAnimation, useBlink } from '~/composables/vrm/animation'
|
||||
import { loadVrm } from '~/composables/vrm/core'
|
||||
import { useVRMEmote } from '~/composables/vrm/expression'
|
||||
|
||||
import { clipFromVRMAnimation, loadVRMAnimation, useBlink } from '../../composables/vrm/animation'
|
||||
import { loadVrm } from '../../composables/vrm/core'
|
||||
import { useVRMEmote } from '../../composables/vrm/expression'
|
||||
|
||||
const props = defineProps<{
|
||||
model: string
|
||||
@@ -0,0 +1,99 @@
|
||||
<script setup lang="ts">
|
||||
import type { Message } from '@xsai/shared-chat-completion'
|
||||
import { storeToRefs } from 'pinia'
|
||||
|
||||
import { ref } from 'vue'
|
||||
import Avatar from '../../assets/live2d/models/hiyori_free_zh/avatar.png'
|
||||
import { useMarkdown } from '../../composables/markdown'
|
||||
import { useSpeakingStore } from '../../stores/audio'
|
||||
|
||||
const messages = ref<Message[]>([])
|
||||
const chatHistoryRef = ref<HTMLDivElement>()
|
||||
const bounding = useElementBounding(chatHistoryRef, { immediate: true, windowScroll: true, windowResize: true })
|
||||
const { y: chatHistoryContainerY } = useScroll(chatHistoryRef)
|
||||
|
||||
const { process } = useMarkdown()
|
||||
const { nowSpeakingAvatarBorderOpacity } = storeToRefs(useSpeakingStore())
|
||||
const { onBeforeMessageComposed, onTokenLiteral } = useChatStore()
|
||||
|
||||
onBeforeMessageComposed(async () => {
|
||||
// Scroll down to the new sent message
|
||||
nextTick().then(() => {
|
||||
bounding.update()
|
||||
chatHistoryContainerY.value = bounding.height.value
|
||||
})
|
||||
})
|
||||
|
||||
onTokenLiteral(async () => {
|
||||
// Scroll down to the new responding message
|
||||
nextTick().then(() => {
|
||||
bounding.update()
|
||||
chatHistoryContainerY.value = bounding.height.value
|
||||
})
|
||||
})
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<div
|
||||
class="relative <lg:(absolute bottom-0 from-zinc-100/80 to-zinc-800/0 bg-gradient-to-t p-2 dark:from-zinc-800/80)"
|
||||
px="<sm:2" py="<sm:2" rounded="lg"
|
||||
w="50% <lg:full" flex="~ col 1" overflow-hidden max-h="[calc(100vh-280px)] <sm:[calc(100vh-96%)]"
|
||||
>
|
||||
<div ref="chatHistoryRef" h-full w-full overflow-scroll>
|
||||
<div v-for="(message, index) in messages" :key="index" mb-2>
|
||||
<div v-if="message.role === 'assistant'" flex mr="12">
|
||||
<div
|
||||
class="block <sm:hidden"
|
||||
mr-2 h-10
|
||||
min-h-10 min-w-10 w-10
|
||||
overflow-hidden rounded-full
|
||||
border="solid 3"
|
||||
transition="all ease-in-out" duration-100
|
||||
:style="{
|
||||
borderColor: `rgba(236, 72, 153, ${nowSpeakingAvatarBorderOpacity.toFixed(2)})`,
|
||||
}"
|
||||
>
|
||||
<img :src="Avatar">
|
||||
</div>
|
||||
<div
|
||||
flex="~ col"
|
||||
bg="pink-50 dark:pink-900"
|
||||
border="2 solid pink dark:pink-700"
|
||||
min-w-20 rounded-lg px-2 py-1
|
||||
h="unset <sm:fit"
|
||||
>
|
||||
<div>
|
||||
<span text-xs text="black/50 dark:white/50" font-semibold class="inline <sm:hidden">Airi</span>
|
||||
</div>
|
||||
<div v-if="message.content" class="markdown-content" text="base <sm:xs" v-html="process(message.content as string)" />
|
||||
<div v-else i-eos-icons:three-dots-loading />
|
||||
</div>
|
||||
</div>
|
||||
<div v-else-if="message.role === 'user'" flex="~ row-reverse" ml="12">
|
||||
<div
|
||||
class="block <sm:hidden"
|
||||
border="purple solid 3"
|
||||
ml="2"
|
||||
h-10 min-h-10 min-w-10 w-10
|
||||
overflow-hidden rounded-full
|
||||
>
|
||||
<div i-carbon:user-avatar-filled text="purple" h-full w-full p="0" m="0" />
|
||||
</div>
|
||||
<div
|
||||
flex="~ col"
|
||||
bg="purple-50 dark:purple-900"
|
||||
px="2"
|
||||
border="2 solid purple dark:purple-700"
|
||||
h="unset <sm:fit" min-w-20 rounded-lg px-2 py-1
|
||||
>
|
||||
<div>
|
||||
<span text-xs text="black/50 dark:white/50" font-semibold class="inline <sm:hidden">You</span>
|
||||
</div>
|
||||
<div v-if="message.content" class="markdown-content" text="base <sm:xs" whitespace-nowrap v-html="process(message.content as string)" />
|
||||
<div v-else />
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
@@ -0,0 +1,244 @@
|
||||
<script setup lang="ts">
|
||||
import { useDevicesList } from '@vueuse/core'
|
||||
import { storeToRefs } from 'pinia'
|
||||
import { ref } from 'vue'
|
||||
|
||||
import WhisperWorker from '../../libs/workers/worker?worker&url'
|
||||
import { encodeWAVToBase64 } from '../../utils/binary'
|
||||
|
||||
const { audioInputs } = useDevicesList({ constraints: { audio: true }, requestPermissions: true })
|
||||
const { openAiModel, openAiApiBaseURL, openAiApiKey, selectedAudioDevice, isAudioInputOn, selectedAudioDeviceId } = storeToRefs(useSettings())
|
||||
const { models } = useLLM()
|
||||
const { send, onAfterSend } = useChatStore()
|
||||
const { audioContext } = useAudioContext()
|
||||
const { t } = useI18n()
|
||||
|
||||
const messageInput = ref('')
|
||||
const supportedModels = ref<{ id: string, name?: string }[]>([])
|
||||
const listening = ref(false)
|
||||
|
||||
function handleModelChange(event: Event) {
|
||||
const target = event.target as HTMLSelectElement
|
||||
const found = supportedModels.value.find(m => m.id === target.value)
|
||||
if (!found) {
|
||||
openAiModel.value = undefined
|
||||
return
|
||||
}
|
||||
|
||||
openAiModel.value = found
|
||||
}
|
||||
|
||||
async function handleAudioInputChange(event: Event) {
|
||||
const target = event.target as HTMLSelectElement
|
||||
const found = audioInputs.value.find(d => d.deviceId === target.value)
|
||||
if (!found) {
|
||||
selectedAudioDevice.value = undefined
|
||||
return
|
||||
}
|
||||
|
||||
selectedAudioDevice.value = found
|
||||
}
|
||||
|
||||
const { transcribe: generate, load: loadWhisper, status: whisperStatus, terminate } = useWhisper(WhisperWorker, {
|
||||
onComplete: async (res) => {
|
||||
await send(res)
|
||||
},
|
||||
})
|
||||
|
||||
function handleLoadWhisper() {
|
||||
if (whisperStatus.value === 'loading')
|
||||
return
|
||||
|
||||
loadWhisper()
|
||||
}
|
||||
|
||||
async function handleTranscription(buffer: Float32Array) {
|
||||
await audioContext.resume()
|
||||
|
||||
// Convert Float32Array to WAV format
|
||||
const audioBase64 = await encodeWAVToBase64(buffer, audioContext.sampleRate)
|
||||
generate({ type: 'generate', data: { audio: audioBase64, language: 'en' } })
|
||||
}
|
||||
|
||||
async function handleSend() {
|
||||
await send(messageInput.value)
|
||||
}
|
||||
|
||||
const { destroy } = useMicVAD(selectedAudioDeviceId, {
|
||||
onSpeechStart: () => {
|
||||
// TODO: interrupt the playback
|
||||
// TODO: interrupt any of the ongoing TTS
|
||||
// TODO: interrupt any of the ongoing LLM requests
|
||||
// TODO: interrupt any of the ongoing animation of Live2D or VRM
|
||||
// TODO: once interrupted, we should somehow switch to listen or thinking
|
||||
// emotion / expression?
|
||||
listening.value = true
|
||||
},
|
||||
// VAD misfire means while speech end is detected but
|
||||
// the frames of the segment of the audio buffer
|
||||
// is not enough to be considered as a speech segment
|
||||
// which controlled by the `minSpeechFrames` parameter
|
||||
onVADMisfire: () => {
|
||||
// TODO: do audio buffer send to whisper
|
||||
listening.value = false
|
||||
},
|
||||
onSpeechEnd: (buffer) => {
|
||||
// TODO: do audio buffer send to whisper
|
||||
listening.value = false
|
||||
handleTranscription(buffer)
|
||||
},
|
||||
})
|
||||
|
||||
watch(isAudioInputOn, async (value) => {
|
||||
if (value === 'false') {
|
||||
destroy()
|
||||
terminate()
|
||||
}
|
||||
})
|
||||
|
||||
watch([openAiApiBaseURL, openAiApiKey], async ([baseUrl, apiKey]) => {
|
||||
if (!baseUrl || !apiKey) {
|
||||
supportedModels.value = []
|
||||
return
|
||||
}
|
||||
|
||||
supportedModels.value = await models(baseUrl, apiKey)
|
||||
})
|
||||
|
||||
onMounted(async () => {
|
||||
if (!openAiApiBaseURL.value || !openAiApiKey.value)
|
||||
return
|
||||
|
||||
supportedModels.value = await models(openAiApiBaseURL.value, openAiApiKey.value)
|
||||
})
|
||||
|
||||
onAfterSend(async () => {
|
||||
messageInput.value = ''
|
||||
})
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<div flex="~ row" my="2" space-x="2" w-full self-end>
|
||||
<div flex="~ col" w="100%" space-y="2">
|
||||
<select
|
||||
p="2"
|
||||
bg="zinc-100 dark:zinc-700" w-full rounded-lg
|
||||
outline-none
|
||||
@change="handleAudioInputChange"
|
||||
>
|
||||
<option disabled>
|
||||
{{ t('stage.select-a-audio-input') }}
|
||||
</option>
|
||||
<option v-if="selectedAudioDevice" :value="selectedAudioDevice.deviceId">
|
||||
{{ selectedAudioDevice.label }}
|
||||
</option>
|
||||
<option v-for="m in audioInputs" :key="m.deviceId" :value="m.deviceId">
|
||||
{{ m.label }}
|
||||
</option>
|
||||
</select>
|
||||
<select
|
||||
p="2"
|
||||
bg="zinc-100 dark:zinc-700" w-full rounded-lg
|
||||
outline-none
|
||||
@change="handleModelChange"
|
||||
>
|
||||
<option disabled>
|
||||
{{ t('stage.select-a-model') }}
|
||||
</option>
|
||||
<option v-if="openAiModel" :value="openAiModel.id">
|
||||
{{ 'name' in openAiModel ? `${openAiModel.name} (${openAiModel.id})` : openAiModel.id }}
|
||||
</option>
|
||||
<option v-for="m in supportedModels" :key="m.id" :value="m.id">
|
||||
{{ 'name' in m ? `${m.name} (${m.id})` : m.id }}
|
||||
</option>
|
||||
</select>
|
||||
<div flex gap-2>
|
||||
<BasicTextarea
|
||||
v-model="messageInput"
|
||||
:placeholder="t('stage.message')"
|
||||
p="2" bg="zinc-100 dark:zinc-700"
|
||||
w-full rounded-lg outline-none min-h="[100px]"
|
||||
@submit="handleSend"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
<div h-full w="[16%] <sm:[50%]" flex="~ col" gap-2>
|
||||
<fieldset
|
||||
flex="~ row"
|
||||
bg="zinc-100 dark:zinc-700"
|
||||
text="sm zinc-400 dark:zinc-500"
|
||||
appearance-none gap-1 rounded-lg rounded-md border-none p-1
|
||||
>
|
||||
<label
|
||||
:class="[isAudioInputOn === 'true' ? 'bg-zinc-300 text-zinc-900 dark:bg-zinc-200 dark:text-zinc-800' : '']"
|
||||
min-h="7.75" flex="~" w-full cursor-pointer items-center justify-center rounded-md
|
||||
>
|
||||
<input
|
||||
v-model="isAudioInputOn"
|
||||
:checked="isAudioInputOn === 'true'"
|
||||
:aria-checked="isAudioInputOn === 'true'"
|
||||
name="isAudioInputOn"
|
||||
type="radio"
|
||||
role="radio"
|
||||
value="true"
|
||||
hidden appearance-none outline-none
|
||||
>
|
||||
<div select-none>ON</div>
|
||||
</label>
|
||||
<label
|
||||
:class="[isAudioInputOn === 'false' ? 'bg-zinc-300 text-zinc-900 dark:bg-zinc-200 dark:text-zinc-800' : '']"
|
||||
min-h="7.75" flex="~" w-full cursor-pointer items-center justify-center rounded-md
|
||||
>
|
||||
<input
|
||||
v-model="isAudioInputOn"
|
||||
:checked="isAudioInputOn === 'false'"
|
||||
:aria-checked="isAudioInputOn === 'false'"
|
||||
name="stageView"
|
||||
type="radio"
|
||||
role="radio"
|
||||
value="false"
|
||||
hidden appearance-none outline-none
|
||||
>
|
||||
<div select-none>OFF</div>
|
||||
</label>
|
||||
</fieldset>
|
||||
<button
|
||||
flex="~ row"
|
||||
p="2" bg="zinc-100 dark:zinc-700" min-h="9.75"
|
||||
min-w-20 w-full items-center justify-center rounded-lg outline-none
|
||||
transition="all ease-in-out"
|
||||
@click="handleLoadWhisper"
|
||||
>
|
||||
<Transition mode="out-in">
|
||||
<div v-if="whisperStatus === null" flex="~ row" items-center justify-center space-x-1>
|
||||
Load
|
||||
</div>
|
||||
<div v-else-if="whisperStatus === 'loading'" flex="~ row" items-center justify-center space-x-1>
|
||||
<div i-svg-spinners:bouncing-ball text-pink />
|
||||
<span>Loading</span>
|
||||
</div>
|
||||
<div v-else-if="whisperStatus === 'ready'" flex="~ row" items-center justify-center space-x-1>
|
||||
<div i-lucide:check text-green />
|
||||
<span>Ready</span>
|
||||
</div>
|
||||
</Transition>
|
||||
</button>
|
||||
<button
|
||||
flex="~ row" h-full
|
||||
p="2" bg="zinc-100 dark:zinc-700"
|
||||
min-w-20 w-full items-center justify-center rounded-lg outline-none
|
||||
transition="all ease-in-out"
|
||||
>
|
||||
<Transition mode="out-in">
|
||||
<div v-if="listening" flex="~ row" items-center justify-center space-x-1>
|
||||
<div i-carbon:microphone-filled text-red />
|
||||
</div>
|
||||
<div v-else flex="~ row" items-center justify-center space-x-1>
|
||||
<div i-carbon:microphone text-inherit />
|
||||
{{ t('stage.waiting') }}
|
||||
</div>
|
||||
</Transition>
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
@@ -0,0 +1,184 @@
|
||||
<script setup lang="ts">
|
||||
import type { Emotion } from '../../constants/emotions'
|
||||
import { storeToRefs } from 'pinia'
|
||||
|
||||
import { onUnmounted, ref } from 'vue'
|
||||
import { useQueue } from '../../composables/queue'
|
||||
import { useDelayMessageQueue, useEmotionsMessageQueue, useMessageContentQueue } from '../../composables/queues'
|
||||
import { llmInferenceEndToken } from '../../constants'
|
||||
import { EMOTION_EmotionMotionName_value, EMOTION_VRMExpressionName_value, EmotionThinkMotionName } from '../../constants/emotions'
|
||||
import { useSpeakingStore } from '../../stores/audio'
|
||||
import { useChatStore } from '../../stores/chat'
|
||||
import { useLLM } from '../../stores/llm'
|
||||
|
||||
import { useSettings } from '../../stores/settings'
|
||||
import Live2DScene from '../Scenes/Live2D.vue'
|
||||
import VRMScene from '../Scenes/VRM.vue'
|
||||
|
||||
const live2DViewerRef = ref<{ setMotion: (motionName: string) => Promise<void> }>()
|
||||
const vrmViewerRef = ref<{ setExpression: (expression: string) => void }>()
|
||||
|
||||
const { stageView, elevenLabsApiKey } = storeToRefs(useSettings())
|
||||
const { mouthOpenSize } = storeToRefs(useSpeakingStore())
|
||||
const { audioContext, calculateVolume } = useAudioContext()
|
||||
const { streamSpeech } = useLLM()
|
||||
const { onBeforeMessageComposed, onBeforeSend, onTokenLiteral, onTokenSpecial, onStreamEnd } = useChatStore()
|
||||
|
||||
const audioAnalyser = ref<AnalyserNode>()
|
||||
const nowSpeaking = ref(false)
|
||||
const lipSyncStarted = ref(false)
|
||||
|
||||
const audioQueue = useQueue<{ audioBuffer: AudioBuffer, text: string }>({
|
||||
handlers: [
|
||||
(ctx) => {
|
||||
return new Promise((resolve) => {
|
||||
// Create an AudioBufferSourceNode
|
||||
const source = audioContext.createBufferSource()
|
||||
source.buffer = ctx.data.audioBuffer
|
||||
|
||||
// Connect the source to the AudioContext's destination (the speakers)
|
||||
source.connect(audioContext.destination)
|
||||
// Connect the source to the analyzer
|
||||
source.connect(audioAnalyser.value!)
|
||||
|
||||
// Start playing the audio
|
||||
nowSpeaking.value = true
|
||||
source.start(0)
|
||||
source.onended = () => {
|
||||
nowSpeaking.value = false
|
||||
resolve()
|
||||
}
|
||||
})
|
||||
},
|
||||
],
|
||||
})
|
||||
|
||||
const ttsQueue = useQueue<string>({
|
||||
handlers: [
|
||||
async (ctx) => {
|
||||
const now = Date.now()
|
||||
const res = await streamSpeech('https://airi-api.ayaka.io', elevenLabsApiKey.value, ctx.data, {
|
||||
// voice: 'ShanShan',
|
||||
// Quite good for English
|
||||
voice: 'Myriam',
|
||||
// Beatrice is not 'childish' like the others
|
||||
// voice: 'Beatrice',
|
||||
model_id: 'eleven_multilingual_v2',
|
||||
voice_settings: {
|
||||
stability: 0.4,
|
||||
similarity_boost: 0.5,
|
||||
},
|
||||
})
|
||||
const elapsed = Date.now() - now
|
||||
|
||||
// eslint-disable-next-line no-console
|
||||
console.debug('TTS took', elapsed, 'ms')
|
||||
|
||||
// Decode the ArrayBuffer into an AudioBuffer
|
||||
const audioBuffer = await audioContext.decodeAudioData(res)
|
||||
await audioQueue.add({ audioBuffer, text: ctx.data })
|
||||
},
|
||||
],
|
||||
})
|
||||
|
||||
ttsQueue.on('add', (content) => {
|
||||
// eslint-disable-next-line no-console
|
||||
console.debug('ttsQueue added', content)
|
||||
})
|
||||
|
||||
const messageContentQueue = useMessageContentQueue(ttsQueue)
|
||||
|
||||
const emotionsQueue = useQueue<Emotion>({
|
||||
handlers: [
|
||||
async (ctx) => {
|
||||
if (stageView.value === '3d') {
|
||||
const value = EMOTION_VRMExpressionName_value[ctx.data]
|
||||
if (!value)
|
||||
return
|
||||
|
||||
await vrmViewerRef.value!.setExpression(value)
|
||||
}
|
||||
else if (stageView.value === '2d') {
|
||||
await live2DViewerRef.value!.setMotion(EMOTION_EmotionMotionName_value[ctx.data])
|
||||
}
|
||||
},
|
||||
],
|
||||
})
|
||||
|
||||
const emotionMessageContentQueue = useEmotionsMessageQueue(emotionsQueue)
|
||||
emotionMessageContentQueue.onHandlerEvent('emotion', (emotion) => {
|
||||
// eslint-disable-next-line no-console
|
||||
console.debug('emotion detected', emotion)
|
||||
})
|
||||
|
||||
const delaysQueue = useDelayMessageQueue()
|
||||
delaysQueue.onHandlerEvent('delay', (delay) => {
|
||||
// eslint-disable-next-line no-console
|
||||
console.debug('delay detected', delay)
|
||||
})
|
||||
|
||||
function getVolumeWithMinMaxNormalizeWithFrameUpdates() {
|
||||
requestAnimationFrame(getVolumeWithMinMaxNormalizeWithFrameUpdates)
|
||||
if (!nowSpeaking.value)
|
||||
return
|
||||
|
||||
mouthOpenSize.value = calculateVolume(audioAnalyser.value!, 'linear')
|
||||
}
|
||||
|
||||
function setupLipSync() {
|
||||
if (!lipSyncStarted.value) {
|
||||
getVolumeWithMinMaxNormalizeWithFrameUpdates()
|
||||
audioContext.resume()
|
||||
lipSyncStarted.value = true
|
||||
}
|
||||
}
|
||||
|
||||
function setupAnalyser() {
|
||||
if (!audioAnalyser.value)
|
||||
audioAnalyser.value = audioContext.createAnalyser()
|
||||
}
|
||||
|
||||
onBeforeMessageComposed(async () => {
|
||||
setupAnalyser()
|
||||
setupLipSync()
|
||||
})
|
||||
|
||||
onBeforeSend(async () => {
|
||||
live2DViewerRef.value?.setMotion(EmotionThinkMotionName)
|
||||
})
|
||||
|
||||
onTokenLiteral(async (literal) => {
|
||||
await messageContentQueue.add(literal)
|
||||
})
|
||||
|
||||
onTokenSpecial(async (special) => {
|
||||
await delaysQueue.add(special)
|
||||
await emotionMessageContentQueue.add(special)
|
||||
})
|
||||
|
||||
onStreamEnd(async () => {
|
||||
await delaysQueue.add(llmInferenceEndToken)
|
||||
})
|
||||
|
||||
onUnmounted(() => {
|
||||
lipSyncStarted.value = false
|
||||
})
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<Live2DScene
|
||||
v-if="stageView === '2d'"
|
||||
ref="live2DViewerRef"
|
||||
:mouth-open-size="mouthOpenSize"
|
||||
model="/assets/live2d/models/hiyori_pro_zh/runtime/hiyori_pro_t11.model3.json"
|
||||
w="50%" min-w="50% <lg:full" min-h="100 sm:100" h-full flex-1
|
||||
/>
|
||||
<VRMScene
|
||||
v-else-if="stageView === '3d'"
|
||||
ref="vrmViewerRef"
|
||||
model="/assets/vrm/models/AvatarSample-B/AvatarSample_B.vrm"
|
||||
idle-animation="/assets/vrm/animations/idle_loop.vrma"
|
||||
w="50%" min-w="50% <lg:full" min-h="100 sm:100" h-full flex-1
|
||||
@error="console.error"
|
||||
/>
|
||||
</template>
|
||||
@@ -1,6 +1,18 @@
|
||||
<script setup lang="ts">
|
||||
import Header from '../components/Layouts/Header.vue'
|
||||
import ChatHistory from '../components/Widgets/ChatHistory.vue'
|
||||
import InputArea from '../components/Widgets/InputArea.vue'
|
||||
import Stage from '../components/Widgets/Stage.vue'
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<div h-full w-full>
|
||||
<MainStage />
|
||||
<div h-full max-h="[100vh]" max-w="[100vw]" p="2" flex="~ col" overflow-hidden>
|
||||
<Header />
|
||||
<div flex="~ row 1" max-h="[calc(100vh-270px)] <sm:[calc(100vh-280px)]" relative h-full w-full items-end gap-2>
|
||||
<Stage />
|
||||
<ChatHistory />
|
||||
</div>
|
||||
<InputArea />
|
||||
</div>
|
||||
</template>
|
||||
|
||||
|
||||
@@ -71,3 +71,24 @@ export const useAudioContext = defineStore('AudioContext', () => {
|
||||
calculateVolume,
|
||||
}
|
||||
})
|
||||
|
||||
export const useSpeakingStore = defineStore('SpeakingStore', () => {
|
||||
const nowSpeakingAvatarBorderOpacityMin = 30
|
||||
const nowSpeakingAvatarBorderOpacityMax = 100
|
||||
const mouthOpenSize = ref(0)
|
||||
const nowSpeaking = ref(false)
|
||||
|
||||
const nowSpeakingAvatarBorderOpacity = computed<number>(() => {
|
||||
if (!nowSpeaking.value)
|
||||
return nowSpeakingAvatarBorderOpacityMin
|
||||
|
||||
return ((nowSpeakingAvatarBorderOpacityMin
|
||||
+ (nowSpeakingAvatarBorderOpacityMax - nowSpeakingAvatarBorderOpacityMin) * mouthOpenSize.value) / 100)
|
||||
})
|
||||
|
||||
return {
|
||||
mouthOpenSize,
|
||||
nowSpeaking,
|
||||
nowSpeakingAvatarBorderOpacity,
|
||||
}
|
||||
})
|
||||
|
||||
@@ -0,0 +1,140 @@
|
||||
import type { AssistantMessage, Message } from '@xsai/shared-chat-completion'
|
||||
|
||||
import { defineStore, storeToRefs } from 'pinia'
|
||||
import SystemPromptV2 from '../constants/prompts/system-v2'
|
||||
import { useLLM } from '../stores/llm'
|
||||
import { asyncIteratorFromReadableStream } from '../utils/iterator'
|
||||
|
||||
export const useChatStore = defineStore('chat', () => {
|
||||
const { stream } = useLLM()
|
||||
const { t } = useI18n()
|
||||
const { openAiApiBaseURL, openAiApiKey, openAiModel } = storeToRefs(useSettings())
|
||||
|
||||
const onBeforeMessageComposedHooks = ref<Array<(message: string) => Promise<void>>>([])
|
||||
const onAfterMessageComposedHooks = ref<Array<(message: string) => Promise<void>>>([])
|
||||
const onBeforeSendHooks = ref<Array<(message: string) => Promise<void>>>([])
|
||||
const onAfterSendHooks = ref<Array<(message: string) => Promise<void>>>([])
|
||||
const onTokenLiteralHooks = ref<Array<(literal: string) => Promise<void>>>([])
|
||||
const onTokenSpecialHooks = ref<Array<(special: string) => Promise<void>>>([])
|
||||
const onStreamEndHooks = ref<Array<() => Promise<void>>>([])
|
||||
|
||||
function onBeforeMessageComposed(cb: (message: string) => Promise<void>) {
|
||||
onBeforeMessageComposedHooks.value.push(cb)
|
||||
}
|
||||
|
||||
function onAfterMessageComposed(cb: (message: string) => Promise<void>) {
|
||||
onAfterMessageComposedHooks.value.push(cb)
|
||||
}
|
||||
|
||||
function onBeforeSend(cb: (message: string) => Promise<void>) {
|
||||
onBeforeSendHooks.value.push(cb)
|
||||
}
|
||||
|
||||
function onAfterSend(cb: (message: string) => Promise<void>) {
|
||||
onAfterSendHooks.value.push(cb)
|
||||
}
|
||||
|
||||
function onTokenLiteral(cb: (literal: string) => Promise<void>) {
|
||||
onTokenLiteralHooks.value.push(cb)
|
||||
}
|
||||
|
||||
function onTokenSpecial(cb: (special: string) => Promise<void>) {
|
||||
onTokenSpecialHooks.value.push(cb)
|
||||
}
|
||||
|
||||
function onStreamEnd(cb: () => Promise<void>) {
|
||||
onStreamEndHooks.value.push(cb)
|
||||
}
|
||||
|
||||
const messages = ref<Array<Message>>([
|
||||
SystemPromptV2(
|
||||
t('prompt.prefix'),
|
||||
t('prompt.suffix'),
|
||||
),
|
||||
])
|
||||
const streamingMessage = ref<AssistantMessage>({ role: 'assistant', content: '' })
|
||||
|
||||
async function send(sendingMessage: string, options?: {
|
||||
baseUrl?: string
|
||||
apiKey?: string
|
||||
model?: { id: string }
|
||||
}) {
|
||||
if (!sendingMessage)
|
||||
return
|
||||
|
||||
for (const hook of onBeforeMessageComposedHooks.value) {
|
||||
await hook(sendingMessage)
|
||||
}
|
||||
|
||||
const {
|
||||
baseUrl = openAiApiBaseURL.value,
|
||||
apiKey = openAiApiKey.value,
|
||||
model = openAiModel.value,
|
||||
} = options ?? { }
|
||||
|
||||
streamingMessage.value = { role: 'assistant', content: '' }
|
||||
messages.value.push({ role: 'user', content: sendingMessage })
|
||||
messages.value.push(streamingMessage.value)
|
||||
const newMessages = messages.value.slice(0, messages.value.length - 1)
|
||||
|
||||
for (const hook of onAfterMessageComposedHooks.value) {
|
||||
await hook(sendingMessage)
|
||||
}
|
||||
|
||||
for (const hook of onBeforeSendHooks.value) {
|
||||
await hook(sendingMessage)
|
||||
}
|
||||
|
||||
const res = await stream(baseUrl, apiKey, model.id, newMessages)
|
||||
|
||||
for (const hook of onAfterSendHooks.value) {
|
||||
await hook(sendingMessage)
|
||||
}
|
||||
|
||||
let fullText = ''
|
||||
|
||||
const parser = useLlmmarkerParser({
|
||||
onLiteral: async (literal) => {
|
||||
for (const hook of onTokenLiteralHooks.value) {
|
||||
await hook(literal)
|
||||
}
|
||||
|
||||
streamingMessage.value.content += literal
|
||||
},
|
||||
onSpecial: async (special) => {
|
||||
for (const hook of onTokenSpecialHooks.value) {
|
||||
await hook(special)
|
||||
}
|
||||
|
||||
streamingMessage.value.content += special
|
||||
},
|
||||
})
|
||||
|
||||
for await (const textPart of asyncIteratorFromReadableStream(res.textStream, async v => v)) {
|
||||
fullText += textPart
|
||||
await parser.consume(textPart)
|
||||
}
|
||||
|
||||
await parser.end()
|
||||
|
||||
for (const hook of onStreamEndHooks.value) {
|
||||
await hook()
|
||||
}
|
||||
|
||||
// eslint-disable-next-line no-console
|
||||
console.debug('LLM output:', fullText)
|
||||
}
|
||||
|
||||
return {
|
||||
messages,
|
||||
streamingMessage,
|
||||
send,
|
||||
onBeforeMessageComposed,
|
||||
onAfterMessageComposed,
|
||||
onBeforeSend,
|
||||
onAfterSend,
|
||||
onTokenLiteral,
|
||||
onTokenSpecial,
|
||||
onStreamEnd,
|
||||
}
|
||||
})
|
||||
@@ -6,16 +6,35 @@ export const useSettings = defineStore('settings', () => {
|
||||
const openAiApiKey = useLocalStorage('settings/credentials/openai-api-key', '')
|
||||
const openAiApiBaseURL = useLocalStorage('settings/credentials/openai-api-base-url', '')
|
||||
const elevenLabsApiKey = useLocalStorage('settings/credentials/elevenlabs-api-key', '')
|
||||
const language = useLocalStorage('settings/language', 'en')
|
||||
|
||||
const language = useLocalStorage('settings/language', 'en-US')
|
||||
const stageView = useLocalStorage('settings/stage/view/model-renderer', '2d')
|
||||
const openAiModel = useLocalStorage<{ id: string, name?: string }>('settings/llm/openai/model', { id: 'openai/gpt-3.5-turbo', name: 'OpenAI GPT3.5 Turbo' })
|
||||
const isAudioInputOn = useLocalStorage('settings/audio/input', 'true')
|
||||
const selectedAudioDevice = useLocalStorage<MediaDeviceInfo | undefined>('settings/audio/input/device', undefined)
|
||||
const selectedAudioDeviceId = computed(() => selectedAudioDevice.value?.deviceId)
|
||||
const { audioInputs } = useDevicesList({ constraints: { audio: true }, requestPermissions: true })
|
||||
|
||||
watch(isAudioInputOn, async (value) => {
|
||||
if (value === 'false') {
|
||||
selectedAudioDevice.value = undefined
|
||||
}
|
||||
if (value === 'true') {
|
||||
selectedAudioDevice.value = audioInputs.value[0]
|
||||
}
|
||||
})
|
||||
|
||||
watch(language, value => i18n.global.locale.value = value)
|
||||
|
||||
return {
|
||||
openAiApiKey,
|
||||
openAiApiBaseURL,
|
||||
openAiModel,
|
||||
elevenLabsApiKey,
|
||||
language,
|
||||
stageView,
|
||||
isAudioInputOn,
|
||||
selectedAudioDevice,
|
||||
selectedAudioDeviceId,
|
||||
}
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user