feat: use whisper result as message

This commit is contained in:
Neko Ayaka
2024-12-11 22:53:37 +08:00
parent 5262c79702
commit cbb13e8d03
9 changed files with 235 additions and 79 deletions
+2
View File
@@ -70,7 +70,9 @@
"@airi-proj/elevenlabs": "workspace:^",
"@iconify-json/carbon": "^1.2.4",
"@iconify-json/eos-icons": "^1.2.1",
"@iconify-json/lucide": "^1.2.18",
"@iconify-json/solar": "^1.2.1",
"@iconify-json/svg-spinners": "^1.2.1",
"@intlify/unplugin-vue-i18n": "^6.0.1",
"@shikijs/markdown-it": "^1.24.2",
"@types/markdown-it-link-attributes": "^3.0.5",
@@ -24,43 +24,43 @@ defineExpose({
<div z="10" top="2" absolute w-full flex="~ col" gap-2>
<div flex="~ row" w-full flex-wrap gap-2>
<button
rounded-lg bg="zinc-100 dark:zinc-800/50" px-2 py-1
rounded-lg bg="zinc-100/70 dark:zinc-800/50" px-2 py-1 backdrop-blur-sm
@click="modelRef?.setMotion(EmotionSurpriseMotionName)"
>
🤯 Surprised
</button>
<button
rounded-lg bg="zinc-100 dark:zinc-800/50" px-2 py-1
rounded-lg bg="zinc-100/70 dark:zinc-800/50" px-2 py-1 backdrop-blur-sm
@click="modelRef?.setMotion(EmotionSadMotionName)"
>
😫 Sad
</button>
<button
rounded-lg bg="zinc-100 dark:zinc-800/50" px-2 py-1
rounded-lg bg="zinc-100/70 dark:zinc-800/50" px-2 py-1 backdrop-blur-sm
@click="modelRef?.setMotion(EmotionAngryMotionName)"
>
😠 Angry
</button>
<button
rounded-lg bg="zinc-100 dark:zinc-800/50" px-2 py-1
rounded-lg bg="zinc-100/70 dark:zinc-800/50" px-2 py-1 backdrop-blur-sm
@click="modelRef?.setMotion(EmotionHappyMotionName)"
>
😄 Happy
</button>
<button
rounded-lg bg="zinc-100 dark:zinc-800/50" px-2 py-1
rounded-lg bg="zinc-100/70 dark:zinc-800/50" px-2 py-1 backdrop-blur-sm
@click="modelRef?.setMotion(EmotionAwkwardMotionName)"
>
😳 Awkward
</button>
<button
rounded-lg bg="zinc-100 dark:zinc-800/50" px-2 py-1
rounded-lg bg="zinc-100/70 dark:zinc-800/50" px-2 py-1 backdrop-blur-sm
@click="modelRef?.setMotion(EmotionQuestionMotionName)"
>
🤔 Question
</button>
<button
rounded-lg bg="zinc-100 dark:zinc-800/50" px-2 py-1
rounded-lg bg="zinc-100/70 dark:zinc-800/50" px-2 py-1 backdrop-blur-sm
@click="modelRef?.setMotion(EmotionThinkMotionName)"
>
🤨 Think
+145 -59
View File
@@ -41,7 +41,6 @@ const { streamSpeech, stream, models } = useLLM()
const { audioContext, calculateVolume } = useAudioContext()
const { process } = useMarkdown()
const { audioInputs } = useDevicesList({ constraints: { audio: true }, requestPermissions: true })
const { transcribe: generate } = useWhisper(WhisperWorker)
const listening = ref(false)
const live2DViewerRef = ref<{ setMotion: (motionName: string) => Promise<void> }>()
@@ -55,6 +54,7 @@ const mouthOpenSize = ref(0)
const nowSpeaking = ref(false)
const lipSyncStarted = ref(false)
const selectedAudioDevice = ref<MediaDeviceInfo>()
const isAudioInputOn = ref('false')
const selectedAudioDeviceId = computed(() => selectedAudioDevice.value?.deviceId)
const nowSpeakingAvatarBorderOpacity = computed<number>(() => {
@@ -65,39 +65,6 @@ const nowSpeakingAvatarBorderOpacity = computed<number>(() => {
+ (nowSpeakingAvatarBorderOpacityMax - nowSpeakingAvatarBorderOpacityMin) * mouthOpenSize.value) / 100)
})
async function handleTranscription(buffer: Float32Array) {
await audioContext.resume()
// Convert Float32Array to WAV format
const audioBase64 = await encodeWAVToBase64(buffer, audioContext.sampleRate)
generate({ type: 'generate', data: { audio: audioBase64, language: 'en' } })
}
useMicVAD(selectedAudioDeviceId, {
onSpeechStart: () => {
// TODO: interrupt the playback
// TODO: interrupt any of the ongoing TTS
// TODO: interrupt any of the ongoing LLM requests
// TODO: interrupt any of the ongoing animation of Live2D or VRM
// TODO: once interrupted, we should somehow switch to listen or thinking
// emotion / expression?
listening.value = true
},
// VAD misfire means while speech end is detected but
// the frames of the segment of the audio buffer
// is not enough to be considered as a speech segment
// which controlled by the `minSpeechFrames` parameter
onVADMisfire: () => {
// TODO: do audio buffer send to whisper
listening.value = false
},
onSpeechEnd: (buffer) => {
// TODO: do audio buffer send to whisper
listening.value = false
handleTranscription(buffer)
},
})
function handleModelChange(event: Event) {
const target = event.target as HTMLSelectElement
const found = supportedModels.value.find(m => m.id === target.value)
@@ -271,6 +238,63 @@ async function onSendMessage(sendingMessage: string) {
console.debug('Full text:', fullText)
}
const { transcribe: generate, load: loadWhisper, status: whisperStatus, terminate } = useWhisper(WhisperWorker, {
onComplete: async (res) => {
await onSendMessage(res)
},
})
function handleLoadWhisper() {
if (whisperStatus.value === 'loading')
return
loadWhisper()
}
async function handleTranscription(buffer: Float32Array) {
await audioContext.resume()
// Convert Float32Array to WAV format
const audioBase64 = await encodeWAVToBase64(buffer, audioContext.sampleRate)
generate({ type: 'generate', data: { audio: audioBase64, language: 'en' } })
}
const { destroy } = useMicVAD(selectedAudioDeviceId, {
onSpeechStart: () => {
// TODO: interrupt the playback
// TODO: interrupt any of the ongoing TTS
// TODO: interrupt any of the ongoing LLM requests
// TODO: interrupt any of the ongoing animation of Live2D or VRM
// TODO: once interrupted, we should somehow switch to listen or thinking
// emotion / expression?
listening.value = true
},
// VAD misfire means while speech end is detected but
// the frames of the segment of the audio buffer
// is not enough to be considered as a speech segment
// which controlled by the `minSpeechFrames` parameter
onVADMisfire: () => {
// TODO: do audio buffer send to whisper
listening.value = false
},
onSpeechEnd: (buffer) => {
// TODO: do audio buffer send to whisper
listening.value = false
handleTranscription(buffer)
},
})
watch(isAudioInputOn, async (value) => {
if (value === 'false') {
selectedAudioDevice.value = undefined
destroy()
terminate()
}
if (value === 'true') {
selectedAudioDevice.value = audioInputs.value[0]
}
})
watch([openAiApiBaseURL, openAiApiKey], async ([baseUrl, apiKey]) => {
if (!baseUrl || !apiKey) {
supportedModels.value = []
@@ -294,7 +318,7 @@ onUnmounted(() => {
<template>
<div h-full max-h="[100vh]" max-w="[100vw]" p="2" flex="~ col" overflow-hidden>
<div flex="~" mb-1 w-full gap-2>
<header flex="~" mb-1 w-full gap-2>
<div flex="~ 1" w-full items-center gap-2 text-nowrap text-2xl>
<div i-solar:cat-outline text="[#ed869d]" />
<div font-cute>
@@ -302,8 +326,8 @@ onUnmounted(() => {
</div>
</div>
<Settings />
</div>
<div flex="~ row 1" max-h="[calc(100vh-220px)] <sm:[calc(100vh-320px)]" relative h-full w-full items-end gap-2>
</header>
<div flex="~ row 1" max-h="[calc(100vh-270px)] <sm:[calc(100vh-320px)]" relative h-full w-full items-end gap-2>
<Live2DScene
v-if="stageView === '2d'"
ref="live2DViewerRef"
@@ -322,7 +346,7 @@ onUnmounted(() => {
<div
class="relative <lg:(absolute bottom-0 from-zinc-100/80 to-zinc-800/0 bg-gradient-to-t p-2 dark:from-zinc-800/80)"
px="<sm:2" py="<sm:2" rounded="lg"
w="50% <lg:full" flex="~ col 1" overflow-hidden max-h="[calc(100vh-220px)]"
w="50% <lg:full" flex="~ col 1" overflow-hidden max-h="[calc(100vh-280px)]"
>
<div h-full w-full overflow-scroll>
<div v-for="(message, index) in messages" :key="index" mb-2>
@@ -383,7 +407,7 @@ onUnmounted(() => {
</div>
</div>
<div flex="~ row" my="2" space-x="2" w-full self-end>
<div flex="~ col" w-full space-y="2">
<div flex="~ col" w="100%" space-y="2">
<select
p="2"
bg="zinc-100 dark:zinc-700" w-full rounded-lg
@@ -421,30 +445,92 @@ onUnmounted(() => {
v-model="messageInput"
placeholder="Message"
p="2" bg="zinc-100 dark:zinc-700"
w="[92%]" rounded-lg outline-none min-h="[100px]"
w-full rounded-lg outline-none min-h="[100px]"
@submit="onSendMessage"
/>
<button
flex="~ row"
p="2" bg="zinc-100 dark:zinc-700"
w="[8%]" items-center justify-center rounded-lg outline-none
transition="all ease-in-out"
@click="listening = !listening"
>
<Transition mode="out-in">
<div v-if="listening" flex="~ row" items-center justify-center space-x-1>
<div i-carbon:microphone-filled text-red />
</div>
<div v-else flex="~ row" items-center justify-center space-x-1>
<div i-carbon:microphone text-inherit />
<span>
Talk
</span>
</div>
</Transition>
</button>
</div>
</div>
<div h-full w="[16%]" flex="~ col" gap-2>
<fieldset
flex="~ row"
bg="zinc-100 dark:zinc-700"
text="sm zinc-400 dark:zinc-500"
appearance-none gap-1 rounded-lg rounded-md border-none p-1
>
<label
:class="[isAudioInputOn === 'true' ? 'bg-zinc-300 text-zinc-900 dark:bg-zinc-200 dark:text-zinc-800' : '']"
min-h="7.75" flex="~" w-full cursor-pointer items-center justify-center rounded-md
>
<input
v-model="isAudioInputOn"
:checked="isAudioInputOn === 'true'"
:aria-checked="isAudioInputOn === 'true'"
name="isAudioInputOn"
type="radio"
role="radio"
value="true"
hidden appearance-none outline-none
>
<div select-none>ON</div>
</label>
<label
:class="[isAudioInputOn === 'false' ? 'bg-zinc-300 text-zinc-900 dark:bg-zinc-200 dark:text-zinc-800' : '']"
min-h="7.75" flex="~" w-full cursor-pointer items-center justify-center rounded-md
>
<input
v-model="isAudioInputOn"
:checked="isAudioInputOn === 'false'"
:aria-checked="isAudioInputOn === 'false'"
name="stageView"
type="radio"
role="radio"
value="false"
hidden appearance-none outline-none
>
<div select-none>OFF</div>
</label>
</fieldset>
<button
flex="~ row"
p="2" bg="zinc-100 dark:zinc-700" min-h="9.75"
min-w-20 w-full items-center justify-center rounded-lg outline-none
transition="all ease-in-out"
@click="handleLoadWhisper"
>
<Transition mode="out-in">
<div v-if="whisperStatus === null" flex="~ row" items-center justify-center space-x-1>
Load Model
</div>
<div v-else-if="whisperStatus === 'loading'" flex="~ row" items-center justify-center space-x-1>
<div i-svg-spinners:bouncing-ball text-pink />
<span>Loading</span>
</div>
<div v-else-if="whisperStatus === 'ready'" flex="~ row" items-center justify-center space-x-1>
<div i-lucide:check text-green />
<span>Ready</span>
</div>
</Transition>
</button>
<button
flex="~ row" h-full
p="2" bg="zinc-100 dark:zinc-700"
min-w-20 w-full items-center justify-center rounded-lg outline-none
transition="all ease-in-out"
@click="listening = !listening"
>
<Transition mode="out-in">
<div v-if="listening" flex="~ row" items-center justify-center space-x-1>
<div i-carbon:microphone-filled text-red />
</div>
<div v-else flex="~ row" items-center justify-center space-x-1>
<div i-carbon:microphone text-inherit />
<span>
Talk
</span>
</div>
</Transition>
</button>
</div>
</div>
</div>
</template>
@@ -114,31 +114,31 @@ defineExpose({
</div>
<div flex="~ row" w-full flex-wrap gap-2>
<button
rounded-lg bg="zinc-100 dark:zinc-800/50" px-2 py-1
rounded-lg bg="zinc-100/70 dark:zinc-800/50" px-2 py-1 backdrop-blur-sm
@click="modelRef?.setExpression('neutral')"
>
🙂 Neutral
</button>
<button
rounded-lg bg="zinc-100 dark:zinc-800/50" px-2 py-1
rounded-lg bg="zinc-100/70 dark:zinc-800/50" px-2 py-1 backdrop-blur-sm
@click="modelRef?.setExpression('surprised')"
>
🤯 Surprised
</button>
<button
rounded-lg bg="zinc-100 dark:zinc-800/50" px-2 py-1
rounded-lg bg="zinc-100/70 dark:zinc-800/50" px-2 py-1 backdrop-blur-sm
@click="modelRef?.setExpression('sad')"
>
😫 Sad
</button>
<button
rounded-lg bg="zinc-100 dark:zinc-800/50" px-2 py-1
rounded-lg bg="zinc-100/70 dark:zinc-800/50" px-2 py-1 backdrop-blur-sm
@click="modelRef?.setExpression('angry')"
>
😠 Angry
</button>
<button
rounded-lg bg="zinc-100 dark:zinc-800/50" px-2 py-1
rounded-lg bg="zinc-100/70 dark:zinc-800/50" px-2 py-1 backdrop-blur-sm
@click="modelRef?.setExpression('happy')"
>
😄 Happy
+4 -3
View File
@@ -7,9 +7,10 @@ import { defu } from 'defu'
export function useMicVAD(deviceId: MaybeRef<ConstrainDOMString | undefined>, options?: Partial<RealTimeVADOptions> & { auto?: boolean }) {
const opts = defu<Partial<RealTimeVADOptions> & { auto?: boolean }, Array<Omit<RealTimeVADOptions, 'stream'> & { auto?: boolean }>>(options ?? {}, {
...getDefaultRealTimeVADOptions('v5'),
positiveSpeechThreshold: 0.2, // default is 0.5
negativeSpeechThreshold: 0.08, // default is 0.5 - 0.15
minSpeechFrames: 5, // default is 9
preSpeechPadFrames: 20,
positiveSpeechThreshold: 0.5, // default is 0.5
negativeSpeechThreshold: 0.5 - 0.15, // default is 0.5 - 0.15
minSpeechFrames: 15, // default is 9
// WORKAROUND: temporary workaround for onnxruntime-web, since @ricky0123/vad-web
// uses hardcoded version of onnxruntime-web@1.14.0 to fetch the already non-existing
// ort-wasm-simd-threaded.mjs file and its WASM binary, we are going to force
+39 -2
View File
@@ -1,7 +1,34 @@
import type { MessageEvents, MessageGenerate, ProgressMessageEvents } from '../libs/workers/types'
import { defu } from 'defu'
export function useWhisper(url: string) {
const { post: whisperPost, data: whisperData, terminate } = useWebWorker<MessageEvents>(url, { type: 'module' })
export interface UseWhisperOptions {
onLoading: (message: string) => void
onInitiate: (message: ProgressMessageEvents) => void
onProgress: (message: ProgressMessageEvents) => void
onDone: (message: ProgressMessageEvents) => void
onReady: () => void
onStart: () => void
onUpdate: (tps: number) => void
onComplete: (output: string) => void
}
export function useWhisper(url: string, options?: Partial<UseWhisperOptions>) {
const opts = defu<Partial<UseWhisperOptions>, UseWhisperOptions[]>(options, {
onLoading: () => {},
onInitiate: () => {},
onProgress: () => {},
onDone: () => {},
onReady: () => {},
onStart: () => {},
onUpdate: () => {},
onComplete: () => {},
})
const {
post: whisperPost,
data: whisperData,
terminate,
} = useWebWorker<MessageEvents>(url, { type: 'module' })
const status = ref<'loading' | 'ready' | null>(null)
const loadingMessage = ref('')
@@ -15,10 +42,12 @@ export function useWhisper(url: string) {
case 'loading':
status.value = 'loading'
loadingMessage.value = e.data
opts.onLoading?.(e.data)
break
case 'initiate':
loadingProgress.value.push(e)
opts.onInitiate?.(e)
break
case 'progress':
@@ -28,22 +57,27 @@ export function useWhisper(url: string) {
}
return item
})
opts.onProgress?.(e)
break
case 'done':
loadingProgress.value = loadingProgress.value.filter(item => item.file !== e.file)
opts.onDone?.(e)
break
case 'ready':
status.value = 'ready'
opts.onReady?.()
break
case 'start':
transcribing.value = true
opts.onStart?.()
break
case 'update':
tps.value = e.tps
opts.onUpdate?.(e.tps)
break
case 'complete':
@@ -51,6 +85,7 @@ export function useWhisper(url: string) {
result.value = e.output[0] || ''
// eslint-disable-next-line no-console
console.debug('Whisper result:', result.value)
opts.onComplete?.(e.output[0])
break
}
})
@@ -67,5 +102,7 @@ export function useWhisper(url: string) {
transcribing,
tps,
result,
load: () => whisperPost({ type: 'load' }),
terminate,
}
}
+6 -3
View File
@@ -27,7 +27,7 @@ class AutomaticSpeechRecognitionPipeline {
static model: Promise<PreTrainedModel>
static async getInstance(progress_callback?: ProgressCallback) {
this.model_id = 'onnx-community/whisper-base'
this.model_id = 'onnx-community/whisper-large-v3-turbo'
this.tokenizer ??= AutoTokenizer.from_pretrained(this.model_id, {
progress_callback,
@@ -39,7 +39,9 @@ class AutomaticSpeechRecognitionPipeline {
this.model ??= WhisperForConditionalGeneration.from_pretrained(this.model_id, {
dtype: {
encoder_model: 'fp32', // 'fp16' works too
// [v3.x] Cannot load whisper-v3-large-turbo · Issue #989 · huggingface/transformers.js
// https://github.com/huggingface/transformers.js/issues/989
encoder_model: 'fp16', // 'fp16' works too
decoder_model_merged: 'q4', // or 'fp32' ('fp16' is broken)
},
device: 'webgpu',
@@ -150,7 +152,8 @@ async function load() {
// Run model with dummy input to compile shaders
await model.generate({
input_features: full([1, 80, 3000], 0.0),
// input_features: full([1, 80, 3000], 0.0), // for fp32
input_features: full([1, 128, 3000], 0.0), // for fp16
max_new_tokens: 1,
})
+7
View File
@@ -18,5 +18,12 @@ declare module 'vue-router/auto-routes' {
* Route name map generated by unplugin-vue-router
*/
export interface RouteNamedMap {
'/': RouteRecordInfo<'/', '/', Record<never, never>, Record<never, never>>,
'/audio': RouteRecordInfo<'/audio', '/audio', Record<never, never>, Record<never, never>>,
'/queue': RouteRecordInfo<'/queue', '/queue', Record<never, never>, Record<never, never>>,
'/test/filter-message': RouteRecordInfo<'/test/filter-message', '/test/filter-message', Record<never, never>, Record<never, never>>,
'/test/queues/delays': RouteRecordInfo<'/test/queues/delays', '/test/queues/delays', Record<never, never>, Record<never, never>>,
'/test/queues/emotions': RouteRecordInfo<'/test/queues/emotions', '/test/queues/emotions', Record<never, never>, Record<never, never>>,
'/test/queues/messages': RouteRecordInfo<'/test/queues/messages', '/test/queues/messages', Record<never, never>, Record<never, never>>,
}
}
+20
View File
@@ -253,9 +253,15 @@ importers:
'@iconify-json/eos-icons':
specifier: ^1.2.1
version: 1.2.1
'@iconify-json/lucide':
specifier: ^1.2.18
version: 1.2.18
'@iconify-json/solar':
specifier: ^1.2.1
version: 1.2.1
'@iconify-json/svg-spinners':
specifier: ^1.2.1
version: 1.2.1
'@intlify/unplugin-vue-i18n':
specifier: ^6.0.1
version: 6.0.1(@vue/compiler-dom@3.5.13)(eslint@9.16.0(jiti@2.4.0))(rollup@2.79.1)(typescript@5.7.2)(vue-i18n@10.0.5(vue@3.5.13(typescript@5.7.2)))(vue@3.5.13(typescript@5.7.2))
@@ -1702,9 +1708,15 @@ packages:
'@iconify-json/eos-icons@1.2.1':
resolution: {integrity: sha512-nqucN9E670dRHDk5osNNObhxQg/eBI8wvYA+zWaOMsJR0g8VodF9z/cpDvMH7YyUm4elOF9r8cGIj3byFBFLRQ==}
'@iconify-json/lucide@1.2.18':
resolution: {integrity: sha512-SFBoHBSULntem84iWxGM9NzlP6QY/dwjxC5t4c1lC7+xq31Fous8JkppoJih0/ICJ3CsbppRZeb11y3iLcwwug==}
'@iconify-json/solar@1.2.1':
resolution: {integrity: sha512-LA3WZ7HOi5O/sEc2M+BcQNJYNFDnUKP+ZldmNf8kwndnCuFMo+fukt6tSu4BqYRsRgp/Ku4xnys1JGBDi+QtCQ==}
'@iconify-json/svg-spinners@1.2.1':
resolution: {integrity: sha512-QZNA4YzFD2zqdC6nIBJM6WlAGakUCjvMt92Ks1R4XFxkd76Ps3rdiauYWESDRZvNYURAByp2b9cwZarFula65g==}
'@iconify/types@2.0.0':
resolution: {integrity: sha512-+wluvCrRhXrhyOmRDJ3q8mux9JkKy5SJ/v8ol2tu4FVjyYvtEzkc/3pK15ET6RKg4b4w4BmTk1+gsCUhf21Ykg==}
@@ -8478,10 +8490,18 @@ snapshots:
dependencies:
'@iconify/types': 2.0.0
'@iconify-json/lucide@1.2.18':
dependencies:
'@iconify/types': 2.0.0
'@iconify-json/solar@1.2.1':
dependencies:
'@iconify/types': 2.0.0
'@iconify-json/svg-spinners@1.2.1':
dependencies:
'@iconify/types': 2.0.0
'@iconify/types@2.0.0': {}
'@iconify/utils@2.1.33':