refactor: migrate out vad related examples
This commit is contained in:
Neko Ayaka
@@ -1,22 +0,0 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<title>Project AIRI VAD + ASR + LLM Chat Playground</title>
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=0" />
|
||||
<link rel="icon" type="image/svg+xml" href="/favicon.svg" />
|
||||
<script>
|
||||
;(function () {
|
||||
const prefersDark = window.matchMedia && window.matchMedia('(prefers-color-scheme: dark)').matches
|
||||
const setting = localStorage.getItem('vueuse-color-scheme') || 'auto'
|
||||
if (setting === 'dark' || (prefersDark && setting !== 'light'))
|
||||
document.documentElement.classList.toggle('dark', true)
|
||||
})()
|
||||
</script>
|
||||
</head>
|
||||
<body class="font-sans">
|
||||
<div id="app"></div>
|
||||
<script type="module" src="/src/main.ts"></script>
|
||||
<noscript> This website requires JavaScript to function properly. Please enable JavaScript to continue. </noscript>
|
||||
</body>
|
||||
</html>
|
||||
@@ -1 +0,0 @@
|
||||
title: VAD
|
||||
@@ -1 +0,0 @@
|
||||
title: VAD
|
||||
@@ -1,13 +0,0 @@
|
||||
[build]
|
||||
base = "/"
|
||||
command = "pnpm -F @proj-airi/vad-asr-chat... run build"
|
||||
publish = "/apps/vad-asr-chat/dist"
|
||||
|
||||
[build.environment]
|
||||
NODE_VERSION = "23"
|
||||
|
||||
[[redirects]]
|
||||
from = "/*"
|
||||
to = "/index.html"
|
||||
status = 200
|
||||
force = false
|
||||
@@ -1,46 +0,0 @@
|
||||
{
|
||||
"name": "@proj-airi/vad-asr-chat",
|
||||
"type": "module",
|
||||
"private": true,
|
||||
"description": "Voice Activity Detector & Automatic Speech Recognition & Chat",
|
||||
"author": {
|
||||
"name": "Neko Ayaka",
|
||||
"email": "neko@ayaka.moe",
|
||||
"url": "https://github.com/nekomeowww"
|
||||
},
|
||||
"license": "MIT",
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/moeru-ai/airi.git",
|
||||
"directory": "apps/vad-asr-chat"
|
||||
},
|
||||
"scripts": {
|
||||
"dev": "vite",
|
||||
"build": "vite build",
|
||||
"preview": "vite preview",
|
||||
"typecheck": "vue-tsc --noEmit"
|
||||
},
|
||||
"dependencies": {
|
||||
"@huggingface/transformers": "^3.4.2",
|
||||
"@llama-flow/core": "^0.3.4",
|
||||
"@vueuse/core": "^13.0.0",
|
||||
"@xsai/generate-transcription": "catalog:",
|
||||
"@xsai/shared": "catalog:",
|
||||
"@xsai/shared-chat": "catalog:",
|
||||
"@xsai/stream-text": "catalog:",
|
||||
"defu": "^6.1.4",
|
||||
"es-toolkit": "^1.34.1",
|
||||
"vue": "^3.5.13"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@iconify-json/solar": "^1.2.2",
|
||||
"@iconify-json/svg-spinners": "^1.2.2",
|
||||
"@types/audioworklet": "^0.0.72",
|
||||
"@unocss/reset": "^66.1.0-beta.10",
|
||||
"@vitejs/plugin-vue": "^5.2.3",
|
||||
"unplugin-vue-router": "^0.12.0",
|
||||
"vite": "^6.2.5",
|
||||
"vue-router": "^4.5.0",
|
||||
"vue-tsc": "^3.0.0-alpha.2"
|
||||
}
|
||||
}
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 15 KiB |
File diff suppressed because one or more lines are too long
|
Before Width: | Height: | Size: 23 KiB |
@@ -1,56 +0,0 @@
|
||||
<script setup lang="ts">
|
||||
import { useDark, useToggle } from '@vueuse/core'
|
||||
import { RouterLink, RouterView } from 'vue-router'
|
||||
|
||||
const isDark = useDark()
|
||||
const toggleDark = useToggle(isDark)
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<div mx-auto h-full max-w-screen-lg flex flex-col gap-2 p-4>
|
||||
<header flex flex-row items-center justify-between>
|
||||
<h1 text-2xl>
|
||||
VAD + ASR + LLM Chat Playground
|
||||
</h1>
|
||||
<div flex flex-row items-center gap-2>
|
||||
<button text-lg @click="() => toggleDark()">
|
||||
<div v-if="isDark" i-solar:moon-stars-bold-duotone />
|
||||
<div v-else i-solar:sun-bold />
|
||||
</button>
|
||||
<a href="https://github.com/moeru-ai/airi/tree/main/apps/vad-asr-chat">
|
||||
<div i-simple-icons:github />
|
||||
</a>
|
||||
</div>
|
||||
</header>
|
||||
<nav bg="neutral-100 dark:neutral-800" w-fit flex items-center of-hidden rounded-lg>
|
||||
<RouterLink
|
||||
to="/" px-3 py-2 bg="hover:neutral-200 dark:hover:neutral-700"
|
||||
transition="all duration-250 ease-in-out"
|
||||
>
|
||||
<h1>WASM</h1>
|
||||
</RouterLink>
|
||||
</nav>
|
||||
<RouterView />
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<style>
|
||||
html,
|
||||
body,
|
||||
#app {
|
||||
height: 100%;
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
overscroll-behavior: none;
|
||||
}
|
||||
|
||||
html {
|
||||
background: #fff;
|
||||
transition: all 0.3s ease-in-out;
|
||||
}
|
||||
|
||||
html.dark {
|
||||
background: #121212;
|
||||
color-scheme: dark;
|
||||
}
|
||||
</style>
|
||||
@@ -1,39 +0,0 @@
|
||||
<script lang="ts" setup>
|
||||
import { watchEffect } from 'vue'
|
||||
|
||||
import TransitionVertical from './TransitionVertical.vue'
|
||||
|
||||
const props = defineProps<{
|
||||
default?: boolean
|
||||
label?: string
|
||||
}>()
|
||||
const visible = defineModel<boolean>({ default: false })
|
||||
watchEffect(() => {
|
||||
if (props.default != null) {
|
||||
visible.value = !!props.default
|
||||
}
|
||||
})
|
||||
|
||||
function setVisible(value: boolean) {
|
||||
visible.value = value
|
||||
return value
|
||||
}
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<div>
|
||||
<slot name="trigger" v-bind="{ visible, setVisible }">
|
||||
<button
|
||||
sticky top-0 z-10 flex items-center justify-between px2 py1 text-sm backdrop-blur-xl
|
||||
@click="visible = !visible"
|
||||
>
|
||||
<span>
|
||||
{{ props.label ?? 'Collapsable' }}
|
||||
</span> <span op50>{{ visible ? '▲' : '▼' }}</span>
|
||||
</button>
|
||||
</slot>
|
||||
<TransitionVertical>
|
||||
<slot v-if="visible" v-bind="{ visible, setVisible }" />
|
||||
</TransitionVertical>
|
||||
</div>
|
||||
</template>
|
||||
@@ -1,36 +0,0 @@
|
||||
<script setup lang="ts">
|
||||
import Input from './Input.vue'
|
||||
|
||||
const props = defineProps<{
|
||||
label?: string
|
||||
description?: string
|
||||
placeholder?: string
|
||||
required?: boolean
|
||||
type?: string
|
||||
inputClass?: string
|
||||
}>()
|
||||
|
||||
const modelValue = defineModel<string>({ required: true })
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<div max-w-full>
|
||||
<label flex="~ col gap-2">
|
||||
<div>
|
||||
<div class="flex items-center gap-1 text-sm font-medium">
|
||||
{{ props.label }}
|
||||
<span v-if="props.required !== false" class="text-red-500">*</span>
|
||||
</div>
|
||||
<div class="text-xs text-neutral-500 dark:text-neutral-400" text-nowrap>
|
||||
{{ props.description }}
|
||||
</div>
|
||||
</div>
|
||||
<Input
|
||||
v-model="modelValue"
|
||||
:type="props.type"
|
||||
:placeholder="props.placeholder"
|
||||
:class="props.inputClass"
|
||||
/>
|
||||
</label>
|
||||
</div>
|
||||
</template>
|
||||
@@ -1,21 +0,0 @@
|
||||
<script setup lang="ts">
|
||||
const props = defineProps<{
|
||||
type?: string
|
||||
}>()
|
||||
|
||||
const modelValue = defineModel<string>({ required: true })
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<input
|
||||
v-model="modelValue"
|
||||
:type="props.type || 'text'"
|
||||
border="focus:blue-300 dark:focus:blue-400/50 2 solid neutral-100 dark:neutral-900"
|
||||
transition="all duration-200 ease-in-out"
|
||||
text="disabled:neutral-400 dark:disabled:neutral-600"
|
||||
cursor="disabled:not-allowed"
|
||||
w-full rounded-lg px-2 py-1 text-nowrap text-sm outline-none
|
||||
shadow="sm"
|
||||
bg="neutral-50 dark:neutral-950 focus:neutral-50 dark:focus:neutral-900"
|
||||
>
|
||||
</template>
|
||||
@@ -1,38 +0,0 @@
|
||||
<script setup lang="ts">
|
||||
import Collapsable from './Collapsable.vue'
|
||||
|
||||
withDefaults(defineProps<{
|
||||
title: string
|
||||
icon: string
|
||||
innerClass?: string
|
||||
expand?: boolean
|
||||
}>(), {
|
||||
expand: true,
|
||||
})
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<Collapsable :default="expand">
|
||||
<template #trigger="slotProps">
|
||||
<button
|
||||
class="w-full flex items-center justify-between rounded-lg px-4 py-3 outline-none transition-all duration-250 ease-in-out"
|
||||
bg="neutral-100 dark:neutral-800"
|
||||
hover="bg-neutral-200 dark:bg-neutral-700"
|
||||
@click="slotProps.setVisible(!slotProps.visible)"
|
||||
>
|
||||
<div flex gap-1.5>
|
||||
<div :class="icon" size-6 />
|
||||
{{ title }}
|
||||
</div>
|
||||
<div
|
||||
i-solar:alt-arrow-down-bold-duotone
|
||||
transition="transform duration-250"
|
||||
:class="{ 'rotate-180': slotProps.visible }"
|
||||
/>
|
||||
</button>
|
||||
</template>
|
||||
<div grid gap-4 p-4 :class="innerClass">
|
||||
<slot />
|
||||
</div>
|
||||
</Collapsable>
|
||||
</template>
|
||||
@@ -1,134 +0,0 @@
|
||||
<script setup lang="ts">
|
||||
// From: https://stackoverflow.com/a/71426342/22392721
|
||||
interface Props {
|
||||
duration?: number
|
||||
easingEnter?: string
|
||||
easingLeave?: string
|
||||
opacityClosed?: number
|
||||
opacityOpened?: number
|
||||
}
|
||||
|
||||
const props = withDefaults(defineProps<Props>(), {
|
||||
duration: 250,
|
||||
easingEnter: 'ease-in-out',
|
||||
easingLeave: 'ease-in-out',
|
||||
opacityClosed: 0,
|
||||
opacityOpened: 1,
|
||||
})
|
||||
|
||||
const closed = '0px'
|
||||
|
||||
interface initialStyle {
|
||||
height: string
|
||||
width: string
|
||||
position: string
|
||||
visibility: string
|
||||
overflow: string
|
||||
paddingTop: string
|
||||
paddingBottom: string
|
||||
borderTopWidth: string
|
||||
borderBottomWidth: string
|
||||
marginTop: string
|
||||
marginBottom: string
|
||||
}
|
||||
|
||||
function getElementStyle(element: HTMLElement) {
|
||||
return {
|
||||
height: element.style.height,
|
||||
width: element.style.width,
|
||||
position: element.style.position,
|
||||
visibility: element.style.visibility,
|
||||
overflow: element.style.overflow,
|
||||
paddingTop: element.style.paddingTop,
|
||||
paddingBottom: element.style.paddingBottom,
|
||||
borderTopWidth: element.style.borderTopWidth,
|
||||
borderBottomWidth: element.style.borderBottomWidth,
|
||||
marginTop: element.style.marginTop,
|
||||
marginBottom: element.style.marginBottom,
|
||||
}
|
||||
}
|
||||
|
||||
function prepareElement(element: HTMLElement, initialStyle: initialStyle) {
|
||||
const { width } = getComputedStyle(element)
|
||||
element.style.width = width
|
||||
element.style.position = 'absolute'
|
||||
element.style.visibility = 'hidden'
|
||||
element.style.height = ''
|
||||
const { height } = getComputedStyle(element)
|
||||
element.style.width = initialStyle.width
|
||||
element.style.position = initialStyle.position
|
||||
element.style.visibility = initialStyle.visibility
|
||||
element.style.height = closed
|
||||
element.style.overflow = 'hidden'
|
||||
return initialStyle.height && initialStyle.height !== closed
|
||||
? initialStyle.height
|
||||
: height
|
||||
}
|
||||
|
||||
function animateTransition(
|
||||
element: HTMLElement,
|
||||
initialStyle: initialStyle,
|
||||
done: () => void,
|
||||
keyframes: Keyframe[] | PropertyIndexedKeyframes | null,
|
||||
options?: number | KeyframeAnimationOptions,
|
||||
) {
|
||||
const animation = element.animate(keyframes, options)
|
||||
// Set height to 'auto' to restore it after animation
|
||||
element.style.height = initialStyle.height
|
||||
animation.onfinish = () => {
|
||||
element.style.overflow = initialStyle.overflow
|
||||
done()
|
||||
}
|
||||
}
|
||||
|
||||
function getEnterKeyframes(height: string, initialStyle: initialStyle) {
|
||||
return [
|
||||
{
|
||||
height: closed,
|
||||
opacity: props.opacityClosed,
|
||||
paddingTop: closed,
|
||||
paddingBottom: closed,
|
||||
borderTopWidth: closed,
|
||||
borderBottomWidth: closed,
|
||||
marginTop: closed,
|
||||
marginBottom: closed,
|
||||
},
|
||||
{
|
||||
height,
|
||||
opacity: props.opacityOpened,
|
||||
paddingTop: initialStyle.paddingTop,
|
||||
paddingBottom: initialStyle.paddingBottom,
|
||||
borderTopWidth: initialStyle.borderTopWidth,
|
||||
borderBottomWidth: initialStyle.borderBottomWidth,
|
||||
marginTop: initialStyle.marginTop,
|
||||
marginBottom: initialStyle.marginBottom,
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
function enterTransition(element: Element, done: () => void) {
|
||||
const HTMLElement = element as HTMLElement
|
||||
const initialStyle = getElementStyle(HTMLElement)
|
||||
const height = prepareElement(HTMLElement, initialStyle)
|
||||
const keyframes = getEnterKeyframes(height, initialStyle)
|
||||
const options = { duration: props.duration, easing: props.easingEnter }
|
||||
animateTransition(HTMLElement, initialStyle, done, keyframes, options)
|
||||
}
|
||||
|
||||
function leaveTransition(element: Element, done: () => void) {
|
||||
const HTMLElement = element as HTMLElement
|
||||
const initialStyle = getElementStyle(HTMLElement)
|
||||
const { height } = getComputedStyle(HTMLElement)
|
||||
HTMLElement.style.height = height
|
||||
HTMLElement.style.overflow = 'hidden'
|
||||
const keyframes = getEnterKeyframes(height, initialStyle).reverse()
|
||||
const options = { duration: props.duration, easing: props.easingLeave }
|
||||
animateTransition(HTMLElement, initialStyle, done, keyframes, options)
|
||||
}
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<Transition :css="false" @enter="enterTransition" @leave="leaveTransition">
|
||||
<slot />
|
||||
</Transition>
|
||||
</template>
|
||||
@@ -1,175 +0,0 @@
|
||||
// vad-audio-manager.ts
|
||||
|
||||
import type { VAD, VADConfig } from './vad'
|
||||
|
||||
export interface VADAudioOptions {
|
||||
/**
|
||||
* Audio context options
|
||||
*/
|
||||
audioContextOptions?: AudioContextOptions
|
||||
|
||||
/**
|
||||
* The minimum size of audio chunks to process
|
||||
*/
|
||||
minChunkSize?: number
|
||||
|
||||
/**
|
||||
* VAD configuration options
|
||||
*/
|
||||
vadConfig?: Partial<VADConfig>
|
||||
}
|
||||
|
||||
/**
|
||||
* Manages audio input and worklet processing for the VAD module
|
||||
*/
|
||||
export class VADAudioManager {
|
||||
private audioContext: AudioContext | null = null
|
||||
private audioWorkletNode: AudioWorkletNode | null = null
|
||||
private mediaStream: MediaStream | null = null
|
||||
private sourceNode: MediaStreamAudioSourceNode | null = null
|
||||
private vad: VAD
|
||||
// private minChunkSize: number;
|
||||
private workletInitialized: boolean = false
|
||||
|
||||
/**
|
||||
* Create a new VAD audio manager
|
||||
*/
|
||||
constructor(vad: VAD, options: VADAudioOptions = {}) {
|
||||
this.vad = vad
|
||||
// this.minChunkSize = options.minChunkSize || 512;
|
||||
|
||||
// Create audio context with user options or defaults
|
||||
this.audioContext = new AudioContext(options.audioContextOptions || {
|
||||
sampleRate: 16000, // Match the VAD sample rate
|
||||
latencyHint: 'interactive',
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize the audio worklet and connect to microphone
|
||||
*/
|
||||
public async initialize(workletUrl: string): Promise<void> {
|
||||
if (!this.audioContext) {
|
||||
throw new Error('Audio context not created')
|
||||
}
|
||||
|
||||
try {
|
||||
if (!this.workletInitialized) {
|
||||
await this.audioContext.audioWorklet.addModule(workletUrl)
|
||||
URL.revokeObjectURL(workletUrl)
|
||||
this.workletInitialized = true
|
||||
}
|
||||
|
||||
// Create the worklet node
|
||||
this.audioWorkletNode = new AudioWorkletNode(this.audioContext, 'vad-processor')
|
||||
|
||||
// Set up message handling from the worklet
|
||||
this.audioWorkletNode.port.onmessage = async (event) => {
|
||||
const { buffer } = event.data
|
||||
if (buffer && buffer.length > 0) {
|
||||
await this.vad.processAudio(new Float32Array(buffer))
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (error) {
|
||||
console.error('Failed to initialize audio worklet:', error)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Start capturing audio from the microphone
|
||||
*/
|
||||
public async startMicrophone(): Promise<void> {
|
||||
if (!this.audioContext || !this.audioWorkletNode) {
|
||||
throw new Error('Audio system not initialized. Call initialize() first.')
|
||||
}
|
||||
|
||||
try {
|
||||
if (this.audioContext.state === 'suspended') {
|
||||
await this.audioContext.resume()
|
||||
}
|
||||
|
||||
// Request microphone access
|
||||
this.mediaStream = await navigator.mediaDevices.getUserMedia({
|
||||
audio: {
|
||||
echoCancellation: true,
|
||||
noiseSuppression: true,
|
||||
autoGainControl: true,
|
||||
sampleRate: this.audioContext.sampleRate,
|
||||
},
|
||||
})
|
||||
|
||||
// Create source node and connect to worklet
|
||||
this.sourceNode = this.audioContext.createMediaStreamSource(this.mediaStream)
|
||||
this.sourceNode.connect(this.audioWorkletNode)
|
||||
|
||||
// Connect worklet to a silent destination (to keep the audio graph active)
|
||||
// Using a GainNode with gain=0 to ensure no sound is output
|
||||
const silentGain = this.audioContext.createGain()
|
||||
silentGain.gain.value = 0
|
||||
this.audioWorkletNode.connect(silentGain)
|
||||
silentGain.connect(this.audioContext.destination)
|
||||
}
|
||||
catch (error) {
|
||||
console.error('Failed to start microphone:', error)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
public async stopMicrophone(): Promise<void> {
|
||||
if (this.mediaStream) {
|
||||
this.mediaStream.getTracks().forEach(track => track.stop())
|
||||
this.mediaStream = null
|
||||
}
|
||||
|
||||
if (this.sourceNode) {
|
||||
this.sourceNode.disconnect()
|
||||
this.sourceNode = null
|
||||
}
|
||||
|
||||
this.audioContext?.suspend()
|
||||
this.audioWorkletNode?.disconnect()
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop capturing audio
|
||||
*/
|
||||
public stop(): void {
|
||||
// Disconnect nodes
|
||||
if (this.sourceNode && this.audioWorkletNode) {
|
||||
this.sourceNode.disconnect()
|
||||
this.audioWorkletNode.disconnect()
|
||||
}
|
||||
|
||||
// Stop all tracks in the media stream
|
||||
if (this.mediaStream) {
|
||||
this.mediaStream.getTracks().forEach(track => track.stop())
|
||||
this.mediaStream = null
|
||||
}
|
||||
|
||||
// Suspend the audio context rather than closing it
|
||||
// This allows us to reuse it later
|
||||
if (this.audioContext && this.audioContext.state !== 'closed') {
|
||||
this.audioContext.suspend()
|
||||
}
|
||||
|
||||
this.sourceNode = null
|
||||
this.audioWorkletNode = null
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean up all resources
|
||||
*/
|
||||
public dispose(): void {
|
||||
this.stop()
|
||||
|
||||
// Now fully close the audio context
|
||||
if (this.audioContext && this.audioContext.state !== 'closed') {
|
||||
this.audioContext.close()
|
||||
this.audioContext = null
|
||||
}
|
||||
|
||||
this.workletInitialized = false
|
||||
}
|
||||
}
|
||||
@@ -1,53 +0,0 @@
|
||||
// vad-worklet-processor.ts
|
||||
// This file needs to be registered as an AudioWorklet
|
||||
|
||||
/**
|
||||
* Minimum chunk size for processing audio
|
||||
*/
|
||||
const MIN_CHUNK_SIZE = 512
|
||||
|
||||
/**
|
||||
* Global state for audio buffer accumulation
|
||||
*/
|
||||
let globalPointer = 0
|
||||
const globalBuffer = new Float32Array(MIN_CHUNK_SIZE)
|
||||
|
||||
/**
|
||||
* VAD AudioWorklet Processor - processes audio chunks and sends them to the main thread
|
||||
*/
|
||||
class VADProcessor extends AudioWorkletProcessor {
|
||||
process(inputs: Float32Array[][], _outputs: Float32Array[][], _parameters: Record<string, Float32Array>) {
|
||||
const buffer = inputs[0][0]
|
||||
if (!buffer)
|
||||
return true // buffer is null when the stream ends
|
||||
|
||||
if (buffer.length > MIN_CHUNK_SIZE) {
|
||||
// If the buffer is larger than the minimum chunk size, send the entire buffer
|
||||
this.port.postMessage({ buffer })
|
||||
}
|
||||
else {
|
||||
const remaining = MIN_CHUNK_SIZE - globalPointer
|
||||
if (buffer.length >= remaining) {
|
||||
// If the buffer is larger than (or equal to) the remaining space in the global buffer, copy the remaining space
|
||||
globalBuffer.set(buffer.subarray(0, remaining), globalPointer)
|
||||
|
||||
// Send the global buffer
|
||||
this.port.postMessage({ buffer: globalBuffer })
|
||||
|
||||
// Reset the global buffer and set the remaining buffer
|
||||
globalBuffer.fill(0)
|
||||
globalBuffer.set(buffer.subarray(remaining), 0)
|
||||
globalPointer = buffer.length - remaining
|
||||
}
|
||||
else {
|
||||
// If the buffer is smaller than the remaining space in the global buffer, copy the buffer to the global buffer
|
||||
globalBuffer.set(buffer, globalPointer)
|
||||
globalPointer += buffer.length
|
||||
}
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
registerProcessor('vad-processor', VADProcessor)
|
||||
@@ -1,319 +0,0 @@
|
||||
import { AutoModel, Tensor } from '@huggingface/transformers'
|
||||
|
||||
// Default configuration parameters
|
||||
export interface VADConfig {
|
||||
// Sample rate of the audio
|
||||
sampleRate: number
|
||||
// Probabilities above this value are considered speech
|
||||
speechThreshold: number
|
||||
// Threshold to exit speech state
|
||||
exitThreshold: number
|
||||
// Minimum silence duration to consider speech ended (ms)
|
||||
minSilenceDurationMs: number
|
||||
// Padding to add before and after speech (ms)
|
||||
speechPadMs: number
|
||||
// Minimum duration of speech to consider valid (ms)
|
||||
minSpeechDurationMs: number
|
||||
// Maximum buffer duration in seconds
|
||||
maxBufferDuration: number
|
||||
// Size of input buffers from audio source
|
||||
newBufferSize: number
|
||||
}
|
||||
|
||||
export interface VADEvents {
|
||||
// Emitted when speech is detected
|
||||
'speech-start': void
|
||||
// Emitted when speech has ended
|
||||
'speech-end': void
|
||||
// Emitted when a complete speech segment is ready for transcription
|
||||
'speech-ready': { buffer: Float32Array, duration: number }
|
||||
// Emitted for status updates and errors
|
||||
'status': { type: string, message: string }
|
||||
// Debug info
|
||||
'debug': { message: string, data?: any }
|
||||
}
|
||||
|
||||
export type VADEventCallback<K extends keyof VADEvents> =
|
||||
(event: VADEvents[K]) => void
|
||||
|
||||
/**
|
||||
* Voice Activity Detection processor
|
||||
*/
|
||||
export class VAD {
|
||||
private config: VADConfig
|
||||
private model: any
|
||||
private state: Tensor
|
||||
private sampleRateTensor: Tensor
|
||||
private buffer: Float32Array
|
||||
private bufferPointer: number = 0
|
||||
private isRecording: boolean = false
|
||||
private postSpeechSamples: number = 0
|
||||
private prevBuffers: Float32Array[] = []
|
||||
private inferenceChain: Promise<any> = Promise.resolve()
|
||||
private eventListeners: Partial<Record<keyof VADEvents, VADEventCallback<any>[]>> = {}
|
||||
private isReady: boolean = false
|
||||
|
||||
constructor(userConfig: Partial<VADConfig> = {}) {
|
||||
// Default configuration
|
||||
const defaultConfig: VADConfig = {
|
||||
sampleRate: 16000,
|
||||
speechThreshold: 0.3,
|
||||
exitThreshold: 0.1,
|
||||
minSilenceDurationMs: 400,
|
||||
speechPadMs: 80,
|
||||
minSpeechDurationMs: 250,
|
||||
maxBufferDuration: 30,
|
||||
newBufferSize: 512,
|
||||
}
|
||||
|
||||
this.config = { ...defaultConfig, ...userConfig }
|
||||
|
||||
// Create buffer based on max duration
|
||||
this.buffer = new Float32Array(this.config.maxBufferDuration * this.config.sampleRate)
|
||||
|
||||
// Initialize state tensor for VAD model
|
||||
this.state = new Tensor('float32', new Float32Array(2 * 1 * 128), [2, 1, 128])
|
||||
|
||||
// Sample rate tensor for the model
|
||||
this.sampleRateTensor = new Tensor('int64', [this.config.sampleRate], [])
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize the VAD model
|
||||
*/
|
||||
public async initialize(): Promise<void> {
|
||||
try {
|
||||
this.emit('status', { type: 'info', message: 'Loading VAD model...' })
|
||||
|
||||
this.model = await AutoModel.from_pretrained('onnx-community/silero-vad', {
|
||||
config: { model_type: 'custom' } as any,
|
||||
dtype: 'fp32', // Full-precision
|
||||
})
|
||||
|
||||
this.isReady = true
|
||||
this.emit('status', { type: 'info', message: 'VAD model loaded successfully' })
|
||||
}
|
||||
catch (error) {
|
||||
this.emit('status', { type: 'error', message: `Failed to load VAD model: ${error}` })
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Add event listener
|
||||
*/
|
||||
public on<K extends keyof VADEvents>(event: K, callback: VADEventCallback<K>): void {
|
||||
if (!this.eventListeners[event]) {
|
||||
this.eventListeners[event] = []
|
||||
}
|
||||
this.eventListeners[event]!.push(callback as any)
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove event listener
|
||||
*/
|
||||
public off<K extends keyof VADEvents>(event: K, callback: VADEventCallback<K>): void {
|
||||
if (!this.eventListeners[event])
|
||||
return
|
||||
this.eventListeners[event] = this.eventListeners[event]!.filter(cb => cb !== callback)
|
||||
}
|
||||
|
||||
/**
|
||||
* Emit event
|
||||
*/
|
||||
private emit<K extends keyof VADEvents>(event: K, data: VADEvents[K]): void {
|
||||
if (!this.eventListeners[event])
|
||||
return
|
||||
for (const callback of this.eventListeners[event]!) {
|
||||
callback(data)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Process audio buffer for speech detection
|
||||
*/
|
||||
public async processAudio(inputBuffer: Float32Array): Promise<void> {
|
||||
if (!this.isReady) {
|
||||
throw new Error('VAD model is not initialized. Call initialize() first.')
|
||||
}
|
||||
|
||||
const wasRecording = this.isRecording
|
||||
|
||||
// Perform VAD on the input buffer
|
||||
const isSpeech = await this.detectSpeech(inputBuffer)
|
||||
|
||||
// Calculate derived constants
|
||||
const sampleRateMs = this.config.sampleRate / 1000
|
||||
const minSilenceDurationSamples = this.config.minSilenceDurationMs * sampleRateMs
|
||||
const speechPadSamples = this.config.speechPadMs * sampleRateMs
|
||||
const minSpeechDurationSamples = this.config.minSpeechDurationMs * sampleRateMs
|
||||
const maxPrevBuffers = Math.ceil(speechPadSamples / this.config.newBufferSize)
|
||||
|
||||
// If not currently in speech and the current buffer isn't speech,
|
||||
// store it in the previous buffers queue for potential padding
|
||||
if (!wasRecording && !isSpeech) {
|
||||
if (this.prevBuffers.length >= maxPrevBuffers) {
|
||||
this.prevBuffers.shift()
|
||||
}
|
||||
this.prevBuffers.push(inputBuffer.slice(0))
|
||||
return
|
||||
}
|
||||
|
||||
// Check if we need to handle buffer overflow
|
||||
const remaining = this.buffer.length - this.bufferPointer
|
||||
if (inputBuffer.length >= remaining) {
|
||||
// The buffer is full, process what we have
|
||||
this.buffer.set(inputBuffer.subarray(0, remaining), this.bufferPointer)
|
||||
this.bufferPointer += remaining
|
||||
|
||||
// Process and reset with overflow
|
||||
const overflow = inputBuffer.subarray(remaining)
|
||||
this.processSpeechSegment(overflow)
|
||||
return
|
||||
}
|
||||
else {
|
||||
// Add input to the buffer
|
||||
this.buffer.set(inputBuffer, this.bufferPointer)
|
||||
this.bufferPointer += inputBuffer.length
|
||||
}
|
||||
|
||||
// Handle speech detection
|
||||
if (isSpeech) {
|
||||
if (!this.isRecording) {
|
||||
// Speech just started
|
||||
this.emit('speech-start', undefined)
|
||||
this.emit('status', { type: 'info', message: 'Speech detected' })
|
||||
}
|
||||
|
||||
// Update state
|
||||
this.isRecording = true
|
||||
this.postSpeechSamples = 0
|
||||
return
|
||||
}
|
||||
|
||||
// At this point, we were recording but the current buffer is not speech
|
||||
this.postSpeechSamples += inputBuffer.length
|
||||
|
||||
// Check if silence is long enough to consider speech ended
|
||||
if (this.postSpeechSamples >= minSilenceDurationSamples) {
|
||||
// Check if the speech segment is long enough to process
|
||||
if (this.bufferPointer < minSpeechDurationSamples) {
|
||||
// Too short, reset without processing
|
||||
this.reset()
|
||||
return
|
||||
}
|
||||
|
||||
// Process the speech segment
|
||||
this.processSpeechSegment()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect speech in an audio buffer
|
||||
*/
|
||||
private async detectSpeech(buffer: Float32Array): Promise<boolean> {
|
||||
const input = new Tensor('float32', buffer, [1, buffer.length])
|
||||
|
||||
const { stateN, output } = await (this.inferenceChain = this.inferenceChain.then(() =>
|
||||
this.model({
|
||||
input,
|
||||
sr: this.sampleRateTensor,
|
||||
state: this.state,
|
||||
}),
|
||||
))
|
||||
|
||||
// Update the state
|
||||
this.state = stateN
|
||||
|
||||
// Get the speech probability
|
||||
const speechProb = output.data[0]
|
||||
|
||||
this.emit('debug', {
|
||||
message: 'VAD score',
|
||||
data: { probability: speechProb },
|
||||
})
|
||||
|
||||
// Apply thresholds
|
||||
return (
|
||||
speechProb > this.config.speechThreshold
|
||||
|| (this.isRecording && speechProb >= this.config.exitThreshold)
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Process a complete speech segment
|
||||
*/
|
||||
private processSpeechSegment(overflow?: Float32Array): void {
|
||||
const sampleRateMs = this.config.sampleRate / 1000
|
||||
const speechPadSamples = this.config.speechPadMs * sampleRateMs
|
||||
|
||||
// Calculate duration info
|
||||
const duration = (this.bufferPointer / this.config.sampleRate) * 1000
|
||||
const overflowLength = overflow?.length ?? 0
|
||||
|
||||
// Create the final buffer with padding
|
||||
const prevLength = this.prevBuffers.reduce((acc, b) => acc + b.length, 0)
|
||||
const finalBuffer = new Float32Array(prevLength + this.bufferPointer + speechPadSamples)
|
||||
|
||||
// Add previous buffers for pre-speech padding
|
||||
let offset = 0
|
||||
for (const prev of this.prevBuffers) {
|
||||
finalBuffer.set(prev, offset)
|
||||
offset += prev.length
|
||||
}
|
||||
|
||||
// Add the main speech segment
|
||||
finalBuffer.set(this.buffer.slice(0, this.bufferPointer + speechPadSamples), offset)
|
||||
|
||||
// Emit the speech segment
|
||||
this.emit('speech-end', undefined)
|
||||
this.emit('speech-ready', {
|
||||
buffer: finalBuffer,
|
||||
duration,
|
||||
})
|
||||
|
||||
// Reset for the next segment
|
||||
if (overflow) {
|
||||
this.buffer.set(overflow, 0)
|
||||
}
|
||||
this.reset(overflowLength)
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset the VAD state
|
||||
*/
|
||||
private reset(offset: number = 0): void {
|
||||
this.buffer.fill(0, offset)
|
||||
this.bufferPointer = offset
|
||||
this.isRecording = false
|
||||
this.postSpeechSamples = 0
|
||||
this.prevBuffers = []
|
||||
}
|
||||
|
||||
/**
|
||||
* Update configuration
|
||||
*/
|
||||
public updateConfig(newConfig: Partial<VADConfig>): void {
|
||||
this.config = { ...this.config, ...newConfig }
|
||||
|
||||
// If buffer size changed, create a new buffer
|
||||
if (newConfig.maxBufferDuration || newConfig.sampleRate) {
|
||||
this.buffer = new Float32Array(this.config.maxBufferDuration * this.config.sampleRate)
|
||||
this.bufferPointer = 0
|
||||
}
|
||||
|
||||
// Update sample rate tensor if needed
|
||||
if (newConfig.sampleRate) {
|
||||
this.sampleRateTensor = new Tensor('int64', [this.config.sampleRate], [])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a VAD processor with the given configuration
|
||||
*/
|
||||
export async function createVAD(config?: Partial<VADConfig>): Promise<VAD> {
|
||||
const vad = new VAD(config)
|
||||
await vad.initialize()
|
||||
return vad
|
||||
}
|
||||
@@ -1,43 +0,0 @@
|
||||
function writeString(dataView: DataView, offset: number, string: string) {
|
||||
for (let i = 0; i < string.length; i++) {
|
||||
dataView.setUint8(offset + i, string.charCodeAt(i))
|
||||
}
|
||||
}
|
||||
|
||||
export function toWav(buffer: Float32Array, sampleRate: number) {
|
||||
const numChannels = 1
|
||||
const numSamples = buffer.length
|
||||
|
||||
// Create the WAV file container
|
||||
const arrayBuffer = new ArrayBuffer(44 + numSamples * 2)
|
||||
const dataView = new DataView(arrayBuffer)
|
||||
|
||||
// RIFF chunk descriptor
|
||||
writeString(dataView, 0, 'RIFF')
|
||||
dataView.setUint32(4, 36 + numSamples * 2, true)
|
||||
writeString(dataView, 8, 'WAVE')
|
||||
|
||||
// fmt sub-chunk
|
||||
writeString(dataView, 12, 'fmt ')
|
||||
dataView.setUint32(16, 16, true)
|
||||
dataView.setUint16(20, 1, true) // PCM format
|
||||
dataView.setUint16(22, numChannels, true)
|
||||
dataView.setUint32(24, sampleRate, true)
|
||||
dataView.setUint32(28, sampleRate * numChannels * 2, true) // byte rate
|
||||
dataView.setUint16(32, numChannels * 2, true) // block align
|
||||
dataView.setUint16(34, 16, true) // bits per sample
|
||||
|
||||
// data sub-chunk
|
||||
writeString(dataView, 36, 'data')
|
||||
dataView.setUint32(40, numSamples * 2, true)
|
||||
|
||||
// Write the PCM samples
|
||||
const offset = 44
|
||||
for (let i = 0; i < numSamples; i++) {
|
||||
const sample = Math.max(-1, Math.min(1, buffer[i]))
|
||||
const value = sample < 0 ? sample * 0x8000 : sample * 0x7FFF
|
||||
dataView.setInt16(offset + i * 2, value, true)
|
||||
}
|
||||
|
||||
return arrayBuffer
|
||||
}
|
||||
@@ -1,13 +0,0 @@
|
||||
import { createApp } from 'vue'
|
||||
import { createRouter, createWebHashHistory } from 'vue-router'
|
||||
import { routes } from 'vue-router/auto-routes'
|
||||
|
||||
import App from './App.vue'
|
||||
import '@unocss/reset/tailwind.css'
|
||||
import 'uno.css'
|
||||
|
||||
const router = createRouter({ routes, history: createWebHashHistory() })
|
||||
|
||||
createApp(App)
|
||||
.use(router)
|
||||
.mount('#app')
|
||||
@@ -1,324 +0,0 @@
|
||||
<script setup lang="ts">
|
||||
import type { Message } from '@xsai/shared-chat'
|
||||
|
||||
import { createWorkflow, getContext, workflowEvent } from '@llama-flow/core'
|
||||
import { useLocalStorage } from '@vueuse/core'
|
||||
import { generateTranscription } from '@xsai/generate-transcription'
|
||||
import { streamText } from '@xsai/stream-text'
|
||||
import { ref, toRaw } from 'vue'
|
||||
|
||||
import FieldInput from '../components/FieldInput.vue'
|
||||
import Section from '../components/Section.vue'
|
||||
import { VADAudioManager } from '../libs/vad/manager'
|
||||
import workletUrl from '../libs/vad/process.worklet?worker&url'
|
||||
import { createVAD } from '../libs/vad/vad'
|
||||
import { toWav } from '../libs/vad/wav'
|
||||
|
||||
interface AudioSegment {
|
||||
buffer: Float32Array
|
||||
duration: number
|
||||
timestamp: number
|
||||
audioSrc: string
|
||||
transcription: string
|
||||
}
|
||||
|
||||
const llmInputSpeechEvent = workflowEvent<{ buffer: Float32Array<ArrayBufferLike>, duration: number }, 'input-speech'>()
|
||||
const llmTranscriptionEvent = workflowEvent<string, 'transcription'>()
|
||||
const llmChatCompletionsTokenEvent = workflowEvent<string, 'chat-completions-token'>()
|
||||
const llmChatCompletionsEndedEvent = workflowEvent<void, 'chat-completions-ended'>()
|
||||
|
||||
const llmProviderBaseURL = useLocalStorage('llmProviderBaseURL', 'https://openrouter.ai/api/v1/')
|
||||
const llmProviderAPIKey = useLocalStorage('llmProviderAPIKey', '')
|
||||
const llmProviderModel = useLocalStorage('llmProviderModel', 'gpt-4o-mini')
|
||||
|
||||
const asrProviderBaseURL = useLocalStorage('asrProviderBaseURL', 'http://localhost:8000/v1/')
|
||||
const asrProviderAPIKey = useLocalStorage('asrProviderAPIKey', '')
|
||||
const asrProviderModel = useLocalStorage('asrProviderModel', 'large-v3-turbo')
|
||||
|
||||
const audioManager = ref<VADAudioManager>()
|
||||
const isInitialized = ref(false)
|
||||
const isRunning = ref(false)
|
||||
const isSpeechDetected = ref(false)
|
||||
const segments = ref<AudioSegment[]>([])
|
||||
const error = ref<string | null>(null)
|
||||
const isModuleLoading = ref(false)
|
||||
|
||||
const sending = ref(false)
|
||||
const messages = ref<Message[]>([
|
||||
{
|
||||
role: 'system',
|
||||
content: 'You are having a phone call with a user, the texts are all transcribed from the audio, it may not be accurate, if you cannot understand what user said, please ask them to repeat it.',
|
||||
},
|
||||
])
|
||||
const streamingMessage = ref<Message>({ role: 'assistant', content: '' })
|
||||
|
||||
const llmWorkflow = createWorkflow()
|
||||
const llmWorkflowContext = llmWorkflow.createContext()
|
||||
|
||||
async function* asyncIteratorFromReadableStream<T, F = Uint8Array>(res: ReadableStream<F>, func: (value: F) => Promise<T>): AsyncGenerator<T, void, unknown> {
|
||||
// react js - TS2504: Type 'ReadableStream<Uint8Array>' must have a '[Symbol.asyncIterator]()' method that returns an async iterator - Stack Overflow
|
||||
// https://stackoverflow.com/questions/76700924/ts2504-type-readablestreamuint8array-must-have-a-symbol-asynciterator
|
||||
const reader = res.getReader()
|
||||
try {
|
||||
while (true) {
|
||||
const { done, value } = await reader.read()
|
||||
if (done) {
|
||||
return
|
||||
}
|
||||
|
||||
yield func(value)
|
||||
}
|
||||
}
|
||||
finally {
|
||||
reader.releaseLock()
|
||||
}
|
||||
}
|
||||
|
||||
llmWorkflow.handle([llmInputSpeechEvent], async (event) => {
|
||||
const context = getContext()
|
||||
|
||||
const wavBuffer = toWav(event.data.buffer, 16000)
|
||||
const audioBlob = new Blob([wavBuffer], { type: 'audio/wav' })
|
||||
|
||||
const obj = {
|
||||
buffer: event.data.buffer,
|
||||
duration: event.data.duration / 1000, // Convert to seconds for display
|
||||
timestamp: Date.now(),
|
||||
audioSrc: URL.createObjectURL(audioBlob),
|
||||
transcription: '',
|
||||
}
|
||||
|
||||
// Store the segment
|
||||
const objIndex = segments.value.push(obj)
|
||||
|
||||
generateTranscription({
|
||||
baseURL: asrProviderBaseURL.value,
|
||||
file: audioBlob,
|
||||
model: asrProviderModel.value,
|
||||
apiKey: asrProviderAPIKey.value,
|
||||
}).then((res) => {
|
||||
segments.value[objIndex - 1].transcription = res.text
|
||||
context.sendEvent(llmTranscriptionEvent.with(res.text))
|
||||
}).catch((err) => {
|
||||
console.error('Failed to generate transcription:', err)
|
||||
})
|
||||
})
|
||||
|
||||
llmWorkflow.handle([llmTranscriptionEvent], async (event) => {
|
||||
const context = getContext()
|
||||
|
||||
try {
|
||||
sending.value = true
|
||||
|
||||
if (!event.data)
|
||||
return
|
||||
|
||||
streamingMessage.value = { role: 'assistant', content: '' }
|
||||
messages.value.push({ role: 'user', content: event.data })
|
||||
messages.value.push(streamingMessage.value)
|
||||
const newMessages = messages.value.slice(0, messages.value.length - 1).map(msg => toRaw(msg))
|
||||
|
||||
const res = await streamText({
|
||||
baseURL: llmProviderBaseURL.value,
|
||||
apiKey: llmProviderAPIKey.value,
|
||||
model: llmProviderModel.value,
|
||||
messages: newMessages as Message[],
|
||||
})
|
||||
|
||||
for await (const textPart of asyncIteratorFromReadableStream(res.textStream, async v => v)) {
|
||||
context.sendEvent(llmChatCompletionsTokenEvent.with(textPart))
|
||||
}
|
||||
|
||||
context.sendEvent(llmChatCompletionsEndedEvent.with())
|
||||
}
|
||||
catch (error) {
|
||||
console.error('Error sending message:', error)
|
||||
throw error
|
||||
}
|
||||
finally {
|
||||
sending.value = false
|
||||
}
|
||||
})
|
||||
|
||||
llmWorkflow.handle([llmChatCompletionsTokenEvent], async (event) => {
|
||||
if (!streamingMessage.value.content) {
|
||||
streamingMessage.value.content = ''
|
||||
}
|
||||
|
||||
streamingMessage.value.content += event.data as any
|
||||
})
|
||||
|
||||
llmWorkflow.handle([llmChatCompletionsEndedEvent], async () => {
|
||||
// eslint-disable-next-line no-console
|
||||
console.log('llmChatCompletionsEndedEvent')
|
||||
})
|
||||
|
||||
async function setupSpeechDetection() {
|
||||
messages.value = [
|
||||
messages.value[0],
|
||||
]
|
||||
|
||||
try {
|
||||
isModuleLoading.value = true
|
||||
|
||||
// Create and initialize the VAD
|
||||
const vad = await createVAD({
|
||||
sampleRate: 16000,
|
||||
speechThreshold: 0.3,
|
||||
exitThreshold: 0.1,
|
||||
minSilenceDurationMs: 400,
|
||||
})
|
||||
|
||||
// Set up event handlers
|
||||
vad.on('speech-start', () => {
|
||||
isSpeechDetected.value = true
|
||||
})
|
||||
|
||||
vad.on('speech-end', () => {
|
||||
isSpeechDetected.value = false
|
||||
})
|
||||
|
||||
vad.on('speech-ready', async ({ buffer, duration }) => {
|
||||
llmWorkflowContext.sendEvent(llmInputSpeechEvent.with({ buffer, duration }))
|
||||
})
|
||||
|
||||
vad.on('status', ({ type, message }) => {
|
||||
if (type === 'error') {
|
||||
error.value = message
|
||||
}
|
||||
})
|
||||
|
||||
// Create and initialize audio manager
|
||||
const m = new VADAudioManager(vad, {
|
||||
minChunkSize: 512,
|
||||
audioContextOptions: {
|
||||
sampleRate: 16000,
|
||||
latencyHint: 'interactive',
|
||||
},
|
||||
})
|
||||
|
||||
await m.initialize(workletUrl)
|
||||
audioManager.value = m
|
||||
isInitialized.value = true
|
||||
isModuleLoading.value = false
|
||||
startVad()
|
||||
}
|
||||
catch (err) {
|
||||
console.error('Setup failed:', err)
|
||||
error.value = err instanceof Error ? err.message : String(err)
|
||||
isModuleLoading.value = false
|
||||
}
|
||||
}
|
||||
|
||||
async function destroySpeechDetection() {
|
||||
await audioManager.value?.stop()
|
||||
await audioManager.value?.dispose()
|
||||
isInitialized.value = false
|
||||
isRunning.value = false
|
||||
isSpeechDetected.value = false
|
||||
|
||||
for (const segment of segments.value) {
|
||||
URL.revokeObjectURL(segment.audioSrc)
|
||||
}
|
||||
|
||||
segments.value = []
|
||||
error.value = null
|
||||
isModuleLoading.value = false
|
||||
}
|
||||
|
||||
async function startVad() {
|
||||
try {
|
||||
await audioManager.value?.startMicrophone()
|
||||
isRunning.value = true
|
||||
error.value = null
|
||||
}
|
||||
catch (err) {
|
||||
console.error('Failed to start microphone:', err)
|
||||
error.value = err instanceof Error ? err.message : String(err)
|
||||
}
|
||||
}
|
||||
|
||||
async function stopVad() {
|
||||
await audioManager.value?.stopMicrophone()
|
||||
isRunning.value = false
|
||||
isSpeechDetected.value = false
|
||||
}
|
||||
|
||||
function toggleListening() {
|
||||
if (isRunning.value) {
|
||||
stopVad()
|
||||
}
|
||||
else {
|
||||
startVad()
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<div mb-6 mt-4 h-full w-full flex flex-col gap-2>
|
||||
<div w-full flex flex-1 flex-col gap-2>
|
||||
<Section title="Settings" icon="i-solar:settings-bold" :expand="!isInitialized">
|
||||
<div flex="~ col gap-4">
|
||||
<FieldInput v-model="llmProviderBaseURL" label="LLM Provider Base URL" description="The base URL of the LLM provider. Generally, Speaches is recommended." />
|
||||
<FieldInput v-model="llmProviderAPIKey" label="LLM Provider API Key" description="The API key of the LLM provider" type="password" />
|
||||
<FieldInput v-model="llmProviderModel" label="LLM Provider Model" description="The model to use for the LLM provider" />
|
||||
<FieldInput v-model="asrProviderBaseURL" label="ASR Provider Base URL" description="The base URL of the ASR provider. Generally, Speaches is recommended." />
|
||||
<FieldInput v-model="asrProviderAPIKey" label="ASR Provider API Key" description="The API key of the ASR provider" type="password" />
|
||||
<FieldInput v-model="asrProviderModel" label="ASR Provider Model" description="The model to use for the ASR provider" />
|
||||
</div>
|
||||
</Section>
|
||||
|
||||
<Section title="Voice Segments" icon="i-solar:microphone-3-bold" :expand="false">
|
||||
<ul v-if="segments?.length && segments.length > 0">
|
||||
<li v-for="(segment, index) in segments" :key="index" class="segment" flex flex-col gap-2>
|
||||
<div class="segment-info" grid="~ cols-[120px_1fr] gap-2">
|
||||
<span text="neutral-400 dark:neutral-500">Duration</span>
|
||||
<span font-mono>
|
||||
{{ segment.duration.toFixed(2) }}s
|
||||
</span>
|
||||
<span text="neutral-400 dark:neutral-500">Transcription</span>
|
||||
<span>
|
||||
{{ segment.transcription }}
|
||||
</span>
|
||||
</div>
|
||||
<audio :src="segment.audioSrc" controls w-full />
|
||||
</li>
|
||||
</ul>
|
||||
</Section>
|
||||
|
||||
<div v-if="isModuleLoading" mt-20 flex items-center justify-center text-5xl>
|
||||
<div i-svg-spinners:3-dots-move />
|
||||
</div>
|
||||
|
||||
<div v-if="error" class="error">
|
||||
{{ error }}
|
||||
</div>
|
||||
|
||||
<div my-4 max-h-120 w-full flex flex-col gap-2 overflow-y-scroll>
|
||||
<template v-for="(message, index) in messages" :key="index">
|
||||
<div v-if="message.role === 'user'" class="w-fit rounded-lg bg-cyan-100 px-3 py-2 dark:bg-cyan-900">
|
||||
{{ message.content }}
|
||||
</div>
|
||||
<div v-else-if="message.role === 'assistant'" class="w-fit rounded-lg bg-neutral-100 px-3 py-2 dark:bg-neutral-800">
|
||||
{{ message.content }}
|
||||
</div>
|
||||
</template>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div w-full flex justify-center gap-4>
|
||||
<button aspect-square size-15 flex items-center justify-center rounded-full text-2xl :class="[isRunning ? 'bg-neutral-700 dark:bg-white text-white dark:text-neutral-900' : 'bg-neutral-900 dark:bg-white/20 text-white dark:text-white', isInitialized ? 'opacity-100' : 'opacity-0']" :disabled="!isInitialized" @click="toggleListening">
|
||||
<div i-solar:microphone-3-bold />
|
||||
</button>
|
||||
<button v-if="!isInitialized" bg="green-500 dark:green-500 hover:green-400 dark:hover:green-400 active:green-500 dark:active:green-500" transition="all duration-250 ease-in-out" aspect-square size-15 flex items-center justify-center rounded-full @click="setupSpeechDetection">
|
||||
<div i-solar:phone-rounded-bold text-4xl text-white />
|
||||
</button>
|
||||
<button v-else bg="red-500 dark:red-500 hover:red-400 dark:hover:red-400 active:red-500 dark:active:red-500" transition="all duration-250 ease-in-out" aspect-square size-15 flex items-center justify-center rounded-full text-4xl text-white @click="destroySpeechDetection">
|
||||
<div i-solar:end-call-rounded-bold />
|
||||
</button>
|
||||
<button aspect-square size-15 flex items-center justify-center rounded-full text-2xl class="bg-neutral-900 text-white dark:bg-neutral-900 dark:bg-white/20 dark:text-white" :class="isInitialized ? 'opacity-100' : 'opacity-0'" :disabled="!isInitialized">
|
||||
<div i-solar:headphones-round-bold />
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
@@ -1,41 +0,0 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ESNext",
|
||||
"jsx": "preserve",
|
||||
"lib": [
|
||||
"DOM",
|
||||
"ESNext",
|
||||
"WebWorker"
|
||||
],
|
||||
"module": "ESNext",
|
||||
"moduleResolution": "Bundler",
|
||||
"resolveJsonModule": true,
|
||||
"types": [
|
||||
"vitest",
|
||||
"vite/client",
|
||||
// Currently AudioWorkletProcessor type is missing, we need to add it manually through @types/audioworklet
|
||||
// https://github.com/microsoft/TypeScript/issues/28308#issuecomment-1512509870
|
||||
"@types/audioworklet",
|
||||
"unplugin-vue-router/client"
|
||||
],
|
||||
"allowJs": true,
|
||||
"strict": true,
|
||||
"strictNullChecks": true,
|
||||
"noUnusedLocals": true,
|
||||
"noEmit": true,
|
||||
"esModuleInterop": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"isolatedModules": true,
|
||||
"skipLibCheck": true
|
||||
},
|
||||
"include": [
|
||||
"src/**/*.ts",
|
||||
"src/**/*.tsx",
|
||||
"src/**/*.vue",
|
||||
"src/**/*.js"
|
||||
],
|
||||
"exclude": [
|
||||
"dist",
|
||||
"node_modules"
|
||||
]
|
||||
}
|
||||
@@ -1,37 +0,0 @@
|
||||
import {
|
||||
defineConfig,
|
||||
presetAttributify,
|
||||
presetIcons,
|
||||
presetTypography,
|
||||
presetWebFonts,
|
||||
presetWind3,
|
||||
transformerDirectives,
|
||||
transformerVariantGroup,
|
||||
} from 'unocss'
|
||||
|
||||
export default defineConfig({
|
||||
presets: [
|
||||
presetWind3(),
|
||||
presetAttributify(),
|
||||
presetTypography(),
|
||||
presetWebFonts({
|
||||
fonts: {
|
||||
sans: 'DM Sans',
|
||||
serif: 'DM Serif Display',
|
||||
mono: 'DM Mono',
|
||||
},
|
||||
timeouts: {
|
||||
warning: 5000,
|
||||
failure: 10000,
|
||||
},
|
||||
}),
|
||||
presetIcons({
|
||||
scale: 1.2,
|
||||
}),
|
||||
],
|
||||
transformers: [
|
||||
transformerDirectives(),
|
||||
transformerVariantGroup(),
|
||||
],
|
||||
safelist: 'prose prose-sm m-auto text-left'.split(' '),
|
||||
})
|
||||
@@ -1,19 +0,0 @@
|
||||
import { resolve } from 'node:path'
|
||||
import Vue from '@vitejs/plugin-vue'
|
||||
import Unocss from 'unocss/vite'
|
||||
import VueRouter from 'unplugin-vue-router/vite'
|
||||
import { defineConfig } from 'vite'
|
||||
|
||||
export default defineConfig({
|
||||
plugins: [
|
||||
// https://github.com/posva/unplugin-vue-router
|
||||
VueRouter({
|
||||
extensions: ['.vue', '.md'],
|
||||
dts: resolve(import.meta.dirname, 'src', 'typed-router.d.ts'),
|
||||
}),
|
||||
Vue(),
|
||||
// https://github.com/antfu/unocss
|
||||
// see uno.config.ts for config
|
||||
Unocss(),
|
||||
],
|
||||
})
|
||||
@@ -1,22 +0,0 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<title>Project AIRI VAD + ASR Playground</title>
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=0" />
|
||||
<link rel="icon" type="image/svg+xml" href="/favicon.svg" />
|
||||
<script>
|
||||
;(function () {
|
||||
const prefersDark = window.matchMedia && window.matchMedia('(prefers-color-scheme: dark)').matches
|
||||
const setting = localStorage.getItem('vueuse-color-scheme') || 'auto'
|
||||
if (setting === 'dark' || (prefersDark && setting !== 'light'))
|
||||
document.documentElement.classList.toggle('dark', true)
|
||||
})()
|
||||
</script>
|
||||
</head>
|
||||
<body class="font-sans">
|
||||
<div id="app"></div>
|
||||
<script type="module" src="/src/main.ts"></script>
|
||||
<noscript> This website requires JavaScript to function properly. Please enable JavaScript to continue. </noscript>
|
||||
</body>
|
||||
</html>
|
||||
@@ -1 +0,0 @@
|
||||
title: VAD
|
||||
@@ -1 +0,0 @@
|
||||
title: VAD
|
||||
@@ -1,13 +0,0 @@
|
||||
[build]
|
||||
base = "/"
|
||||
command = "pnpm -F @proj-airi/vad-asr... run build"
|
||||
publish = "/apps/vad-asr/dist"
|
||||
|
||||
[build.environment]
|
||||
NODE_VERSION = "23"
|
||||
|
||||
[[redirects]]
|
||||
from = "/*"
|
||||
to = "/index.html"
|
||||
status = 200
|
||||
force = false
|
||||
@@ -1,42 +0,0 @@
|
||||
{
|
||||
"name": "@proj-airi/vad-asr",
|
||||
"type": "module",
|
||||
"private": true,
|
||||
"description": "Voice Activity Detector & Automatic Speech Recognition",
|
||||
"author": {
|
||||
"name": "Neko Ayaka",
|
||||
"email": "neko@ayaka.moe",
|
||||
"url": "https://github.com/nekomeowww"
|
||||
},
|
||||
"license": "MIT",
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/moeru-ai/airi.git",
|
||||
"directory": "apps/vad-asr"
|
||||
},
|
||||
"scripts": {
|
||||
"dev": "vite",
|
||||
"build": "vite build",
|
||||
"preview": "vite preview",
|
||||
"typecheck": "vue-tsc --noEmit"
|
||||
},
|
||||
"dependencies": {
|
||||
"@huggingface/transformers": "^3.4.2",
|
||||
"@vueuse/core": "^13.0.0",
|
||||
"@xsai/generate-transcription": "catalog:",
|
||||
"defu": "^6.1.4",
|
||||
"es-toolkit": "^1.34.1",
|
||||
"vue": "^3.5.13"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@iconify-json/solar": "^1.2.2",
|
||||
"@iconify-json/svg-spinners": "^1.2.2",
|
||||
"@types/audioworklet": "^0.0.72",
|
||||
"@unocss/reset": "^66.1.0-beta.10",
|
||||
"@vitejs/plugin-vue": "^5.2.3",
|
||||
"unplugin-vue-router": "^0.12.0",
|
||||
"vite": "^6.2.5",
|
||||
"vue-router": "^4.5.0",
|
||||
"vue-tsc": "^3.0.0-alpha.2"
|
||||
}
|
||||
}
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 15 KiB |
File diff suppressed because one or more lines are too long
|
Before Width: | Height: | Size: 23 KiB |
@@ -1,56 +0,0 @@
|
||||
<script setup lang="ts">
|
||||
import { useDark, useToggle } from '@vueuse/core'
|
||||
import { RouterLink, RouterView } from 'vue-router'
|
||||
|
||||
const isDark = useDark()
|
||||
const toggleDark = useToggle(isDark)
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<div mx-auto h-full max-w-screen-lg flex flex-col gap-2 p-4>
|
||||
<header flex flex-row items-center justify-between>
|
||||
<h1 text-2xl>
|
||||
VAD + ASR Playground
|
||||
</h1>
|
||||
<div flex flex-row items-center gap-2>
|
||||
<button text-lg @click="() => toggleDark()">
|
||||
<div v-if="isDark" i-solar:moon-stars-bold-duotone />
|
||||
<div v-else i-solar:sun-bold />
|
||||
</button>
|
||||
<a href="https://github.com/moeru-ai/airi/tree/main/apps/vad-asr">
|
||||
<div i-simple-icons:github />
|
||||
</a>
|
||||
</div>
|
||||
</header>
|
||||
<nav bg="neutral-100 dark:neutral-800" w-fit flex items-center of-hidden rounded-lg>
|
||||
<RouterLink
|
||||
to="/" px-3 py-2 bg="hover:neutral-200 dark:hover:neutral-700"
|
||||
transition="all duration-250 ease-in-out"
|
||||
>
|
||||
<h1>WASM</h1>
|
||||
</RouterLink>
|
||||
</nav>
|
||||
<RouterView />
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<style>
|
||||
html,
|
||||
body,
|
||||
#app {
|
||||
height: 100%;
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
overscroll-behavior: none;
|
||||
}
|
||||
|
||||
html {
|
||||
background: #fff;
|
||||
transition: all 0.3s ease-in-out;
|
||||
}
|
||||
|
||||
html.dark {
|
||||
background: #121212;
|
||||
color-scheme: dark;
|
||||
}
|
||||
</style>
|
||||
@@ -1,36 +0,0 @@
|
||||
<script setup lang="ts">
|
||||
import Input from './Input.vue'
|
||||
|
||||
const props = defineProps<{
|
||||
label?: string
|
||||
description?: string
|
||||
placeholder?: string
|
||||
required?: boolean
|
||||
type?: string
|
||||
inputClass?: string
|
||||
}>()
|
||||
|
||||
const modelValue = defineModel<string>({ required: true })
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<div max-w-full>
|
||||
<label flex="~ col gap-2">
|
||||
<div>
|
||||
<div class="flex items-center gap-1 text-sm font-medium">
|
||||
{{ props.label }}
|
||||
<span v-if="props.required !== false" class="text-red-500">*</span>
|
||||
</div>
|
||||
<div class="text-xs text-neutral-500 dark:text-neutral-400" text-nowrap>
|
||||
{{ props.description }}
|
||||
</div>
|
||||
</div>
|
||||
<Input
|
||||
v-model="modelValue"
|
||||
:type="props.type"
|
||||
:placeholder="props.placeholder"
|
||||
:class="props.inputClass"
|
||||
/>
|
||||
</label>
|
||||
</div>
|
||||
</template>
|
||||
@@ -1,21 +0,0 @@
|
||||
<script setup lang="ts">
|
||||
const props = defineProps<{
|
||||
type?: string
|
||||
}>()
|
||||
|
||||
const modelValue = defineModel<string>({ required: true })
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<input
|
||||
v-model="modelValue"
|
||||
:type="props.type || 'text'"
|
||||
border="focus:blue-300 dark:focus:blue-400/50 2 solid neutral-100 dark:neutral-900"
|
||||
transition="all duration-200 ease-in-out"
|
||||
text="disabled:neutral-400 dark:disabled:neutral-600"
|
||||
cursor="disabled:not-allowed"
|
||||
w-full rounded-lg px-2 py-1 text-nowrap text-sm outline-none
|
||||
shadow="sm"
|
||||
bg="neutral-50 dark:neutral-950 focus:neutral-50 dark:focus:neutral-900"
|
||||
>
|
||||
</template>
|
||||
@@ -1,175 +0,0 @@
|
||||
// vad-audio-manager.ts
|
||||
|
||||
import type { VAD, VADConfig } from './vad'
|
||||
|
||||
export interface VADAudioOptions {
|
||||
/**
|
||||
* Audio context options
|
||||
*/
|
||||
audioContextOptions?: AudioContextOptions
|
||||
|
||||
/**
|
||||
* The minimum size of audio chunks to process
|
||||
*/
|
||||
minChunkSize?: number
|
||||
|
||||
/**
|
||||
* VAD configuration options
|
||||
*/
|
||||
vadConfig?: Partial<VADConfig>
|
||||
}
|
||||
|
||||
/**
|
||||
* Manages audio input and worklet processing for the VAD module
|
||||
*/
|
||||
export class VADAudioManager {
|
||||
private audioContext: AudioContext | null = null
|
||||
private audioWorkletNode: AudioWorkletNode | null = null
|
||||
private mediaStream: MediaStream | null = null
|
||||
private sourceNode: MediaStreamAudioSourceNode | null = null
|
||||
private vad: VAD
|
||||
// private minChunkSize: number;
|
||||
private workletInitialized: boolean = false
|
||||
|
||||
/**
|
||||
* Create a new VAD audio manager
|
||||
*/
|
||||
constructor(vad: VAD, options: VADAudioOptions = {}) {
|
||||
this.vad = vad
|
||||
// this.minChunkSize = options.minChunkSize || 512;
|
||||
|
||||
// Create audio context with user options or defaults
|
||||
this.audioContext = new AudioContext(options.audioContextOptions || {
|
||||
sampleRate: 16000, // Match the VAD sample rate
|
||||
latencyHint: 'interactive',
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize the audio worklet and connect to microphone
|
||||
*/
|
||||
public async initialize(workletUrl: string): Promise<void> {
|
||||
if (!this.audioContext) {
|
||||
throw new Error('Audio context not created')
|
||||
}
|
||||
|
||||
try {
|
||||
if (!this.workletInitialized) {
|
||||
await this.audioContext.audioWorklet.addModule(workletUrl)
|
||||
URL.revokeObjectURL(workletUrl)
|
||||
this.workletInitialized = true
|
||||
}
|
||||
|
||||
// Create the worklet node
|
||||
this.audioWorkletNode = new AudioWorkletNode(this.audioContext, 'vad-processor')
|
||||
|
||||
// Set up message handling from the worklet
|
||||
this.audioWorkletNode.port.onmessage = async (event) => {
|
||||
const { buffer } = event.data
|
||||
if (buffer && buffer.length > 0) {
|
||||
await this.vad.processAudio(new Float32Array(buffer))
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (error) {
|
||||
console.error('Failed to initialize audio worklet:', error)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Start capturing audio from the microphone
|
||||
*/
|
||||
public async startMicrophone(): Promise<void> {
|
||||
if (!this.audioContext || !this.audioWorkletNode) {
|
||||
throw new Error('Audio system not initialized. Call initialize() first.')
|
||||
}
|
||||
|
||||
try {
|
||||
if (this.audioContext.state === 'suspended') {
|
||||
await this.audioContext.resume()
|
||||
}
|
||||
|
||||
// Request microphone access
|
||||
this.mediaStream = await navigator.mediaDevices.getUserMedia({
|
||||
audio: {
|
||||
echoCancellation: true,
|
||||
noiseSuppression: true,
|
||||
autoGainControl: true,
|
||||
sampleRate: this.audioContext.sampleRate,
|
||||
},
|
||||
})
|
||||
|
||||
// Create source node and connect to worklet
|
||||
this.sourceNode = this.audioContext.createMediaStreamSource(this.mediaStream)
|
||||
this.sourceNode.connect(this.audioWorkletNode)
|
||||
|
||||
// Connect worklet to a silent destination (to keep the audio graph active)
|
||||
// Using a GainNode with gain=0 to ensure no sound is output
|
||||
const silentGain = this.audioContext.createGain()
|
||||
silentGain.gain.value = 0
|
||||
this.audioWorkletNode.connect(silentGain)
|
||||
silentGain.connect(this.audioContext.destination)
|
||||
}
|
||||
catch (error) {
|
||||
console.error('Failed to start microphone:', error)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
public async stopMicrophone(): Promise<void> {
|
||||
if (this.mediaStream) {
|
||||
this.mediaStream.getTracks().forEach(track => track.stop())
|
||||
this.mediaStream = null
|
||||
}
|
||||
|
||||
if (this.sourceNode) {
|
||||
this.sourceNode.disconnect()
|
||||
this.sourceNode = null
|
||||
}
|
||||
|
||||
this.audioContext?.suspend()
|
||||
this.audioWorkletNode?.disconnect()
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop capturing audio
|
||||
*/
|
||||
public stop(): void {
|
||||
// Disconnect nodes
|
||||
if (this.sourceNode && this.audioWorkletNode) {
|
||||
this.sourceNode.disconnect()
|
||||
this.audioWorkletNode.disconnect()
|
||||
}
|
||||
|
||||
// Stop all tracks in the media stream
|
||||
if (this.mediaStream) {
|
||||
this.mediaStream.getTracks().forEach(track => track.stop())
|
||||
this.mediaStream = null
|
||||
}
|
||||
|
||||
// Suspend the audio context rather than closing it
|
||||
// This allows us to reuse it later
|
||||
if (this.audioContext && this.audioContext.state !== 'closed') {
|
||||
this.audioContext.suspend()
|
||||
}
|
||||
|
||||
this.sourceNode = null
|
||||
this.audioWorkletNode = null
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean up all resources
|
||||
*/
|
||||
public dispose(): void {
|
||||
this.stop()
|
||||
|
||||
// Now fully close the audio context
|
||||
if (this.audioContext && this.audioContext.state !== 'closed') {
|
||||
this.audioContext.close()
|
||||
this.audioContext = null
|
||||
}
|
||||
|
||||
this.workletInitialized = false
|
||||
}
|
||||
}
|
||||
@@ -1,53 +0,0 @@
|
||||
// vad-worklet-processor.ts
|
||||
// This file needs to be registered as an AudioWorklet
|
||||
|
||||
/**
|
||||
* Minimum chunk size for processing audio
|
||||
*/
|
||||
const MIN_CHUNK_SIZE = 512
|
||||
|
||||
/**
|
||||
* Global state for audio buffer accumulation
|
||||
*/
|
||||
let globalPointer = 0
|
||||
const globalBuffer = new Float32Array(MIN_CHUNK_SIZE)
|
||||
|
||||
/**
|
||||
* VAD AudioWorklet Processor - processes audio chunks and sends them to the main thread
|
||||
*/
|
||||
class VADProcessor extends AudioWorkletProcessor {
|
||||
process(inputs: Float32Array[][], _outputs: Float32Array[][], _parameters: Record<string, Float32Array>) {
|
||||
const buffer = inputs[0][0]
|
||||
if (!buffer)
|
||||
return true // buffer is null when the stream ends
|
||||
|
||||
if (buffer.length > MIN_CHUNK_SIZE) {
|
||||
// If the buffer is larger than the minimum chunk size, send the entire buffer
|
||||
this.port.postMessage({ buffer })
|
||||
}
|
||||
else {
|
||||
const remaining = MIN_CHUNK_SIZE - globalPointer
|
||||
if (buffer.length >= remaining) {
|
||||
// If the buffer is larger than (or equal to) the remaining space in the global buffer, copy the remaining space
|
||||
globalBuffer.set(buffer.subarray(0, remaining), globalPointer)
|
||||
|
||||
// Send the global buffer
|
||||
this.port.postMessage({ buffer: globalBuffer })
|
||||
|
||||
// Reset the global buffer and set the remaining buffer
|
||||
globalBuffer.fill(0)
|
||||
globalBuffer.set(buffer.subarray(remaining), 0)
|
||||
globalPointer = buffer.length - remaining
|
||||
}
|
||||
else {
|
||||
// If the buffer is smaller than the remaining space in the global buffer, copy the buffer to the global buffer
|
||||
globalBuffer.set(buffer, globalPointer)
|
||||
globalPointer += buffer.length
|
||||
}
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
registerProcessor('vad-processor', VADProcessor)
|
||||
@@ -1,319 +0,0 @@
|
||||
import { AutoModel, Tensor } from '@huggingface/transformers'
|
||||
|
||||
// Default configuration parameters
|
||||
export interface VADConfig {
|
||||
// Sample rate of the audio
|
||||
sampleRate: number
|
||||
// Probabilities above this value are considered speech
|
||||
speechThreshold: number
|
||||
// Threshold to exit speech state
|
||||
exitThreshold: number
|
||||
// Minimum silence duration to consider speech ended (ms)
|
||||
minSilenceDurationMs: number
|
||||
// Padding to add before and after speech (ms)
|
||||
speechPadMs: number
|
||||
// Minimum duration of speech to consider valid (ms)
|
||||
minSpeechDurationMs: number
|
||||
// Maximum buffer duration in seconds
|
||||
maxBufferDuration: number
|
||||
// Size of input buffers from audio source
|
||||
newBufferSize: number
|
||||
}
|
||||
|
||||
export interface VADEvents {
|
||||
// Emitted when speech is detected
|
||||
'speech-start': void
|
||||
// Emitted when speech has ended
|
||||
'speech-end': void
|
||||
// Emitted when a complete speech segment is ready for transcription
|
||||
'speech-ready': { buffer: Float32Array, duration: number }
|
||||
// Emitted for status updates and errors
|
||||
'status': { type: string, message: string }
|
||||
// Debug info
|
||||
'debug': { message: string, data?: any }
|
||||
}
|
||||
|
||||
export type VADEventCallback<K extends keyof VADEvents> =
|
||||
(event: VADEvents[K]) => void
|
||||
|
||||
/**
|
||||
* Voice Activity Detection processor
|
||||
*/
|
||||
export class VAD {
|
||||
private config: VADConfig
|
||||
private model: any
|
||||
private state: Tensor
|
||||
private sampleRateTensor: Tensor
|
||||
private buffer: Float32Array
|
||||
private bufferPointer: number = 0
|
||||
private isRecording: boolean = false
|
||||
private postSpeechSamples: number = 0
|
||||
private prevBuffers: Float32Array[] = []
|
||||
private inferenceChain: Promise<any> = Promise.resolve()
|
||||
private eventListeners: Partial<Record<keyof VADEvents, VADEventCallback<any>[]>> = {}
|
||||
private isReady: boolean = false
|
||||
|
||||
constructor(userConfig: Partial<VADConfig> = {}) {
|
||||
// Default configuration
|
||||
const defaultConfig: VADConfig = {
|
||||
sampleRate: 16000,
|
||||
speechThreshold: 0.3,
|
||||
exitThreshold: 0.1,
|
||||
minSilenceDurationMs: 400,
|
||||
speechPadMs: 80,
|
||||
minSpeechDurationMs: 250,
|
||||
maxBufferDuration: 30,
|
||||
newBufferSize: 512,
|
||||
}
|
||||
|
||||
this.config = { ...defaultConfig, ...userConfig }
|
||||
|
||||
// Create buffer based on max duration
|
||||
this.buffer = new Float32Array(this.config.maxBufferDuration * this.config.sampleRate)
|
||||
|
||||
// Initialize state tensor for VAD model
|
||||
this.state = new Tensor('float32', new Float32Array(2 * 1 * 128), [2, 1, 128])
|
||||
|
||||
// Sample rate tensor for the model
|
||||
this.sampleRateTensor = new Tensor('int64', [this.config.sampleRate], [])
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize the VAD model
|
||||
*/
|
||||
public async initialize(): Promise<void> {
|
||||
try {
|
||||
this.emit('status', { type: 'info', message: 'Loading VAD model...' })
|
||||
|
||||
this.model = await AutoModel.from_pretrained('onnx-community/silero-vad', {
|
||||
config: { model_type: 'custom' } as any,
|
||||
dtype: 'fp32', // Full-precision
|
||||
})
|
||||
|
||||
this.isReady = true
|
||||
this.emit('status', { type: 'info', message: 'VAD model loaded successfully' })
|
||||
}
|
||||
catch (error) {
|
||||
this.emit('status', { type: 'error', message: `Failed to load VAD model: ${error}` })
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Add event listener
|
||||
*/
|
||||
public on<K extends keyof VADEvents>(event: K, callback: VADEventCallback<K>): void {
|
||||
if (!this.eventListeners[event]) {
|
||||
this.eventListeners[event] = []
|
||||
}
|
||||
this.eventListeners[event]!.push(callback as any)
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove event listener
|
||||
*/
|
||||
public off<K extends keyof VADEvents>(event: K, callback: VADEventCallback<K>): void {
|
||||
if (!this.eventListeners[event])
|
||||
return
|
||||
this.eventListeners[event] = this.eventListeners[event]!.filter(cb => cb !== callback)
|
||||
}
|
||||
|
||||
/**
|
||||
* Emit event
|
||||
*/
|
||||
private emit<K extends keyof VADEvents>(event: K, data: VADEvents[K]): void {
|
||||
if (!this.eventListeners[event])
|
||||
return
|
||||
for (const callback of this.eventListeners[event]!) {
|
||||
callback(data)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Process audio buffer for speech detection
|
||||
*/
|
||||
public async processAudio(inputBuffer: Float32Array): Promise<void> {
|
||||
if (!this.isReady) {
|
||||
throw new Error('VAD model is not initialized. Call initialize() first.')
|
||||
}
|
||||
|
||||
const wasRecording = this.isRecording
|
||||
|
||||
// Perform VAD on the input buffer
|
||||
const isSpeech = await this.detectSpeech(inputBuffer)
|
||||
|
||||
// Calculate derived constants
|
||||
const sampleRateMs = this.config.sampleRate / 1000
|
||||
const minSilenceDurationSamples = this.config.minSilenceDurationMs * sampleRateMs
|
||||
const speechPadSamples = this.config.speechPadMs * sampleRateMs
|
||||
const minSpeechDurationSamples = this.config.minSpeechDurationMs * sampleRateMs
|
||||
const maxPrevBuffers = Math.ceil(speechPadSamples / this.config.newBufferSize)
|
||||
|
||||
// If not currently in speech and the current buffer isn't speech,
|
||||
// store it in the previous buffers queue for potential padding
|
||||
if (!wasRecording && !isSpeech) {
|
||||
if (this.prevBuffers.length >= maxPrevBuffers) {
|
||||
this.prevBuffers.shift()
|
||||
}
|
||||
this.prevBuffers.push(inputBuffer.slice(0))
|
||||
return
|
||||
}
|
||||
|
||||
// Check if we need to handle buffer overflow
|
||||
const remaining = this.buffer.length - this.bufferPointer
|
||||
if (inputBuffer.length >= remaining) {
|
||||
// The buffer is full, process what we have
|
||||
this.buffer.set(inputBuffer.subarray(0, remaining), this.bufferPointer)
|
||||
this.bufferPointer += remaining
|
||||
|
||||
// Process and reset with overflow
|
||||
const overflow = inputBuffer.subarray(remaining)
|
||||
this.processSpeechSegment(overflow)
|
||||
return
|
||||
}
|
||||
else {
|
||||
// Add input to the buffer
|
||||
this.buffer.set(inputBuffer, this.bufferPointer)
|
||||
this.bufferPointer += inputBuffer.length
|
||||
}
|
||||
|
||||
// Handle speech detection
|
||||
if (isSpeech) {
|
||||
if (!this.isRecording) {
|
||||
// Speech just started
|
||||
this.emit('speech-start', undefined)
|
||||
this.emit('status', { type: 'info', message: 'Speech detected' })
|
||||
}
|
||||
|
||||
// Update state
|
||||
this.isRecording = true
|
||||
this.postSpeechSamples = 0
|
||||
return
|
||||
}
|
||||
|
||||
// At this point, we were recording but the current buffer is not speech
|
||||
this.postSpeechSamples += inputBuffer.length
|
||||
|
||||
// Check if silence is long enough to consider speech ended
|
||||
if (this.postSpeechSamples >= minSilenceDurationSamples) {
|
||||
// Check if the speech segment is long enough to process
|
||||
if (this.bufferPointer < minSpeechDurationSamples) {
|
||||
// Too short, reset without processing
|
||||
this.reset()
|
||||
return
|
||||
}
|
||||
|
||||
// Process the speech segment
|
||||
this.processSpeechSegment()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect speech in an audio buffer
|
||||
*/
|
||||
private async detectSpeech(buffer: Float32Array): Promise<boolean> {
|
||||
const input = new Tensor('float32', buffer, [1, buffer.length])
|
||||
|
||||
const { stateN, output } = await (this.inferenceChain = this.inferenceChain.then(() =>
|
||||
this.model({
|
||||
input,
|
||||
sr: this.sampleRateTensor,
|
||||
state: this.state,
|
||||
}),
|
||||
))
|
||||
|
||||
// Update the state
|
||||
this.state = stateN
|
||||
|
||||
// Get the speech probability
|
||||
const speechProb = output.data[0]
|
||||
|
||||
this.emit('debug', {
|
||||
message: 'VAD score',
|
||||
data: { probability: speechProb },
|
||||
})
|
||||
|
||||
// Apply thresholds
|
||||
return (
|
||||
speechProb > this.config.speechThreshold
|
||||
|| (this.isRecording && speechProb >= this.config.exitThreshold)
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Process a complete speech segment
|
||||
*/
|
||||
private processSpeechSegment(overflow?: Float32Array): void {
|
||||
const sampleRateMs = this.config.sampleRate / 1000
|
||||
const speechPadSamples = this.config.speechPadMs * sampleRateMs
|
||||
|
||||
// Calculate duration info
|
||||
const duration = (this.bufferPointer / this.config.sampleRate) * 1000
|
||||
const overflowLength = overflow?.length ?? 0
|
||||
|
||||
// Create the final buffer with padding
|
||||
const prevLength = this.prevBuffers.reduce((acc, b) => acc + b.length, 0)
|
||||
const finalBuffer = new Float32Array(prevLength + this.bufferPointer + speechPadSamples)
|
||||
|
||||
// Add previous buffers for pre-speech padding
|
||||
let offset = 0
|
||||
for (const prev of this.prevBuffers) {
|
||||
finalBuffer.set(prev, offset)
|
||||
offset += prev.length
|
||||
}
|
||||
|
||||
// Add the main speech segment
|
||||
finalBuffer.set(this.buffer.slice(0, this.bufferPointer + speechPadSamples), offset)
|
||||
|
||||
// Emit the speech segment
|
||||
this.emit('speech-end', undefined)
|
||||
this.emit('speech-ready', {
|
||||
buffer: finalBuffer,
|
||||
duration,
|
||||
})
|
||||
|
||||
// Reset for the next segment
|
||||
if (overflow) {
|
||||
this.buffer.set(overflow, 0)
|
||||
}
|
||||
this.reset(overflowLength)
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset the VAD state
|
||||
*/
|
||||
private reset(offset: number = 0): void {
|
||||
this.buffer.fill(0, offset)
|
||||
this.bufferPointer = offset
|
||||
this.isRecording = false
|
||||
this.postSpeechSamples = 0
|
||||
this.prevBuffers = []
|
||||
}
|
||||
|
||||
/**
|
||||
* Update configuration
|
||||
*/
|
||||
public updateConfig(newConfig: Partial<VADConfig>): void {
|
||||
this.config = { ...this.config, ...newConfig }
|
||||
|
||||
// If buffer size changed, create a new buffer
|
||||
if (newConfig.maxBufferDuration || newConfig.sampleRate) {
|
||||
this.buffer = new Float32Array(this.config.maxBufferDuration * this.config.sampleRate)
|
||||
this.bufferPointer = 0
|
||||
}
|
||||
|
||||
// Update sample rate tensor if needed
|
||||
if (newConfig.sampleRate) {
|
||||
this.sampleRateTensor = new Tensor('int64', [this.config.sampleRate], [])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a VAD processor with the given configuration
|
||||
*/
|
||||
export async function createVAD(config?: Partial<VADConfig>): Promise<VAD> {
|
||||
const vad = new VAD(config)
|
||||
await vad.initialize()
|
||||
return vad
|
||||
}
|
||||
@@ -1,43 +0,0 @@
|
||||
function writeString(dataView: DataView, offset: number, string: string) {
|
||||
for (let i = 0; i < string.length; i++) {
|
||||
dataView.setUint8(offset + i, string.charCodeAt(i))
|
||||
}
|
||||
}
|
||||
|
||||
export function toWav(buffer: Float32Array, sampleRate: number) {
|
||||
const numChannels = 1
|
||||
const numSamples = buffer.length
|
||||
|
||||
// Create the WAV file container
|
||||
const arrayBuffer = new ArrayBuffer(44 + numSamples * 2)
|
||||
const dataView = new DataView(arrayBuffer)
|
||||
|
||||
// RIFF chunk descriptor
|
||||
writeString(dataView, 0, 'RIFF')
|
||||
dataView.setUint32(4, 36 + numSamples * 2, true)
|
||||
writeString(dataView, 8, 'WAVE')
|
||||
|
||||
// fmt sub-chunk
|
||||
writeString(dataView, 12, 'fmt ')
|
||||
dataView.setUint32(16, 16, true)
|
||||
dataView.setUint16(20, 1, true) // PCM format
|
||||
dataView.setUint16(22, numChannels, true)
|
||||
dataView.setUint32(24, sampleRate, true)
|
||||
dataView.setUint32(28, sampleRate * numChannels * 2, true) // byte rate
|
||||
dataView.setUint16(32, numChannels * 2, true) // block align
|
||||
dataView.setUint16(34, 16, true) // bits per sample
|
||||
|
||||
// data sub-chunk
|
||||
writeString(dataView, 36, 'data')
|
||||
dataView.setUint32(40, numSamples * 2, true)
|
||||
|
||||
// Write the PCM samples
|
||||
const offset = 44
|
||||
for (let i = 0; i < numSamples; i++) {
|
||||
const sample = Math.max(-1, Math.min(1, buffer[i]))
|
||||
const value = sample < 0 ? sample * 0x8000 : sample * 0x7FFF
|
||||
dataView.setInt16(offset + i * 2, value, true)
|
||||
}
|
||||
|
||||
return arrayBuffer
|
||||
}
|
||||
@@ -1,13 +0,0 @@
|
||||
import { createApp } from 'vue'
|
||||
import { createRouter, createWebHashHistory } from 'vue-router'
|
||||
import { routes } from 'vue-router/auto-routes'
|
||||
|
||||
import App from './App.vue'
|
||||
import '@unocss/reset/tailwind.css'
|
||||
import 'uno.css'
|
||||
|
||||
const router = createRouter({ routes, history: createWebHashHistory() })
|
||||
|
||||
createApp(App)
|
||||
.use(router)
|
||||
.mount('#app')
|
||||
@@ -1,205 +0,0 @@
|
||||
<script setup lang="ts">
|
||||
import { generateTranscription } from '@xsai/generate-transcription'
|
||||
import { ref } from 'vue'
|
||||
|
||||
import FieldInput from '../components/FieldInput.vue'
|
||||
import { VADAudioManager } from '../libs/vad/manager'
|
||||
import workletUrl from '../libs/vad/process.worklet?worker&url'
|
||||
import { createVAD } from '../libs/vad/vad'
|
||||
import { toWav } from '../libs/vad/wav'
|
||||
|
||||
interface AudioSegment {
|
||||
buffer: Float32Array
|
||||
duration: number
|
||||
timestamp: number
|
||||
audioSrc: string
|
||||
transcription: string
|
||||
}
|
||||
|
||||
const asrProviderBaseURL = ref('http://localhost:8000/v1/')
|
||||
const asrProviderAPIKey = ref('')
|
||||
const asrProviderModel = ref('large-v3-turbo')
|
||||
|
||||
const audioManager = ref<VADAudioManager>()
|
||||
const isInitialized = ref(false)
|
||||
const isRunning = ref(false)
|
||||
const isSpeechDetected = ref(false)
|
||||
const segments = ref<AudioSegment[]>([])
|
||||
const error = ref<string | null>(null)
|
||||
const isModuleLoading = ref(false)
|
||||
|
||||
async function setupSpeechDetection() {
|
||||
try {
|
||||
isModuleLoading.value = true
|
||||
|
||||
// Create and initialize the VAD
|
||||
const vad = await createVAD({
|
||||
sampleRate: 16000,
|
||||
speechThreshold: 0.3,
|
||||
exitThreshold: 0.1,
|
||||
minSilenceDurationMs: 400,
|
||||
})
|
||||
|
||||
// Set up event handlers
|
||||
vad.on('speech-start', () => {
|
||||
isSpeechDetected.value = true
|
||||
})
|
||||
|
||||
vad.on('speech-end', () => {
|
||||
isSpeechDetected.value = false
|
||||
})
|
||||
|
||||
vad.on('speech-ready', async ({ buffer, duration }) => {
|
||||
const wavBuffer = toWav(buffer, 16000)
|
||||
const audioBlob = new Blob([wavBuffer], { type: 'audio/wav' })
|
||||
|
||||
const obj = {
|
||||
buffer,
|
||||
duration: duration / 1000, // Convert to seconds for display
|
||||
timestamp: Date.now(),
|
||||
audioSrc: URL.createObjectURL(audioBlob),
|
||||
transcription: '',
|
||||
}
|
||||
|
||||
// Store the segment
|
||||
const objIndex = segments.value.push(obj)
|
||||
|
||||
generateTranscription({
|
||||
baseURL: asrProviderBaseURL.value,
|
||||
file: audioBlob,
|
||||
model: asrProviderModel.value,
|
||||
apiKey: asrProviderAPIKey.value,
|
||||
}).then((res) => {
|
||||
segments.value[objIndex - 1].transcription = res.text
|
||||
}).catch((err) => {
|
||||
console.error('Failed to generate transcription:', err)
|
||||
})
|
||||
})
|
||||
|
||||
vad.on('status', ({ type, message }) => {
|
||||
if (type === 'error') {
|
||||
error.value = message
|
||||
}
|
||||
})
|
||||
|
||||
// Create and initialize audio manager
|
||||
const m = new VADAudioManager(vad, {
|
||||
minChunkSize: 512,
|
||||
audioContextOptions: {
|
||||
sampleRate: 16000,
|
||||
latencyHint: 'interactive',
|
||||
},
|
||||
})
|
||||
|
||||
await m.initialize(workletUrl)
|
||||
audioManager.value = m
|
||||
isInitialized.value = true
|
||||
isModuleLoading.value = false
|
||||
startVad()
|
||||
}
|
||||
catch (err) {
|
||||
console.error('Setup failed:', err)
|
||||
error.value = err instanceof Error ? err.message : String(err)
|
||||
isModuleLoading.value = false
|
||||
}
|
||||
}
|
||||
|
||||
async function destroySpeechDetection() {
|
||||
await audioManager.value?.stop()
|
||||
await audioManager.value?.dispose()
|
||||
isInitialized.value = false
|
||||
isRunning.value = false
|
||||
isSpeechDetected.value = false
|
||||
|
||||
for (const segment of segments.value) {
|
||||
URL.revokeObjectURL(segment.audioSrc)
|
||||
}
|
||||
|
||||
segments.value = []
|
||||
error.value = null
|
||||
isModuleLoading.value = false
|
||||
}
|
||||
|
||||
async function startVad() {
|
||||
try {
|
||||
await audioManager.value?.startMicrophone()
|
||||
isRunning.value = true
|
||||
error.value = null
|
||||
}
|
||||
catch (err) {
|
||||
console.error('Failed to start microphone:', err)
|
||||
error.value = err instanceof Error ? err.message : String(err)
|
||||
}
|
||||
}
|
||||
|
||||
async function stopVad() {
|
||||
await audioManager.value?.stopMicrophone()
|
||||
isRunning.value = false
|
||||
isSpeechDetected.value = false
|
||||
}
|
||||
|
||||
function toggleListening() {
|
||||
if (isRunning.value) {
|
||||
stopVad()
|
||||
}
|
||||
else {
|
||||
startVad()
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<div mb-6 mt-4 h-full w-full flex flex-col gap-2>
|
||||
<div w-full flex-1>
|
||||
<div v-if="!isRunning">
|
||||
<div flex="~ col gap-4">
|
||||
<FieldInput v-model="asrProviderBaseURL" label="ASR Provider Base URL" description="The base URL of the ASR provider. Generally, Speaches is recommended." />
|
||||
<FieldInput v-model="asrProviderAPIKey" label="ASR Provider API Key" description="The API key of the ASR provider" />
|
||||
<FieldInput v-model="asrProviderModel" label="ASR Provider Model" description="The model to use for the ASR provider" />
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div v-if="isModuleLoading" mt-20 flex items-center justify-center text-5xl>
|
||||
<div i-svg-spinners:3-dots-move />
|
||||
</div>
|
||||
|
||||
<div v-if="error" class="error">
|
||||
{{ error }}
|
||||
</div>
|
||||
|
||||
<div v-if="segments?.length && segments.length > 0" class="segments" my-4 w-full flex flex-col gap-2>
|
||||
<h3>Voice Segments ({{ segments.length }})</h3>
|
||||
<ul>
|
||||
<li v-for="(segment, index) in segments" :key="index" class="segment" flex flex-col gap-2>
|
||||
<div class="segment-info" grid="~ cols-[120px_1fr] gap-2">
|
||||
<span text="neutral-400 dark:neutral-500">Duration</span>
|
||||
<span font-mono>
|
||||
{{ segment.duration.toFixed(2) }}s
|
||||
</span>
|
||||
<span text="neutral-400 dark:neutral-500">Transcription</span>
|
||||
<span>
|
||||
{{ segment.transcription }}
|
||||
</span>
|
||||
</div>
|
||||
<audio :src="segment.audioSrc" controls w-full />
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div w-full flex justify-center gap-4>
|
||||
<button aspect-square size-15 flex items-center justify-center rounded-full text-2xl :class="[isRunning ? 'bg-neutral-700 dark:bg-white text-white dark:text-neutral-900' : 'bg-neutral-900 dark:bg-white/20 text-white dark:text-white', isInitialized ? 'opacity-100' : 'opacity-0']" :disabled="!isInitialized" @click="toggleListening">
|
||||
<div i-solar:microphone-3-bold />
|
||||
</button>
|
||||
<button v-if="!isInitialized" bg="green-500 dark:green-500 hover:green-400 dark:hover:green-400 active:green-500 dark:active:green-500" transition="all duration-250 ease-in-out" aspect-square size-15 flex items-center justify-center rounded-full @click="setupSpeechDetection">
|
||||
<div i-solar:phone-rounded-bold text-4xl text-white />
|
||||
</button>
|
||||
<button v-else bg="red-500 dark:red-500 hover:red-400 dark:hover:red-400 active:red-500 dark:active:red-500" transition="all duration-250 ease-in-out" aspect-square size-15 flex items-center justify-center rounded-full text-4xl text-white @click="destroySpeechDetection">
|
||||
<div i-solar:end-call-rounded-bold />
|
||||
</button>
|
||||
<button aspect-square size-15 flex items-center justify-center rounded-full text-2xl class="bg-neutral-900 text-white dark:bg-neutral-900 dark:bg-white/20 dark:text-white" :class="isInitialized ? 'opacity-100' : 'opacity-0'" :disabled="!isInitialized">
|
||||
<div i-solar:headphones-round-bold />
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
@@ -1,41 +0,0 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ESNext",
|
||||
"jsx": "preserve",
|
||||
"lib": [
|
||||
"DOM",
|
||||
"ESNext",
|
||||
"WebWorker"
|
||||
],
|
||||
"module": "ESNext",
|
||||
"moduleResolution": "Bundler",
|
||||
"resolveJsonModule": true,
|
||||
"types": [
|
||||
"vitest",
|
||||
"vite/client",
|
||||
// Currently AudioWorkletProcessor type is missing, we need to add it manually through @types/audioworklet
|
||||
// https://github.com/microsoft/TypeScript/issues/28308#issuecomment-1512509870
|
||||
"@types/audioworklet",
|
||||
"unplugin-vue-router/client"
|
||||
],
|
||||
"allowJs": true,
|
||||
"strict": true,
|
||||
"strictNullChecks": true,
|
||||
"noUnusedLocals": true,
|
||||
"noEmit": true,
|
||||
"esModuleInterop": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"isolatedModules": true,
|
||||
"skipLibCheck": true
|
||||
},
|
||||
"include": [
|
||||
"src/**/*.ts",
|
||||
"src/**/*.tsx",
|
||||
"src/**/*.vue",
|
||||
"src/**/*.js"
|
||||
],
|
||||
"exclude": [
|
||||
"dist",
|
||||
"node_modules"
|
||||
]
|
||||
}
|
||||
@@ -1,37 +0,0 @@
|
||||
import {
|
||||
defineConfig,
|
||||
presetAttributify,
|
||||
presetIcons,
|
||||
presetTypography,
|
||||
presetWebFonts,
|
||||
presetWind3,
|
||||
transformerDirectives,
|
||||
transformerVariantGroup,
|
||||
} from 'unocss'
|
||||
|
||||
export default defineConfig({
|
||||
presets: [
|
||||
presetWind3(),
|
||||
presetAttributify(),
|
||||
presetTypography(),
|
||||
presetWebFonts({
|
||||
fonts: {
|
||||
sans: 'DM Sans',
|
||||
serif: 'DM Serif Display',
|
||||
mono: 'DM Mono',
|
||||
},
|
||||
timeouts: {
|
||||
warning: 5000,
|
||||
failure: 10000,
|
||||
},
|
||||
}),
|
||||
presetIcons({
|
||||
scale: 1.2,
|
||||
}),
|
||||
],
|
||||
transformers: [
|
||||
transformerDirectives(),
|
||||
transformerVariantGroup(),
|
||||
],
|
||||
safelist: 'prose prose-sm m-auto text-left'.split(' '),
|
||||
})
|
||||
@@ -1,19 +0,0 @@
|
||||
import { resolve } from 'node:path'
|
||||
import Vue from '@vitejs/plugin-vue'
|
||||
import Unocss from 'unocss/vite'
|
||||
import VueRouter from 'unplugin-vue-router/vite'
|
||||
import { defineConfig } from 'vite'
|
||||
|
||||
export default defineConfig({
|
||||
plugins: [
|
||||
// https://github.com/posva/unplugin-vue-router
|
||||
VueRouter({
|
||||
extensions: ['.vue', '.md'],
|
||||
dts: resolve(import.meta.dirname, 'src', 'typed-router.d.ts'),
|
||||
}),
|
||||
Vue(),
|
||||
// https://github.com/antfu/unocss
|
||||
// see uno.config.ts for config
|
||||
Unocss(),
|
||||
],
|
||||
})
|
||||
@@ -1,22 +0,0 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<title>Project AIRI VAD Playground</title>
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=0" />
|
||||
<link rel="icon" type="image/svg+xml" href="/favicon.svg" />
|
||||
<script>
|
||||
;(function () {
|
||||
const prefersDark = window.matchMedia && window.matchMedia('(prefers-color-scheme: dark)').matches
|
||||
const setting = localStorage.getItem('vueuse-color-scheme') || 'auto'
|
||||
if (setting === 'dark' || (prefersDark && setting !== 'light'))
|
||||
document.documentElement.classList.toggle('dark', true)
|
||||
})()
|
||||
</script>
|
||||
</head>
|
||||
<body class="font-sans">
|
||||
<div id="app"></div>
|
||||
<script type="module" src="/src/main.ts"></script>
|
||||
<noscript> This website requires JavaScript to function properly. Please enable JavaScript to continue. </noscript>
|
||||
</body>
|
||||
</html>
|
||||
@@ -1 +0,0 @@
|
||||
title: VAD
|
||||
@@ -1 +0,0 @@
|
||||
title: VAD
|
||||
@@ -1,13 +0,0 @@
|
||||
[build]
|
||||
base = "/"
|
||||
command = "pnpm -F @proj-airi/vad... run build"
|
||||
publish = "/apps/vad/dist"
|
||||
|
||||
[build.environment]
|
||||
NODE_VERSION = "23"
|
||||
|
||||
[[redirects]]
|
||||
from = "/*"
|
||||
to = "/index.html"
|
||||
status = 200
|
||||
force = false
|
||||
@@ -1,41 +0,0 @@
|
||||
{
|
||||
"name": "@proj-airi/vad",
|
||||
"type": "module",
|
||||
"private": true,
|
||||
"description": "Voice Activity Detector",
|
||||
"author": {
|
||||
"name": "Neko Ayaka",
|
||||
"email": "neko@ayaka.moe",
|
||||
"url": "https://github.com/nekomeowww"
|
||||
},
|
||||
"license": "MIT",
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/moeru-ai/airi.git",
|
||||
"directory": "apps/vad"
|
||||
},
|
||||
"scripts": {
|
||||
"dev": "vite",
|
||||
"build": "vite build",
|
||||
"preview": "vite preview",
|
||||
"typecheck": "vue-tsc --noEmit"
|
||||
},
|
||||
"dependencies": {
|
||||
"@huggingface/transformers": "^3.4.2",
|
||||
"@vueuse/core": "^13.0.0",
|
||||
"defu": "^6.1.4",
|
||||
"es-toolkit": "^1.34.1",
|
||||
"vue": "^3.5.13"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@iconify-json/solar": "^1.2.2",
|
||||
"@iconify-json/svg-spinners": "^1.2.2",
|
||||
"@types/audioworklet": "^0.0.72",
|
||||
"@unocss/reset": "^66.1.0-beta.10",
|
||||
"@vitejs/plugin-vue": "^5.2.3",
|
||||
"unplugin-vue-router": "^0.12.0",
|
||||
"vite": "^6.2.5",
|
||||
"vue-router": "^4.5.0",
|
||||
"vue-tsc": "^3.0.0-alpha.2"
|
||||
}
|
||||
}
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 15 KiB |
File diff suppressed because one or more lines are too long
|
Before Width: | Height: | Size: 23 KiB |
@@ -1,56 +0,0 @@
|
||||
<script setup lang="ts">
|
||||
import { useDark, useToggle } from '@vueuse/core'
|
||||
import { RouterLink, RouterView } from 'vue-router'
|
||||
|
||||
const isDark = useDark()
|
||||
const toggleDark = useToggle(isDark)
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<div mx-auto h-full max-w-screen-lg flex flex-col gap-2 p-4>
|
||||
<header flex flex-row items-center justify-between>
|
||||
<h1 text-2xl>
|
||||
VAD Playground
|
||||
</h1>
|
||||
<div flex flex-row items-center gap-2>
|
||||
<button text-lg @click="() => toggleDark()">
|
||||
<div v-if="isDark" i-solar:moon-stars-bold-duotone />
|
||||
<div v-else i-solar:sun-bold />
|
||||
</button>
|
||||
<a href="https://github.com/moeru-ai/airi/tree/main/apps/vad">
|
||||
<div i-simple-icons:github />
|
||||
</a>
|
||||
</div>
|
||||
</header>
|
||||
<nav bg="neutral-100 dark:neutral-800" w-fit flex items-center of-hidden rounded-lg>
|
||||
<RouterLink
|
||||
to="/" px-3 py-2 bg="hover:neutral-200 dark:hover:neutral-700"
|
||||
transition="all duration-250 ease-in-out"
|
||||
>
|
||||
<h1>WASM</h1>
|
||||
</RouterLink>
|
||||
</nav>
|
||||
<RouterView />
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<style>
|
||||
html,
|
||||
body,
|
||||
#app {
|
||||
height: 100%;
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
overscroll-behavior: none;
|
||||
}
|
||||
|
||||
html {
|
||||
background: #fff;
|
||||
transition: all 0.3s ease-in-out;
|
||||
}
|
||||
|
||||
html.dark {
|
||||
background: #121212;
|
||||
color-scheme: dark;
|
||||
}
|
||||
</style>
|
||||
@@ -1,175 +0,0 @@
|
||||
// vad-audio-manager.ts
|
||||
|
||||
import type { VAD, VADConfig } from './vad'
|
||||
|
||||
export interface VADAudioOptions {
|
||||
/**
|
||||
* Audio context options
|
||||
*/
|
||||
audioContextOptions?: AudioContextOptions
|
||||
|
||||
/**
|
||||
* The minimum size of audio chunks to process
|
||||
*/
|
||||
minChunkSize?: number
|
||||
|
||||
/**
|
||||
* VAD configuration options
|
||||
*/
|
||||
vadConfig?: Partial<VADConfig>
|
||||
}
|
||||
|
||||
/**
|
||||
* Manages audio input and worklet processing for the VAD module
|
||||
*/
|
||||
export class VADAudioManager {
|
||||
private audioContext: AudioContext | null = null
|
||||
private audioWorkletNode: AudioWorkletNode | null = null
|
||||
private mediaStream: MediaStream | null = null
|
||||
private sourceNode: MediaStreamAudioSourceNode | null = null
|
||||
private vad: VAD
|
||||
// private minChunkSize: number;
|
||||
private workletInitialized: boolean = false
|
||||
|
||||
/**
|
||||
* Create a new VAD audio manager
|
||||
*/
|
||||
constructor(vad: VAD, options: VADAudioOptions = {}) {
|
||||
this.vad = vad
|
||||
// this.minChunkSize = options.minChunkSize || 512;
|
||||
|
||||
// Create audio context with user options or defaults
|
||||
this.audioContext = new AudioContext(options.audioContextOptions || {
|
||||
sampleRate: 16000, // Match the VAD sample rate
|
||||
latencyHint: 'interactive',
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize the audio worklet and connect to microphone
|
||||
*/
|
||||
public async initialize(workletUrl: string): Promise<void> {
|
||||
if (!this.audioContext) {
|
||||
throw new Error('Audio context not created')
|
||||
}
|
||||
|
||||
try {
|
||||
if (!this.workletInitialized) {
|
||||
await this.audioContext.audioWorklet.addModule(workletUrl)
|
||||
URL.revokeObjectURL(workletUrl)
|
||||
this.workletInitialized = true
|
||||
}
|
||||
|
||||
// Create the worklet node
|
||||
this.audioWorkletNode = new AudioWorkletNode(this.audioContext, 'vad-processor')
|
||||
|
||||
// Set up message handling from the worklet
|
||||
this.audioWorkletNode.port.onmessage = async (event) => {
|
||||
const { buffer } = event.data
|
||||
if (buffer && buffer.length > 0) {
|
||||
await this.vad.processAudio(new Float32Array(buffer))
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (error) {
|
||||
console.error('Failed to initialize audio worklet:', error)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Start capturing audio from the microphone
|
||||
*/
|
||||
public async startMicrophone(): Promise<void> {
|
||||
if (!this.audioContext || !this.audioWorkletNode) {
|
||||
throw new Error('Audio system not initialized. Call initialize() first.')
|
||||
}
|
||||
|
||||
try {
|
||||
if (this.audioContext.state === 'suspended') {
|
||||
await this.audioContext.resume()
|
||||
}
|
||||
|
||||
// Request microphone access
|
||||
this.mediaStream = await navigator.mediaDevices.getUserMedia({
|
||||
audio: {
|
||||
echoCancellation: true,
|
||||
noiseSuppression: true,
|
||||
autoGainControl: true,
|
||||
sampleRate: this.audioContext.sampleRate,
|
||||
},
|
||||
})
|
||||
|
||||
// Create source node and connect to worklet
|
||||
this.sourceNode = this.audioContext.createMediaStreamSource(this.mediaStream)
|
||||
this.sourceNode.connect(this.audioWorkletNode)
|
||||
|
||||
// Connect worklet to a silent destination (to keep the audio graph active)
|
||||
// Using a GainNode with gain=0 to ensure no sound is output
|
||||
const silentGain = this.audioContext.createGain()
|
||||
silentGain.gain.value = 0
|
||||
this.audioWorkletNode.connect(silentGain)
|
||||
silentGain.connect(this.audioContext.destination)
|
||||
}
|
||||
catch (error) {
|
||||
console.error('Failed to start microphone:', error)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
public async stopMicrophone(): Promise<void> {
|
||||
if (this.mediaStream) {
|
||||
this.mediaStream.getTracks().forEach(track => track.stop())
|
||||
this.mediaStream = null
|
||||
}
|
||||
|
||||
if (this.sourceNode) {
|
||||
this.sourceNode.disconnect()
|
||||
this.sourceNode = null
|
||||
}
|
||||
|
||||
this.audioContext?.suspend()
|
||||
this.audioWorkletNode?.disconnect()
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop capturing audio
|
||||
*/
|
||||
public stop(): void {
|
||||
// Disconnect nodes
|
||||
if (this.sourceNode && this.audioWorkletNode) {
|
||||
this.sourceNode.disconnect()
|
||||
this.audioWorkletNode.disconnect()
|
||||
}
|
||||
|
||||
// Stop all tracks in the media stream
|
||||
if (this.mediaStream) {
|
||||
this.mediaStream.getTracks().forEach(track => track.stop())
|
||||
this.mediaStream = null
|
||||
}
|
||||
|
||||
// Suspend the audio context rather than closing it
|
||||
// This allows us to reuse it later
|
||||
if (this.audioContext && this.audioContext.state !== 'closed') {
|
||||
this.audioContext.suspend()
|
||||
}
|
||||
|
||||
this.sourceNode = null
|
||||
this.audioWorkletNode = null
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean up all resources
|
||||
*/
|
||||
public dispose(): void {
|
||||
this.stop()
|
||||
|
||||
// Now fully close the audio context
|
||||
if (this.audioContext && this.audioContext.state !== 'closed') {
|
||||
this.audioContext.close()
|
||||
this.audioContext = null
|
||||
}
|
||||
|
||||
this.workletInitialized = false
|
||||
}
|
||||
}
|
||||
@@ -1,53 +0,0 @@
|
||||
// vad-worklet-processor.ts
|
||||
// This file needs to be registered as an AudioWorklet
|
||||
|
||||
/**
|
||||
* Minimum chunk size for processing audio
|
||||
*/
|
||||
const MIN_CHUNK_SIZE = 512
|
||||
|
||||
/**
|
||||
* Global state for audio buffer accumulation
|
||||
*/
|
||||
let globalPointer = 0
|
||||
const globalBuffer = new Float32Array(MIN_CHUNK_SIZE)
|
||||
|
||||
/**
|
||||
* VAD AudioWorklet Processor - processes audio chunks and sends them to the main thread
|
||||
*/
|
||||
class VADProcessor extends AudioWorkletProcessor {
|
||||
process(inputs: Float32Array[][], _outputs: Float32Array[][], _parameters: Record<string, Float32Array>) {
|
||||
const buffer = inputs[0][0]
|
||||
if (!buffer)
|
||||
return true // buffer is null when the stream ends
|
||||
|
||||
if (buffer.length > MIN_CHUNK_SIZE) {
|
||||
// If the buffer is larger than the minimum chunk size, send the entire buffer
|
||||
this.port.postMessage({ buffer })
|
||||
}
|
||||
else {
|
||||
const remaining = MIN_CHUNK_SIZE - globalPointer
|
||||
if (buffer.length >= remaining) {
|
||||
// If the buffer is larger than (or equal to) the remaining space in the global buffer, copy the remaining space
|
||||
globalBuffer.set(buffer.subarray(0, remaining), globalPointer)
|
||||
|
||||
// Send the global buffer
|
||||
this.port.postMessage({ buffer: globalBuffer })
|
||||
|
||||
// Reset the global buffer and set the remaining buffer
|
||||
globalBuffer.fill(0)
|
||||
globalBuffer.set(buffer.subarray(remaining), 0)
|
||||
globalPointer = buffer.length - remaining
|
||||
}
|
||||
else {
|
||||
// If the buffer is smaller than the remaining space in the global buffer, copy the buffer to the global buffer
|
||||
globalBuffer.set(buffer, globalPointer)
|
||||
globalPointer += buffer.length
|
||||
}
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
registerProcessor('vad-processor', VADProcessor)
|
||||
@@ -1,319 +0,0 @@
|
||||
import { AutoModel, Tensor } from '@huggingface/transformers'
|
||||
|
||||
// Default configuration parameters
|
||||
export interface VADConfig {
|
||||
// Sample rate of the audio
|
||||
sampleRate: number
|
||||
// Probabilities above this value are considered speech
|
||||
speechThreshold: number
|
||||
// Threshold to exit speech state
|
||||
exitThreshold: number
|
||||
// Minimum silence duration to consider speech ended (ms)
|
||||
minSilenceDurationMs: number
|
||||
// Padding to add before and after speech (ms)
|
||||
speechPadMs: number
|
||||
// Minimum duration of speech to consider valid (ms)
|
||||
minSpeechDurationMs: number
|
||||
// Maximum buffer duration in seconds
|
||||
maxBufferDuration: number
|
||||
// Size of input buffers from audio source
|
||||
newBufferSize: number
|
||||
}
|
||||
|
||||
export interface VADEvents {
|
||||
// Emitted when speech is detected
|
||||
'speech-start': void
|
||||
// Emitted when speech has ended
|
||||
'speech-end': void
|
||||
// Emitted when a complete speech segment is ready for transcription
|
||||
'speech-ready': { buffer: Float32Array, duration: number }
|
||||
// Emitted for status updates and errors
|
||||
'status': { type: string, message: string }
|
||||
// Debug info
|
||||
'debug': { message: string, data?: any }
|
||||
}
|
||||
|
||||
export type VADEventCallback<K extends keyof VADEvents> =
|
||||
(event: VADEvents[K]) => void
|
||||
|
||||
/**
|
||||
* Voice Activity Detection processor
|
||||
*/
|
||||
export class VAD {
|
||||
private config: VADConfig
|
||||
private model: any
|
||||
private state: Tensor
|
||||
private sampleRateTensor: Tensor
|
||||
private buffer: Float32Array
|
||||
private bufferPointer: number = 0
|
||||
private isRecording: boolean = false
|
||||
private postSpeechSamples: number = 0
|
||||
private prevBuffers: Float32Array[] = []
|
||||
private inferenceChain: Promise<any> = Promise.resolve()
|
||||
private eventListeners: Partial<Record<keyof VADEvents, VADEventCallback<any>[]>> = {}
|
||||
private isReady: boolean = false
|
||||
|
||||
constructor(userConfig: Partial<VADConfig> = {}) {
|
||||
// Default configuration
|
||||
const defaultConfig: VADConfig = {
|
||||
sampleRate: 16000,
|
||||
speechThreshold: 0.3,
|
||||
exitThreshold: 0.1,
|
||||
minSilenceDurationMs: 400,
|
||||
speechPadMs: 80,
|
||||
minSpeechDurationMs: 250,
|
||||
maxBufferDuration: 30,
|
||||
newBufferSize: 512,
|
||||
}
|
||||
|
||||
this.config = { ...defaultConfig, ...userConfig }
|
||||
|
||||
// Create buffer based on max duration
|
||||
this.buffer = new Float32Array(this.config.maxBufferDuration * this.config.sampleRate)
|
||||
|
||||
// Initialize state tensor for VAD model
|
||||
this.state = new Tensor('float32', new Float32Array(2 * 1 * 128), [2, 1, 128])
|
||||
|
||||
// Sample rate tensor for the model
|
||||
this.sampleRateTensor = new Tensor('int64', [this.config.sampleRate], [])
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize the VAD model
|
||||
*/
|
||||
public async initialize(): Promise<void> {
|
||||
try {
|
||||
this.emit('status', { type: 'info', message: 'Loading VAD model...' })
|
||||
|
||||
this.model = await AutoModel.from_pretrained('onnx-community/silero-vad', {
|
||||
config: { model_type: 'custom' } as any,
|
||||
dtype: 'fp32', // Full-precision
|
||||
})
|
||||
|
||||
this.isReady = true
|
||||
this.emit('status', { type: 'info', message: 'VAD model loaded successfully' })
|
||||
}
|
||||
catch (error) {
|
||||
this.emit('status', { type: 'error', message: `Failed to load VAD model: ${error}` })
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Add event listener
|
||||
*/
|
||||
public on<K extends keyof VADEvents>(event: K, callback: VADEventCallback<K>): void {
|
||||
if (!this.eventListeners[event]) {
|
||||
this.eventListeners[event] = []
|
||||
}
|
||||
this.eventListeners[event]!.push(callback as any)
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove event listener
|
||||
*/
|
||||
public off<K extends keyof VADEvents>(event: K, callback: VADEventCallback<K>): void {
|
||||
if (!this.eventListeners[event])
|
||||
return
|
||||
this.eventListeners[event] = this.eventListeners[event]!.filter(cb => cb !== callback)
|
||||
}
|
||||
|
||||
/**
|
||||
* Emit event
|
||||
*/
|
||||
private emit<K extends keyof VADEvents>(event: K, data: VADEvents[K]): void {
|
||||
if (!this.eventListeners[event])
|
||||
return
|
||||
for (const callback of this.eventListeners[event]!) {
|
||||
callback(data)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Process audio buffer for speech detection
|
||||
*/
|
||||
public async processAudio(inputBuffer: Float32Array): Promise<void> {
|
||||
if (!this.isReady) {
|
||||
throw new Error('VAD model is not initialized. Call initialize() first.')
|
||||
}
|
||||
|
||||
const wasRecording = this.isRecording
|
||||
|
||||
// Perform VAD on the input buffer
|
||||
const isSpeech = await this.detectSpeech(inputBuffer)
|
||||
|
||||
// Calculate derived constants
|
||||
const sampleRateMs = this.config.sampleRate / 1000
|
||||
const minSilenceDurationSamples = this.config.minSilenceDurationMs * sampleRateMs
|
||||
const speechPadSamples = this.config.speechPadMs * sampleRateMs
|
||||
const minSpeechDurationSamples = this.config.minSpeechDurationMs * sampleRateMs
|
||||
const maxPrevBuffers = Math.ceil(speechPadSamples / this.config.newBufferSize)
|
||||
|
||||
// If not currently in speech and the current buffer isn't speech,
|
||||
// store it in the previous buffers queue for potential padding
|
||||
if (!wasRecording && !isSpeech) {
|
||||
if (this.prevBuffers.length >= maxPrevBuffers) {
|
||||
this.prevBuffers.shift()
|
||||
}
|
||||
this.prevBuffers.push(inputBuffer.slice(0))
|
||||
return
|
||||
}
|
||||
|
||||
// Check if we need to handle buffer overflow
|
||||
const remaining = this.buffer.length - this.bufferPointer
|
||||
if (inputBuffer.length >= remaining) {
|
||||
// The buffer is full, process what we have
|
||||
this.buffer.set(inputBuffer.subarray(0, remaining), this.bufferPointer)
|
||||
this.bufferPointer += remaining
|
||||
|
||||
// Process and reset with overflow
|
||||
const overflow = inputBuffer.subarray(remaining)
|
||||
this.processSpeechSegment(overflow)
|
||||
return
|
||||
}
|
||||
else {
|
||||
// Add input to the buffer
|
||||
this.buffer.set(inputBuffer, this.bufferPointer)
|
||||
this.bufferPointer += inputBuffer.length
|
||||
}
|
||||
|
||||
// Handle speech detection
|
||||
if (isSpeech) {
|
||||
if (!this.isRecording) {
|
||||
// Speech just started
|
||||
this.emit('speech-start', undefined)
|
||||
this.emit('status', { type: 'info', message: 'Speech detected' })
|
||||
}
|
||||
|
||||
// Update state
|
||||
this.isRecording = true
|
||||
this.postSpeechSamples = 0
|
||||
return
|
||||
}
|
||||
|
||||
// At this point, we were recording but the current buffer is not speech
|
||||
this.postSpeechSamples += inputBuffer.length
|
||||
|
||||
// Check if silence is long enough to consider speech ended
|
||||
if (this.postSpeechSamples >= minSilenceDurationSamples) {
|
||||
// Check if the speech segment is long enough to process
|
||||
if (this.bufferPointer < minSpeechDurationSamples) {
|
||||
// Too short, reset without processing
|
||||
this.reset()
|
||||
return
|
||||
}
|
||||
|
||||
// Process the speech segment
|
||||
this.processSpeechSegment()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Detect speech in an audio buffer
|
||||
*/
|
||||
private async detectSpeech(buffer: Float32Array): Promise<boolean> {
|
||||
const input = new Tensor('float32', buffer, [1, buffer.length])
|
||||
|
||||
const { stateN, output } = await (this.inferenceChain = this.inferenceChain.then(() =>
|
||||
this.model({
|
||||
input,
|
||||
sr: this.sampleRateTensor,
|
||||
state: this.state,
|
||||
}),
|
||||
))
|
||||
|
||||
// Update the state
|
||||
this.state = stateN
|
||||
|
||||
// Get the speech probability
|
||||
const speechProb = output.data[0]
|
||||
|
||||
this.emit('debug', {
|
||||
message: 'VAD score',
|
||||
data: { probability: speechProb },
|
||||
})
|
||||
|
||||
// Apply thresholds
|
||||
return (
|
||||
speechProb > this.config.speechThreshold
|
||||
|| (this.isRecording && speechProb >= this.config.exitThreshold)
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Process a complete speech segment
|
||||
*/
|
||||
private processSpeechSegment(overflow?: Float32Array): void {
|
||||
const sampleRateMs = this.config.sampleRate / 1000
|
||||
const speechPadSamples = this.config.speechPadMs * sampleRateMs
|
||||
|
||||
// Calculate duration info
|
||||
const duration = (this.bufferPointer / this.config.sampleRate) * 1000
|
||||
const overflowLength = overflow?.length ?? 0
|
||||
|
||||
// Create the final buffer with padding
|
||||
const prevLength = this.prevBuffers.reduce((acc, b) => acc + b.length, 0)
|
||||
const finalBuffer = new Float32Array(prevLength + this.bufferPointer + speechPadSamples)
|
||||
|
||||
// Add previous buffers for pre-speech padding
|
||||
let offset = 0
|
||||
for (const prev of this.prevBuffers) {
|
||||
finalBuffer.set(prev, offset)
|
||||
offset += prev.length
|
||||
}
|
||||
|
||||
// Add the main speech segment
|
||||
finalBuffer.set(this.buffer.slice(0, this.bufferPointer + speechPadSamples), offset)
|
||||
|
||||
// Emit the speech segment
|
||||
this.emit('speech-end', undefined)
|
||||
this.emit('speech-ready', {
|
||||
buffer: finalBuffer,
|
||||
duration,
|
||||
})
|
||||
|
||||
// Reset for the next segment
|
||||
if (overflow) {
|
||||
this.buffer.set(overflow, 0)
|
||||
}
|
||||
this.reset(overflowLength)
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset the VAD state
|
||||
*/
|
||||
private reset(offset: number = 0): void {
|
||||
this.buffer.fill(0, offset)
|
||||
this.bufferPointer = offset
|
||||
this.isRecording = false
|
||||
this.postSpeechSamples = 0
|
||||
this.prevBuffers = []
|
||||
}
|
||||
|
||||
/**
|
||||
* Update configuration
|
||||
*/
|
||||
public updateConfig(newConfig: Partial<VADConfig>): void {
|
||||
this.config = { ...this.config, ...newConfig }
|
||||
|
||||
// If buffer size changed, create a new buffer
|
||||
if (newConfig.maxBufferDuration || newConfig.sampleRate) {
|
||||
this.buffer = new Float32Array(this.config.maxBufferDuration * this.config.sampleRate)
|
||||
this.bufferPointer = 0
|
||||
}
|
||||
|
||||
// Update sample rate tensor if needed
|
||||
if (newConfig.sampleRate) {
|
||||
this.sampleRateTensor = new Tensor('int64', [this.config.sampleRate], [])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a VAD processor with the given configuration
|
||||
*/
|
||||
export async function createVAD(config?: Partial<VADConfig>): Promise<VAD> {
|
||||
const vad = new VAD(config)
|
||||
await vad.initialize()
|
||||
return vad
|
||||
}
|
||||
@@ -1,43 +0,0 @@
|
||||
function writeString(dataView: DataView, offset: number, string: string) {
|
||||
for (let i = 0; i < string.length; i++) {
|
||||
dataView.setUint8(offset + i, string.charCodeAt(i))
|
||||
}
|
||||
}
|
||||
|
||||
export function toWav(buffer: Float32Array, sampleRate: number) {
|
||||
const numChannels = 1
|
||||
const numSamples = buffer.length
|
||||
|
||||
// Create the WAV file container
|
||||
const arrayBuffer = new ArrayBuffer(44 + numSamples * 2)
|
||||
const dataView = new DataView(arrayBuffer)
|
||||
|
||||
// RIFF chunk descriptor
|
||||
writeString(dataView, 0, 'RIFF')
|
||||
dataView.setUint32(4, 36 + numSamples * 2, true)
|
||||
writeString(dataView, 8, 'WAVE')
|
||||
|
||||
// fmt sub-chunk
|
||||
writeString(dataView, 12, 'fmt ')
|
||||
dataView.setUint32(16, 16, true)
|
||||
dataView.setUint16(20, 1, true) // PCM format
|
||||
dataView.setUint16(22, numChannels, true)
|
||||
dataView.setUint32(24, sampleRate, true)
|
||||
dataView.setUint32(28, sampleRate * numChannels * 2, true) // byte rate
|
||||
dataView.setUint16(32, numChannels * 2, true) // block align
|
||||
dataView.setUint16(34, 16, true) // bits per sample
|
||||
|
||||
// data sub-chunk
|
||||
writeString(dataView, 36, 'data')
|
||||
dataView.setUint32(40, numSamples * 2, true)
|
||||
|
||||
// Write the PCM samples
|
||||
const offset = 44
|
||||
for (let i = 0; i < numSamples; i++) {
|
||||
const sample = Math.max(-1, Math.min(1, buffer[i]))
|
||||
const value = sample < 0 ? sample * 0x8000 : sample * 0x7FFF
|
||||
dataView.setInt16(offset + i * 2, value, true)
|
||||
}
|
||||
|
||||
return arrayBuffer
|
||||
}
|
||||
@@ -1,13 +0,0 @@
|
||||
import { createApp } from 'vue'
|
||||
import { createRouter, createWebHashHistory } from 'vue-router'
|
||||
import { routes } from 'vue-router/auto-routes'
|
||||
|
||||
import App from './App.vue'
|
||||
import '@unocss/reset/tailwind.css'
|
||||
import 'uno.css'
|
||||
|
||||
const router = createRouter({ routes, history: createWebHashHistory() })
|
||||
|
||||
createApp(App)
|
||||
.use(router)
|
||||
.mount('#app')
|
||||
@@ -1,169 +0,0 @@
|
||||
<script setup lang="ts">
|
||||
import { ref } from 'vue'
|
||||
|
||||
import { VADAudioManager } from '../libs/vad/manager'
|
||||
import workletUrl from '../libs/vad/process.worklet?url'
|
||||
import { createVAD } from '../libs/vad/vad'
|
||||
import { toWav } from '../libs/vad/wav'
|
||||
|
||||
interface AudioSegment {
|
||||
buffer: Float32Array
|
||||
duration: number
|
||||
timestamp: number
|
||||
audioSrc: string
|
||||
}
|
||||
|
||||
const audioManager = ref<VADAudioManager>()
|
||||
const isInitialized = ref(false)
|
||||
const isRunning = ref(false)
|
||||
const isSpeechDetected = ref(false)
|
||||
const segments = ref<AudioSegment[]>([])
|
||||
const error = ref<string | null>(null)
|
||||
const isModuleLoading = ref(false)
|
||||
|
||||
async function setupSpeechDetection() {
|
||||
try {
|
||||
isModuleLoading.value = true
|
||||
|
||||
// Create and initialize the VAD
|
||||
const vad = await createVAD({
|
||||
sampleRate: 16000,
|
||||
speechThreshold: 0.3,
|
||||
exitThreshold: 0.1,
|
||||
minSilenceDurationMs: 400,
|
||||
})
|
||||
|
||||
// Set up event handlers
|
||||
vad.on('speech-start', () => {
|
||||
isSpeechDetected.value = true
|
||||
})
|
||||
|
||||
vad.on('speech-end', () => {
|
||||
isSpeechDetected.value = false
|
||||
})
|
||||
|
||||
vad.on('speech-ready', async ({ buffer, duration }) => {
|
||||
const wavBuffer = toWav(buffer, 16000)
|
||||
const audioBlob = new Blob([wavBuffer], { type: 'audio/wav' })
|
||||
|
||||
// Store the segment
|
||||
segments.value.push({
|
||||
buffer,
|
||||
duration: duration / 1000, // Convert to seconds for display
|
||||
timestamp: Date.now(),
|
||||
audioSrc: URL.createObjectURL(audioBlob),
|
||||
})
|
||||
})
|
||||
|
||||
vad.on('status', ({ type, message }) => {
|
||||
if (type === 'error') {
|
||||
error.value = message
|
||||
}
|
||||
})
|
||||
|
||||
// Create and initialize audio manager
|
||||
const m = new VADAudioManager(vad, {
|
||||
minChunkSize: 512,
|
||||
audioContextOptions: {
|
||||
sampleRate: 16000,
|
||||
latencyHint: 'interactive',
|
||||
},
|
||||
})
|
||||
|
||||
await m.initialize(workletUrl)
|
||||
audioManager.value = m
|
||||
isInitialized.value = true
|
||||
isModuleLoading.value = false
|
||||
startVad()
|
||||
}
|
||||
catch (err) {
|
||||
console.error('Setup failed:', err)
|
||||
error.value = err instanceof Error ? err.message : String(err)
|
||||
isModuleLoading.value = false
|
||||
}
|
||||
}
|
||||
|
||||
async function destroySpeechDetection() {
|
||||
await audioManager.value?.stop()
|
||||
await audioManager.value?.dispose()
|
||||
isInitialized.value = false
|
||||
isRunning.value = false
|
||||
isSpeechDetected.value = false
|
||||
|
||||
for (const segment of segments.value) {
|
||||
URL.revokeObjectURL(segment.audioSrc)
|
||||
}
|
||||
|
||||
segments.value = []
|
||||
error.value = null
|
||||
isModuleLoading.value = false
|
||||
}
|
||||
|
||||
async function startVad() {
|
||||
try {
|
||||
await audioManager.value?.startMicrophone()
|
||||
isRunning.value = true
|
||||
error.value = null
|
||||
}
|
||||
catch (err) {
|
||||
console.error('Failed to start microphone:', err)
|
||||
error.value = err instanceof Error ? err.message : String(err)
|
||||
}
|
||||
}
|
||||
|
||||
async function stopVad() {
|
||||
await audioManager.value?.stopMicrophone()
|
||||
isRunning.value = false
|
||||
isSpeechDetected.value = false
|
||||
}
|
||||
|
||||
function toggleListening() {
|
||||
if (isRunning.value) {
|
||||
stopVad()
|
||||
}
|
||||
else {
|
||||
startVad()
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<div mb-6 h-full w-full flex flex-col gap-2>
|
||||
<div w-full flex-1>
|
||||
<div v-if="isModuleLoading" mt-20 flex items-center justify-center text-5xl>
|
||||
<div i-svg-spinners:3-dots-move />
|
||||
</div>
|
||||
|
||||
<div v-if="error" class="error">
|
||||
{{ error }}
|
||||
</div>
|
||||
|
||||
<div v-if="segments?.length && segments.length > 0" class="segments" w-full flex flex-col gap-2>
|
||||
<h3>Voice Segments ({{ segments.length }})</h3>
|
||||
<ul>
|
||||
<li v-for="(segment, index) in segments" :key="index" class="segment" flex flex-col gap-2>
|
||||
<div class="segment-info">
|
||||
<span>Duration: {{ segment.duration.toFixed(2) }}s</span>
|
||||
</div>
|
||||
<audio :src="segment.audioSrc" controls w-full />
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div w-full flex justify-center gap-4>
|
||||
<button aspect-square size-15 flex items-center justify-center rounded-full text-2xl :class="[isRunning ? 'bg-neutral-900 dark:bg-white text-white dark:text-neutral-900' : 'bg-neutral-900 dark:bg-white/20 text-white dark:text-white', isInitialized ? 'opacity-100' : 'opacity-0']" :disabled="!isInitialized" @click="toggleListening">
|
||||
<div i-solar:microphone-3-bold />
|
||||
</button>
|
||||
<button v-if="!isInitialized" bg="green-500 dark:green-500 hover:green-400 dark:hover:green-400 active:green-500 dark:active:green-500" transition="all duration-250 ease-in-out" aspect-square size-15 flex items-center justify-center rounded-full @click="setupSpeechDetection">
|
||||
<div i-solar:phone-rounded-bold text-4xl text-white />
|
||||
</button>
|
||||
<button v-else bg="red-500 dark:red-500 hover:red-400 dark:hover:red-400 active:red-500 dark:active:red-500" transition="all duration-250 ease-in-out" aspect-square size-15 flex items-center justify-center rounded-full text-4xl text-white @click="destroySpeechDetection">
|
||||
<div i-solar:end-call-rounded-bold />
|
||||
</button>
|
||||
<button aspect-square size-15 flex items-center justify-center rounded-full text-2xl class="bg-neutral-900 text-white dark:bg-neutral-900 dark:bg-white/20 dark:text-white" :class="isInitialized ? 'opacity-100' : 'opacity-0'" :disabled="!isInitialized">
|
||||
<div i-solar:headphones-round-bold />
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
@@ -1,41 +0,0 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ESNext",
|
||||
"jsx": "preserve",
|
||||
"lib": [
|
||||
"DOM",
|
||||
"ESNext",
|
||||
"WebWorker"
|
||||
],
|
||||
"module": "ESNext",
|
||||
"moduleResolution": "Bundler",
|
||||
"resolveJsonModule": true,
|
||||
"types": [
|
||||
"vitest",
|
||||
"vite/client",
|
||||
// Currently AudioWorkletProcessor type is missing, we need to add it manually through @types/audioworklet
|
||||
// https://github.com/microsoft/TypeScript/issues/28308#issuecomment-1512509870
|
||||
"@types/audioworklet",
|
||||
"unplugin-vue-router/client"
|
||||
],
|
||||
"allowJs": true,
|
||||
"strict": true,
|
||||
"strictNullChecks": true,
|
||||
"noUnusedLocals": true,
|
||||
"noEmit": true,
|
||||
"esModuleInterop": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"isolatedModules": true,
|
||||
"skipLibCheck": true
|
||||
},
|
||||
"include": [
|
||||
"src/**/*.ts",
|
||||
"src/**/*.tsx",
|
||||
"src/**/*.vue",
|
||||
"src/**/*.js"
|
||||
],
|
||||
"exclude": [
|
||||
"dist",
|
||||
"node_modules"
|
||||
]
|
||||
}
|
||||
@@ -1,37 +0,0 @@
|
||||
import {
|
||||
defineConfig,
|
||||
presetAttributify,
|
||||
presetIcons,
|
||||
presetTypography,
|
||||
presetWebFonts,
|
||||
presetWind3,
|
||||
transformerDirectives,
|
||||
transformerVariantGroup,
|
||||
} from 'unocss'
|
||||
|
||||
export default defineConfig({
|
||||
presets: [
|
||||
presetWind3(),
|
||||
presetAttributify(),
|
||||
presetTypography(),
|
||||
presetWebFonts({
|
||||
fonts: {
|
||||
sans: 'DM Sans',
|
||||
serif: 'DM Serif Display',
|
||||
mono: 'DM Mono',
|
||||
},
|
||||
timeouts: {
|
||||
warning: 5000,
|
||||
failure: 10000,
|
||||
},
|
||||
}),
|
||||
presetIcons({
|
||||
scale: 1.2,
|
||||
}),
|
||||
],
|
||||
transformers: [
|
||||
transformerDirectives(),
|
||||
transformerVariantGroup(),
|
||||
],
|
||||
safelist: 'prose prose-sm m-auto text-left'.split(' '),
|
||||
})
|
||||
@@ -1,19 +0,0 @@
|
||||
import { resolve } from 'node:path'
|
||||
import Vue from '@vitejs/plugin-vue'
|
||||
import Unocss from 'unocss/vite'
|
||||
import VueRouter from 'unplugin-vue-router/vite'
|
||||
import { defineConfig } from 'vite'
|
||||
|
||||
export default defineConfig({
|
||||
plugins: [
|
||||
// https://github.com/posva/unplugin-vue-router
|
||||
VueRouter({
|
||||
extensions: ['.vue', '.md'],
|
||||
dts: resolve(import.meta.dirname, 'src', 'typed-router.d.ts'),
|
||||
}),
|
||||
Vue(),
|
||||
// https://github.com/antfu/unocss
|
||||
// see uno.config.ts for config
|
||||
Unocss(),
|
||||
],
|
||||
})
|
||||
Generated
+5
-191
@@ -524,7 +524,7 @@ importers:
|
||||
version: 28.4.1(@babel/parser@7.26.10)(@nuxt/kit@3.14.1592(magicast@0.3.5)(rollup@2.79.1))(vue@3.5.13(typescript@5.8.3))
|
||||
unplugin-vue-macros:
|
||||
specifier: ^2.14.5
|
||||
version: 2.14.5(@vueuse/core@13.0.0(vue@3.5.13(typescript@5.8.3)))(esbuild@0.25.0)(rollup@2.79.1)(typescript@5.8.3)(vite@6.2.5(@types/node@22.14.0)(jiti@2.4.2)(less@4.3.0)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0))(vue-tsc@3.0.0-alpha.2(typescript@5.8.3))(vue@3.5.13(typescript@5.8.3))
|
||||
version: 2.14.5(@vueuse/core@13.0.0(vue@3.5.13(typescript@5.8.3)))(esbuild@0.19.12)(rollup@2.79.1)(typescript@5.8.3)(vite@6.2.5(@types/node@22.14.0)(jiti@2.4.2)(less@4.3.0)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0))(vue-tsc@3.0.0-alpha.2(typescript@5.8.3))(vue@3.5.13(typescript@5.8.3))
|
||||
unplugin-vue-markdown:
|
||||
specifier: ^28.3.1
|
||||
version: 28.3.1(vite@6.2.5(@types/node@22.14.0)(jiti@2.4.2)(less@4.3.0)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0))
|
||||
@@ -887,162 +887,6 @@ importers:
|
||||
specifier: ^3.0.0-alpha.2
|
||||
version: 3.0.0-alpha.2(typescript@5.8.3)
|
||||
|
||||
apps/vad:
|
||||
dependencies:
|
||||
'@huggingface/transformers':
|
||||
specifier: ^3.4.2
|
||||
version: 3.4.2
|
||||
'@vueuse/core':
|
||||
specifier: ^13.0.0
|
||||
version: 13.0.0(vue@3.5.13(typescript@5.8.3))
|
||||
defu:
|
||||
specifier: ^6.1.4
|
||||
version: 6.1.4
|
||||
es-toolkit:
|
||||
specifier: ^1.34.1
|
||||
version: 1.34.1
|
||||
vue:
|
||||
specifier: ^3.5.13
|
||||
version: 3.5.13(typescript@5.8.3)
|
||||
devDependencies:
|
||||
'@iconify-json/solar':
|
||||
specifier: ^1.2.2
|
||||
version: 1.2.2
|
||||
'@iconify-json/svg-spinners':
|
||||
specifier: ^1.2.2
|
||||
version: 1.2.2
|
||||
'@types/audioworklet':
|
||||
specifier: ^0.0.72
|
||||
version: 0.0.72
|
||||
'@unocss/reset':
|
||||
specifier: ^66.1.0-beta.10
|
||||
version: 66.1.0-beta.10
|
||||
'@vitejs/plugin-vue':
|
||||
specifier: ^5.2.3
|
||||
version: 5.2.3(vite@6.2.5(@types/node@22.14.0)(jiti@2.4.2)(less@4.3.0)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0))(vue@3.5.13(typescript@5.8.3))
|
||||
unplugin-vue-router:
|
||||
specifier: ^0.12.0
|
||||
version: 0.12.0(vue-router@4.5.0(vue@3.5.13(typescript@5.8.3)))(vue@3.5.13(typescript@5.8.3))
|
||||
vite:
|
||||
specifier: ^6.2.5
|
||||
version: 6.2.5(@types/node@22.14.0)(jiti@2.4.2)(less@4.3.0)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0)
|
||||
vue-router:
|
||||
specifier: ^4.5.0
|
||||
version: 4.5.0(vue@3.5.13(typescript@5.8.3))
|
||||
vue-tsc:
|
||||
specifier: ^3.0.0-alpha.2
|
||||
version: 3.0.0-alpha.2(typescript@5.8.3)
|
||||
|
||||
apps/vad-asr:
|
||||
dependencies:
|
||||
'@huggingface/transformers':
|
||||
specifier: ^3.4.2
|
||||
version: 3.4.2
|
||||
'@vueuse/core':
|
||||
specifier: ^13.0.0
|
||||
version: 13.0.0(vue@3.5.13(typescript@5.8.3))
|
||||
'@xsai/generate-transcription':
|
||||
specifier: 'catalog:'
|
||||
version: 0.2.0-beta.3
|
||||
defu:
|
||||
specifier: ^6.1.4
|
||||
version: 6.1.4
|
||||
es-toolkit:
|
||||
specifier: ^1.34.1
|
||||
version: 1.34.1
|
||||
vue:
|
||||
specifier: ^3.5.13
|
||||
version: 3.5.13(typescript@5.8.3)
|
||||
devDependencies:
|
||||
'@iconify-json/solar':
|
||||
specifier: ^1.2.2
|
||||
version: 1.2.2
|
||||
'@iconify-json/svg-spinners':
|
||||
specifier: ^1.2.2
|
||||
version: 1.2.2
|
||||
'@types/audioworklet':
|
||||
specifier: ^0.0.72
|
||||
version: 0.0.72
|
||||
'@unocss/reset':
|
||||
specifier: ^66.1.0-beta.10
|
||||
version: 66.1.0-beta.10
|
||||
'@vitejs/plugin-vue':
|
||||
specifier: ^5.2.3
|
||||
version: 5.2.3(vite@6.2.5(@types/node@22.14.0)(jiti@2.4.2)(less@4.3.0)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0))(vue@3.5.13(typescript@5.8.3))
|
||||
unplugin-vue-router:
|
||||
specifier: ^0.12.0
|
||||
version: 0.12.0(vue-router@4.5.0(vue@3.5.13(typescript@5.8.3)))(vue@3.5.13(typescript@5.8.3))
|
||||
vite:
|
||||
specifier: ^6.2.5
|
||||
version: 6.2.5(@types/node@22.14.0)(jiti@2.4.2)(less@4.3.0)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0)
|
||||
vue-router:
|
||||
specifier: ^4.5.0
|
||||
version: 4.5.0(vue@3.5.13(typescript@5.8.3))
|
||||
vue-tsc:
|
||||
specifier: ^3.0.0-alpha.2
|
||||
version: 3.0.0-alpha.2(typescript@5.8.3)
|
||||
|
||||
apps/vad-asr-chat:
|
||||
dependencies:
|
||||
'@huggingface/transformers':
|
||||
specifier: ^3.4.2
|
||||
version: 3.4.2
|
||||
'@llama-flow/core':
|
||||
specifier: ^0.3.4
|
||||
version: 0.3.4(@modelcontextprotocol/sdk@1.8.0)(zod@3.24.2)
|
||||
'@vueuse/core':
|
||||
specifier: ^13.0.0
|
||||
version: 13.0.0(vue@3.5.13(typescript@5.8.3))
|
||||
'@xsai/generate-transcription':
|
||||
specifier: 'catalog:'
|
||||
version: 0.2.0-beta.3
|
||||
'@xsai/shared':
|
||||
specifier: 'catalog:'
|
||||
version: 0.2.0-beta.3
|
||||
'@xsai/shared-chat':
|
||||
specifier: 'catalog:'
|
||||
version: 0.2.0-beta.3
|
||||
'@xsai/stream-text':
|
||||
specifier: 'catalog:'
|
||||
version: 0.2.0-beta.3
|
||||
defu:
|
||||
specifier: ^6.1.4
|
||||
version: 6.1.4
|
||||
es-toolkit:
|
||||
specifier: ^1.34.1
|
||||
version: 1.34.1
|
||||
vue:
|
||||
specifier: ^3.5.13
|
||||
version: 3.5.13(typescript@5.8.3)
|
||||
devDependencies:
|
||||
'@iconify-json/solar':
|
||||
specifier: ^1.2.2
|
||||
version: 1.2.2
|
||||
'@iconify-json/svg-spinners':
|
||||
specifier: ^1.2.2
|
||||
version: 1.2.2
|
||||
'@types/audioworklet':
|
||||
specifier: ^0.0.72
|
||||
version: 0.0.72
|
||||
'@unocss/reset':
|
||||
specifier: ^66.1.0-beta.10
|
||||
version: 66.1.0-beta.10
|
||||
'@vitejs/plugin-vue':
|
||||
specifier: ^5.2.3
|
||||
version: 5.2.3(vite@6.2.5(@types/node@22.14.0)(jiti@2.4.2)(less@4.3.0)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0))(vue@3.5.13(typescript@5.8.3))
|
||||
unplugin-vue-router:
|
||||
specifier: ^0.12.0
|
||||
version: 0.12.0(vue-router@4.5.0(vue@3.5.13(typescript@5.8.3)))(vue@3.5.13(typescript@5.8.3))
|
||||
vite:
|
||||
specifier: ^6.2.5
|
||||
version: 6.2.5(@types/node@22.14.0)(jiti@2.4.2)(less@4.3.0)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0)
|
||||
vue-router:
|
||||
specifier: ^4.5.0
|
||||
version: 4.5.0(vue@3.5.13(typescript@5.8.3))
|
||||
vue-tsc:
|
||||
specifier: ^3.0.0-alpha.2
|
||||
version: 3.0.0-alpha.2(typescript@5.8.3)
|
||||
|
||||
docs:
|
||||
devDependencies:
|
||||
98.css:
|
||||
@@ -3765,26 +3609,6 @@ packages:
|
||||
'@lezer/lr@1.4.2':
|
||||
resolution: {integrity: sha512-pu0K1jCIdnQ12aWNaAVU5bzi7Bd1w54J3ECgANPmYLtQKP0HBj2cE/5coBD66MT10xbtIuUr7tg0Shbsvk0mDA==}
|
||||
|
||||
'@llama-flow/core@0.3.4':
|
||||
resolution: {integrity: sha512-BOe23pfm7j9hKMH7u0jFS8bPKLsShKgz8KdN/rXeicgnopCwhDMO4ppOg7Cy7tWap8kYZIY8ZliN7Q9SmNfjkg==}
|
||||
peerDependencies:
|
||||
'@modelcontextprotocol/sdk': ^1.7.0
|
||||
hono: ^4.7.4
|
||||
next: ^15.2.2
|
||||
p-retry: ^6.2.1
|
||||
zod: ^3.24.2
|
||||
peerDependenciesMeta:
|
||||
'@modelcontextprotocol/sdk':
|
||||
optional: true
|
||||
hono:
|
||||
optional: true
|
||||
next:
|
||||
optional: true
|
||||
p-retry:
|
||||
optional: true
|
||||
zod:
|
||||
optional: true
|
||||
|
||||
'@marijn/find-cluster-break@1.0.2':
|
||||
resolution: {integrity: sha512-l0h88YhZFyKdXIFNfSWpyjStDjGHwZ/U7iobcK1cQQD8sejsONdQtTVU+1wVN1PBw40PiiHB1vA5S7VTfQiP9g==}
|
||||
|
||||
@@ -4909,9 +4733,6 @@ packages:
|
||||
'@types/acorn@4.0.6':
|
||||
resolution: {integrity: sha512-veQTnWP+1D/xbxVrPC3zHnCZRjSrKfhbMUlEA43iMZLu7EsnTtkJklIuwrCPbOi8YkvDQAiW05VQQFvvz9oieQ==}
|
||||
|
||||
'@types/audioworklet@0.0.72':
|
||||
resolution: {integrity: sha512-qrsjMZxB0k5DJSLpcDXlX/OYa5iDEAc9dgEwCZO1op5jka7Jgc/fMEGq8LuHO+8MmpOInqco6hET1gawCTVeEg==}
|
||||
|
||||
'@types/babel__core@7.20.5':
|
||||
resolution: {integrity: sha512-qoQprZvz5wQFJwMDqeseRXWv3rqMvhgpbXFfVyWhbx9X47POIA6i/+dXefEmZKoAgOaTdaIgNSMqMIU61yRyzA==}
|
||||
|
||||
@@ -14331,11 +14152,6 @@ snapshots:
|
||||
dependencies:
|
||||
'@lezer/common': 1.2.3
|
||||
|
||||
'@llama-flow/core@0.3.4(@modelcontextprotocol/sdk@1.8.0)(zod@3.24.2)':
|
||||
optionalDependencies:
|
||||
'@modelcontextprotocol/sdk': 1.8.0
|
||||
zod: 3.24.2
|
||||
|
||||
'@marijn/find-cluster-break@1.0.2': {}
|
||||
|
||||
'@mdit-vue/plugin-component@2.1.3':
|
||||
@@ -15571,8 +15387,6 @@ snapshots:
|
||||
dependencies:
|
||||
'@types/estree': 1.0.7
|
||||
|
||||
'@types/audioworklet@0.0.72': {}
|
||||
|
||||
'@types/babel__core@7.20.5':
|
||||
dependencies:
|
||||
'@babel/parser': 7.26.10
|
||||
@@ -23856,9 +23670,9 @@ snapshots:
|
||||
'@nuxt/kit': 3.14.1592(magicast@0.3.5)(rollup@4.39.0)
|
||||
'@vueuse/core': 13.0.0(vue@3.5.13(typescript@5.8.3))
|
||||
|
||||
unplugin-combine@1.2.1(esbuild@0.25.0)(rollup@2.79.1)(unplugin@1.16.1)(vite@6.2.5(@types/node@22.14.0)(jiti@2.4.2)(less@4.3.0)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0)):
|
||||
unplugin-combine@1.2.1(esbuild@0.19.12)(rollup@2.79.1)(unplugin@1.16.1)(vite@6.2.5(@types/node@22.14.0)(jiti@2.4.2)(less@4.3.0)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0)):
|
||||
optionalDependencies:
|
||||
esbuild: 0.25.0
|
||||
esbuild: 0.19.12
|
||||
rollup: 2.79.1
|
||||
unplugin: 1.16.1
|
||||
vite: 6.2.5(@types/node@22.14.0)(jiti@2.4.2)(less@4.3.0)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0)
|
||||
@@ -23917,7 +23731,7 @@ snapshots:
|
||||
transitivePeerDependencies:
|
||||
- vue
|
||||
|
||||
unplugin-vue-macros@2.14.5(@vueuse/core@13.0.0(vue@3.5.13(typescript@5.8.3)))(esbuild@0.25.0)(rollup@2.79.1)(typescript@5.8.3)(vite@6.2.5(@types/node@22.14.0)(jiti@2.4.2)(less@4.3.0)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0))(vue-tsc@3.0.0-alpha.2(typescript@5.8.3))(vue@3.5.13(typescript@5.8.3)):
|
||||
unplugin-vue-macros@2.14.5(@vueuse/core@13.0.0(vue@3.5.13(typescript@5.8.3)))(esbuild@0.19.12)(rollup@2.79.1)(typescript@5.8.3)(vite@6.2.5(@types/node@22.14.0)(jiti@2.4.2)(less@4.3.0)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0))(vue-tsc@3.0.0-alpha.2(typescript@5.8.3))(vue@3.5.13(typescript@5.8.3)):
|
||||
dependencies:
|
||||
'@vue-macros/better-define': 1.11.4(vue@3.5.13(typescript@5.8.3))
|
||||
'@vue-macros/boolean-prop': 0.5.5(vue@3.5.13(typescript@5.8.3))
|
||||
@@ -23949,7 +23763,7 @@ snapshots:
|
||||
'@vue-macros/short-vmodel': 1.5.5(vue@3.5.13(typescript@5.8.3))
|
||||
'@vue-macros/volar': 0.30.15(typescript@5.8.3)(vue-tsc@3.0.0-alpha.2(typescript@5.8.3))(vue@3.5.13(typescript@5.8.3))
|
||||
unplugin: 1.16.1
|
||||
unplugin-combine: 1.2.1(esbuild@0.25.0)(rollup@2.79.1)(unplugin@1.16.1)(vite@6.2.5(@types/node@22.14.0)(jiti@2.4.2)(less@4.3.0)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0))
|
||||
unplugin-combine: 1.2.1(esbuild@0.19.12)(rollup@2.79.1)(unplugin@1.16.1)(vite@6.2.5(@types/node@22.14.0)(jiti@2.4.2)(less@4.3.0)(terser@5.17.6)(tsx@4.19.3)(yaml@2.7.0))
|
||||
unplugin-vue-define-options: 1.5.5(vue@3.5.13(typescript@5.8.3))
|
||||
vue: 3.5.13(typescript@5.8.3)
|
||||
transitivePeerDependencies:
|
||||
|
||||
Reference in New Issue
Block a user