nanakura/moeka-project
1
0
Fork 0
Code Issues Pull Requests 1 Actions 2 Packages Projects Releases Wiki Activity

refacor(audio): unified audio package

Browse Source
This commit is contained in:
Neko Ayaka
2025-07-03 14:07:37 +08:00
parent 4d40e6fe2c
commit 28222603c7
12 changed files with 32 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/stage-web/src/components/Layouts/InteractiveArea.vue
+4 -5
View File
@@ -3,6 +3,7 @@ import type { ChatProvider } from '@xsai-ext/shared-providers'
import WhisperWorker from '@proj-airi/stage-ui/libs/workers/worker?worker&url'
import { toWAVBase64 } from '@proj-airi/audio'
import { useMicVAD, useWhisper } from '@proj-airi/stage-ui/composables'
import { useAudioContext, useChatStore, useConsciousnessStore, useProvidersStore, useSettings } from '@proj-airi/stage-ui/stores'
import { BasicTextarea, TransitionVertical } from '@proj-airi/ui'
@@ -13,8 +14,6 @@ import { useI18n } from 'vue-i18n'
import ChatHistory from '../Widgets/ChatHistory.vue'
import { encodeWAVToBase64 } from '../../utils/binary'
const messageInput = ref('')
const listening = ref(false)
const tab = ref<'chat' | 'custom' | 'clothes'>('chat')
@@ -92,16 +91,16 @@ const { destroy, start } = useMicVAD(selectedAudioDeviceId, {
onSpeechEnd: (buffer) => {
// TODO: do audio buffer send to whisper
listening.value = false
handleTranscription(buffer)
handleTranscription(buffer.buffer)
},
auto: false,
})
async function handleTranscription(buffer: Float32Array) {
async function handleTranscription(buffer: ArrayBufferLike) {
await audioContext.resume()
// Convert Float32Array to WAV format
const audioBase64 = await encodeWAVToBase64(buffer, audioContext.sampleRate)
const audioBase64 = await toWAVBase64(buffer, audioContext.sampleRate)
generate({ type: 'generate', data: { audio: audioBase64, language: 'en' } })
}
apps/stage-web/src/utils/binary.ts
-47
View File
@@ -1,47 +0,0 @@
export async function encodeWAVToBase64(samples: Float32Array, sampleRate: number): Promise<string> {
const buffer = new ArrayBuffer(44 + samples.length * 2)
const view = new DataView(buffer)
// Write WAV header
writeString(view, 0, 'RIFF')
view.setUint32(4, 36 + samples.length * 2, true)
writeString(view, 8, 'WAVE')
writeString(view, 12, 'fmt ')
view.setUint32(16, 16, true)
view.setUint16(20, 1, true)
view.setUint16(22, 1, true)
view.setUint32(24, sampleRate, true)
view.setUint32(28, sampleRate * 2, true)
view.setUint16(32, 2, true)
view.setUint16(34, 16, true)
writeString(view, 36, 'data')
view.setUint32(40, samples.length * 2, true)
// Write audio data
floatTo16BitPCM(view, 44, samples)
// Convert to base64 using Blob and FileReader
const blob = new Blob([view], { type: 'audio/wav' })
return new Promise((resolve, reject) => {
const reader = new FileReader()
reader.onload = () => {
const base64 = (reader.result as string).split(',')[1]
resolve(base64)
}
reader.onerror = reject
reader.readAsDataURL(blob)
})
}
function writeString(view: DataView, offset: number, string: string) {
for (let i = 0; i < string.length; i++) {
view.setUint8(offset + i, string.charCodeAt(i))
}
}
function floatTo16BitPCM(view: DataView, offset: number, input: Float32Array) {
for (let i = 0; i < input.length; i++, offset += 2) {
const s = Math.max(-1, Math.min(1, input[i]))
view.setInt16(offset, s < 0 ? s * 0x8000 : s * 0x7FFF, true)
}
}
apps/stage-web/src/utils/vad/index.ts
-1
View File
@@ -2,4 +2,3 @@ export { VADAudioManager } from './manager'
export type { VADAudioOptions } from './manager'
export { createVAD, VAD } from './vad'
export type { VADConfig, VADEventCallback, VADEvents } from './vad'
export { toWav } from './wav'
packages/audio/package.json
+5
View File
@@ -30,6 +30,10 @@
"./vue": {
"types": "./dist/vue/index.d.mts",
"default": "./dist/vue/index.mjs"
},
"./encoding": {
"types": "./dist/encoding/index.d.mts",
"default": "./dist/encoding/index.mjs"
}
},
"main": "./dist/index.mjs",
@@ -56,6 +60,7 @@
"@alexanderolsen/libsamplerate-js": "^2.1.2"
},
"devDependencies": {
"@moeru/std": "catalog:",
"@types/audioworklet": "^0.0.77",
"vite": "^7.0.0"
}
packages/audio/src/encoding/index.ts
+1
View File
@@ -0,0 +1 @@
export * from './wav'
apps/stage-web/src/utils/vad/wav.ts → packages/audio/src/encoding/wav.ts
+10 -3
View File
@@ -1,12 +1,14 @@
import { encodeBase64 } from '@moeru/std/base64'
function writeString(dataView: DataView, offset: number, string: string) {
for (let i = 0; i < string.length; i++) {
dataView.setUint8(offset + i, string.charCodeAt(i))
}
}
export function toWav(buffer: Float32Array, sampleRate: number) {
const numChannels = 1
const numSamples = buffer.length
export function toWav(buffer: ArrayBufferLike, sampleRate: number, channel = 1) {
const numChannels = channel
const numSamples = buffer.byteLength
// Create the WAV file container
const arrayBuffer = new ArrayBuffer(44 + numSamples * 2)
@@ -25,6 +27,7 @@ export function toWav(buffer: Float32Array, sampleRate: number) {
dataView.setUint32(24, sampleRate, true)
dataView.setUint32(28, sampleRate * numChannels * 2, true) // byte rate
dataView.setUint16(32, numChannels * 2, true) // block align
dataView.setUint16(34, 16, true) // bits per sample
// data sub-chunk
@@ -41,3 +44,7 @@ export function toWav(buffer: Float32Array, sampleRate: number) {
return arrayBuffer
}
export function toWAVBase64(buffer: ArrayBufferLike, sampleRate: number) {
return encodeBase64(toWav(buffer, sampleRate))
}
packages/audio/src/index.ts
+1
View File
@@ -0,0 +1 @@
export * from './encoding'
packages/audio/tsdown.config.ts
+1
View File
@@ -5,6 +5,7 @@ export default defineConfig({
'index': 'src/index.ts',
'audio-context/index': 'src/audio-context/index.ts',
'audio-context/processor.worklet': 'src/audio-context/processor.worklet.ts',
'encoding/index': 'src/encoding/index.ts',
'vue/index': 'src/vue/index.ts',
},
unbundle: true,
pnpm-lock.yaml
Generated
+6
View File
@@ -876,6 +876,9 @@ importers:
specifier: '>=3'
version: 3.5.17(typescript@5.8.3)
devDependencies:
'@moeru/std':
specifier: 'catalog:'
version: 0.1.0-beta.4
'@types/audioworklet':
specifier: ^0.0.77
version: 0.0.77
@@ -1311,6 +1314,9 @@ importers:
'@huggingface/transformers':
specifier: ^3.6.0
version: 3.6.0
'@proj-airi/audio':
specifier: workspace:^
version: link:../../packages/audio
'@proj-airi/server-sdk':
specifier: workspace:^
version: link:../../packages/server-sdk
services/discord-bot/package.json
+1
View File
@@ -23,6 +23,7 @@
"@dotenvx/dotenvx": "^1.45.1",
"@guiiai/logg": "^1.0.10",
"@huggingface/transformers": "^3.6.0",
"@proj-airi/audio": "workspace:^",
"@proj-airi/server-sdk": "workspace:^",
"@proj-airi/server-shared": "workspace:^",
"@xsai-ext/providers-cloud": "catalog:",
services/discord-bot/src/pipelines/tts.ts
+3 -4
View File
@@ -8,11 +8,10 @@ import wavefile from 'wavefile'
import { useLogg } from '@guiiai/logg'
import { pipeline } from '@huggingface/transformers'
import { toWav } from '@proj-airi/audio'
import { createOpenAI } from '@xsai-ext/providers-cloud'
import { generateTranscription } from '@xsai/generate-transcription'
import { pcmToWav } from '../utils/audio'
export class WhisperLargeV3Pipeline {
static task: PipelineType = 'automatic-speech-recognition'
static model = 'Xenova/whisper-medium.en'
@@ -49,13 +48,13 @@ export function textFromResult(result: Array<{ text: string }> | { text: string
export async function transcribe(pcmBuffer: Buffer) {
const log = useLogg('Memory:Transcribe').useGlobalConfig()
const pcmConvertedWav = pcmToWav(pcmBuffer, 48000, 2)
const pcmConvertedWav = toWav(pcmBuffer.buffer, 48000, 2)
log.withFields({ from: pcmBuffer.byteLength, to: pcmConvertedWav.byteLength }).log('Audio data received')
const transcriber = await WhisperLargeV3Pipeline.getInstance() as (audio: Float32Array | Float64Array) => Promise<Array<{ text: string }> | { text: string }>
log.log('Transcribing audio')
const wav = new wavefile.WaveFile(pcmConvertedWav)
const wav = new wavefile.WaveFile(new Uint8Array(pcmConvertedWav))
wav.toBitDepth('32f') // Pipeline expects input as a Float32Array
wav.toSampleRate(16000) // Whisper expects audio with a sampling rate of 16000
const audioData = wav.getSamples()
services/discord-bot/src/utils/audio.ts
-50
View File
@@ -2,56 +2,6 @@ import { Buffer } from 'node:buffer'
import { DECODE_SAMPLE_RATE } from '../constants/audio'
export function pcmToWav(pcmBuffer: Buffer, sampleRate: number, numChannels: number): Uint8Array {
const byteRate = sampleRate * numChannels * 2 // Assuming 16-bit PCM (2 bytes per sample)
const blockAlign = numChannels * 2 // Block align for 16-bit PCM
// Create WAV header
const header = new ArrayBuffer(44)
const view = new DataView(header)
// Write RIFF identifier
writeString(view, 0, 'RIFF')
// Write file length (size of data + header)
view.setUint32(4, 36 + pcmBuffer.byteLength, true)
// Write WAVE identifier
writeString(view, 8, 'WAVE')
// Write format chunk identifier
writeString(view, 12, 'fmt ')
// Write format chunk length (16 for PCM)
view.setUint32(16, 16, true)
// Write audio format (1 for PCM)
view.setUint16(20, 1, true)
// Write number of channels
view.setUint16(22, numChannels, true)
// Write sample rate
view.setUint32(24, sampleRate, true)
// Write byte rate
view.setUint32(28, byteRate, true)
// Write block align
view.setUint16(32, blockAlign, true)
// Write bits per sample (16)
view.setUint16(34, 16, true)
// Write data chunk identifier
writeString(view, 36, 'data')
// Write data chunk length (size of PCM data)
view.setUint32(40, pcmBuffer.byteLength, true)
// Combine header and PCM data into one buffer
const wavBuffer = new Uint8Array(header.byteLength + pcmBuffer.byteLength)
wavBuffer.set(new Uint8Array(header), 0)
wavBuffer.set(new Uint8Array(pcmBuffer), header.byteLength)
return wavBuffer
}
function writeString(view, offset, string) {
for (let i = 0; i < string.length; i++) {
view.setUint8(offset + i, string.charCodeAt(i))
}
}
export function getWavHeader(
audioLength: number,
sampleRate: number,