From 28222603c722b241826358cbbc35d59a6adb2091 Mon Sep 17 00:00:00 2001 From: Neko Ayaka Date: Thu, 3 Jul 2025 14:07:37 +0800 Subject: [PATCH] refacor(audio): unified audio package --- .../components/Layouts/InteractiveArea.vue | 9 ++-- apps/stage-web/src/utils/binary.ts | 47 ----------------- apps/stage-web/src/utils/vad/index.ts | 1 - packages/audio/package.json | 5 ++ packages/audio/src/encoding/index.ts | 1 + .../audio/src/encoding}/wav.ts | 13 +++-- packages/audio/src/index.ts | 1 + packages/audio/tsdown.config.ts | 1 + pnpm-lock.yaml | 6 +++ services/discord-bot/package.json | 1 + services/discord-bot/src/pipelines/tts.ts | 7 ++- services/discord-bot/src/utils/audio.ts | 50 ------------------- 12 files changed, 32 insertions(+), 110 deletions(-) delete mode 100644 apps/stage-web/src/utils/binary.ts create mode 100644 packages/audio/src/encoding/index.ts rename {apps/stage-web/src/utils/vad => packages/audio/src/encoding}/wav.ts (79%) diff --git a/apps/stage-web/src/components/Layouts/InteractiveArea.vue b/apps/stage-web/src/components/Layouts/InteractiveArea.vue index 96cfcebab..872140c61 100644 --- a/apps/stage-web/src/components/Layouts/InteractiveArea.vue +++ b/apps/stage-web/src/components/Layouts/InteractiveArea.vue @@ -3,6 +3,7 @@ import type { ChatProvider } from '@xsai-ext/shared-providers' import WhisperWorker from '@proj-airi/stage-ui/libs/workers/worker?worker&url' +import { toWAVBase64 } from '@proj-airi/audio' import { useMicVAD, useWhisper } from '@proj-airi/stage-ui/composables' import { useAudioContext, useChatStore, useConsciousnessStore, useProvidersStore, useSettings } from '@proj-airi/stage-ui/stores' import { BasicTextarea, TransitionVertical } from '@proj-airi/ui' @@ -13,8 +14,6 @@ import { useI18n } from 'vue-i18n' import ChatHistory from '../Widgets/ChatHistory.vue' -import { encodeWAVToBase64 } from '../../utils/binary' - const messageInput = ref('') const listening = ref(false) const tab = ref<'chat' | 'custom' | 'clothes'>('chat') @@ -92,16 +91,16 @@ const { destroy, start } = useMicVAD(selectedAudioDeviceId, { onSpeechEnd: (buffer) => { // TODO: do audio buffer send to whisper listening.value = false - handleTranscription(buffer) + handleTranscription(buffer.buffer) }, auto: false, }) -async function handleTranscription(buffer: Float32Array) { +async function handleTranscription(buffer: ArrayBufferLike) { await audioContext.resume() // Convert Float32Array to WAV format - const audioBase64 = await encodeWAVToBase64(buffer, audioContext.sampleRate) + const audioBase64 = await toWAVBase64(buffer, audioContext.sampleRate) generate({ type: 'generate', data: { audio: audioBase64, language: 'en' } }) } diff --git a/apps/stage-web/src/utils/binary.ts b/apps/stage-web/src/utils/binary.ts deleted file mode 100644 index 207c72f78..000000000 --- a/apps/stage-web/src/utils/binary.ts +++ /dev/null @@ -1,47 +0,0 @@ -export async function encodeWAVToBase64(samples: Float32Array, sampleRate: number): Promise { - const buffer = new ArrayBuffer(44 + samples.length * 2) - const view = new DataView(buffer) - - // Write WAV header - writeString(view, 0, 'RIFF') - view.setUint32(4, 36 + samples.length * 2, true) - writeString(view, 8, 'WAVE') - writeString(view, 12, 'fmt ') - view.setUint32(16, 16, true) - view.setUint16(20, 1, true) - view.setUint16(22, 1, true) - view.setUint32(24, sampleRate, true) - view.setUint32(28, sampleRate * 2, true) - view.setUint16(32, 2, true) - view.setUint16(34, 16, true) - writeString(view, 36, 'data') - view.setUint32(40, samples.length * 2, true) - - // Write audio data - floatTo16BitPCM(view, 44, samples) - - // Convert to base64 using Blob and FileReader - const blob = new Blob([view], { type: 'audio/wav' }) - return new Promise((resolve, reject) => { - const reader = new FileReader() - reader.onload = () => { - const base64 = (reader.result as string).split(',')[1] - resolve(base64) - } - reader.onerror = reject - reader.readAsDataURL(blob) - }) -} - -function writeString(view: DataView, offset: number, string: string) { - for (let i = 0; i < string.length; i++) { - view.setUint8(offset + i, string.charCodeAt(i)) - } -} - -function floatTo16BitPCM(view: DataView, offset: number, input: Float32Array) { - for (let i = 0; i < input.length; i++, offset += 2) { - const s = Math.max(-1, Math.min(1, input[i])) - view.setInt16(offset, s < 0 ? s * 0x8000 : s * 0x7FFF, true) - } -} diff --git a/apps/stage-web/src/utils/vad/index.ts b/apps/stage-web/src/utils/vad/index.ts index 1e1dd2ac7..2591dd3da 100644 --- a/apps/stage-web/src/utils/vad/index.ts +++ b/apps/stage-web/src/utils/vad/index.ts @@ -2,4 +2,3 @@ export { VADAudioManager } from './manager' export type { VADAudioOptions } from './manager' export { createVAD, VAD } from './vad' export type { VADConfig, VADEventCallback, VADEvents } from './vad' -export { toWav } from './wav' diff --git a/packages/audio/package.json b/packages/audio/package.json index 6c02dcc9d..3ed18d535 100644 --- a/packages/audio/package.json +++ b/packages/audio/package.json @@ -30,6 +30,10 @@ "./vue": { "types": "./dist/vue/index.d.mts", "default": "./dist/vue/index.mjs" + }, + "./encoding": { + "types": "./dist/encoding/index.d.mts", + "default": "./dist/encoding/index.mjs" } }, "main": "./dist/index.mjs", @@ -56,6 +60,7 @@ "@alexanderolsen/libsamplerate-js": "^2.1.2" }, "devDependencies": { + "@moeru/std": "catalog:", "@types/audioworklet": "^0.0.77", "vite": "^7.0.0" } diff --git a/packages/audio/src/encoding/index.ts b/packages/audio/src/encoding/index.ts new file mode 100644 index 000000000..7e51d4280 --- /dev/null +++ b/packages/audio/src/encoding/index.ts @@ -0,0 +1 @@ +export * from './wav' diff --git a/apps/stage-web/src/utils/vad/wav.ts b/packages/audio/src/encoding/wav.ts similarity index 79% rename from apps/stage-web/src/utils/vad/wav.ts rename to packages/audio/src/encoding/wav.ts index 22cadbac9..e86dad5e1 100644 --- a/apps/stage-web/src/utils/vad/wav.ts +++ b/packages/audio/src/encoding/wav.ts @@ -1,12 +1,14 @@ +import { encodeBase64 } from '@moeru/std/base64' + function writeString(dataView: DataView, offset: number, string: string) { for (let i = 0; i < string.length; i++) { dataView.setUint8(offset + i, string.charCodeAt(i)) } } -export function toWav(buffer: Float32Array, sampleRate: number) { - const numChannels = 1 - const numSamples = buffer.length +export function toWav(buffer: ArrayBufferLike, sampleRate: number, channel = 1) { + const numChannels = channel + const numSamples = buffer.byteLength // Create the WAV file container const arrayBuffer = new ArrayBuffer(44 + numSamples * 2) @@ -25,6 +27,7 @@ export function toWav(buffer: Float32Array, sampleRate: number) { dataView.setUint32(24, sampleRate, true) dataView.setUint32(28, sampleRate * numChannels * 2, true) // byte rate dataView.setUint16(32, numChannels * 2, true) // block align + dataView.setUint16(34, 16, true) // bits per sample // data sub-chunk @@ -41,3 +44,7 @@ export function toWav(buffer: Float32Array, sampleRate: number) { return arrayBuffer } + +export function toWAVBase64(buffer: ArrayBufferLike, sampleRate: number) { + return encodeBase64(toWav(buffer, sampleRate)) +} diff --git a/packages/audio/src/index.ts b/packages/audio/src/index.ts index e69de29bb..718de6aef 100644 --- a/packages/audio/src/index.ts +++ b/packages/audio/src/index.ts @@ -0,0 +1 @@ +export * from './encoding' diff --git a/packages/audio/tsdown.config.ts b/packages/audio/tsdown.config.ts index 40a2ccb7f..1b43f059e 100644 --- a/packages/audio/tsdown.config.ts +++ b/packages/audio/tsdown.config.ts @@ -5,6 +5,7 @@ export default defineConfig({ 'index': 'src/index.ts', 'audio-context/index': 'src/audio-context/index.ts', 'audio-context/processor.worklet': 'src/audio-context/processor.worklet.ts', + 'encoding/index': 'src/encoding/index.ts', 'vue/index': 'src/vue/index.ts', }, unbundle: true, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index fbbdbbc53..ef6d3e4cb 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -876,6 +876,9 @@ importers: specifier: '>=3' version: 3.5.17(typescript@5.8.3) devDependencies: + '@moeru/std': + specifier: 'catalog:' + version: 0.1.0-beta.4 '@types/audioworklet': specifier: ^0.0.77 version: 0.0.77 @@ -1311,6 +1314,9 @@ importers: '@huggingface/transformers': specifier: ^3.6.0 version: 3.6.0 + '@proj-airi/audio': + specifier: workspace:^ + version: link:../../packages/audio '@proj-airi/server-sdk': specifier: workspace:^ version: link:../../packages/server-sdk diff --git a/services/discord-bot/package.json b/services/discord-bot/package.json index d2108e3eb..229ce7c39 100644 --- a/services/discord-bot/package.json +++ b/services/discord-bot/package.json @@ -23,6 +23,7 @@ "@dotenvx/dotenvx": "^1.45.1", "@guiiai/logg": "^1.0.10", "@huggingface/transformers": "^3.6.0", + "@proj-airi/audio": "workspace:^", "@proj-airi/server-sdk": "workspace:^", "@proj-airi/server-shared": "workspace:^", "@xsai-ext/providers-cloud": "catalog:", diff --git a/services/discord-bot/src/pipelines/tts.ts b/services/discord-bot/src/pipelines/tts.ts index 4dd9f88fc..1bb50be41 100644 --- a/services/discord-bot/src/pipelines/tts.ts +++ b/services/discord-bot/src/pipelines/tts.ts @@ -8,11 +8,10 @@ import wavefile from 'wavefile' import { useLogg } from '@guiiai/logg' import { pipeline } from '@huggingface/transformers' +import { toWav } from '@proj-airi/audio' import { createOpenAI } from '@xsai-ext/providers-cloud' import { generateTranscription } from '@xsai/generate-transcription' -import { pcmToWav } from '../utils/audio' - export class WhisperLargeV3Pipeline { static task: PipelineType = 'automatic-speech-recognition' static model = 'Xenova/whisper-medium.en' @@ -49,13 +48,13 @@ export function textFromResult(result: Array<{ text: string }> | { text: string export async function transcribe(pcmBuffer: Buffer) { const log = useLogg('Memory:Transcribe').useGlobalConfig() - const pcmConvertedWav = pcmToWav(pcmBuffer, 48000, 2) + const pcmConvertedWav = toWav(pcmBuffer.buffer, 48000, 2) log.withFields({ from: pcmBuffer.byteLength, to: pcmConvertedWav.byteLength }).log('Audio data received') const transcriber = await WhisperLargeV3Pipeline.getInstance() as (audio: Float32Array | Float64Array) => Promise | { text: string }> log.log('Transcribing audio') - const wav = new wavefile.WaveFile(pcmConvertedWav) + const wav = new wavefile.WaveFile(new Uint8Array(pcmConvertedWav)) wav.toBitDepth('32f') // Pipeline expects input as a Float32Array wav.toSampleRate(16000) // Whisper expects audio with a sampling rate of 16000 const audioData = wav.getSamples() diff --git a/services/discord-bot/src/utils/audio.ts b/services/discord-bot/src/utils/audio.ts index d83f0c08e..f51fcfcc1 100644 --- a/services/discord-bot/src/utils/audio.ts +++ b/services/discord-bot/src/utils/audio.ts @@ -2,56 +2,6 @@ import { Buffer } from 'node:buffer' import { DECODE_SAMPLE_RATE } from '../constants/audio' -export function pcmToWav(pcmBuffer: Buffer, sampleRate: number, numChannels: number): Uint8Array { - const byteRate = sampleRate * numChannels * 2 // Assuming 16-bit PCM (2 bytes per sample) - const blockAlign = numChannels * 2 // Block align for 16-bit PCM - - // Create WAV header - const header = new ArrayBuffer(44) - const view = new DataView(header) - - // Write RIFF identifier - writeString(view, 0, 'RIFF') - // Write file length (size of data + header) - view.setUint32(4, 36 + pcmBuffer.byteLength, true) - // Write WAVE identifier - writeString(view, 8, 'WAVE') - // Write format chunk identifier - writeString(view, 12, 'fmt ') - // Write format chunk length (16 for PCM) - view.setUint32(16, 16, true) - // Write audio format (1 for PCM) - view.setUint16(20, 1, true) - // Write number of channels - view.setUint16(22, numChannels, true) - // Write sample rate - view.setUint32(24, sampleRate, true) - // Write byte rate - view.setUint32(28, byteRate, true) - // Write block align - view.setUint16(32, blockAlign, true) - // Write bits per sample (16) - view.setUint16(34, 16, true) - // Write data chunk identifier - writeString(view, 36, 'data') - // Write data chunk length (size of PCM data) - view.setUint32(40, pcmBuffer.byteLength, true) - - // Combine header and PCM data into one buffer - const wavBuffer = new Uint8Array(header.byteLength + pcmBuffer.byteLength) - - wavBuffer.set(new Uint8Array(header), 0) - wavBuffer.set(new Uint8Array(pcmBuffer), header.byteLength) - - return wavBuffer -} - -function writeString(view, offset, string) { - for (let i = 0; i < string.length; i++) { - view.setUint8(offset + i, string.charCodeAt(i)) - } -} - export function getWavHeader( audioLength: number, sampleRate: number,