feat(audio): new package
This commit is contained in:
Neko Ayaka
@@ -19,6 +19,7 @@
|
||||
"@formkit/auto-animate": "^0.8.2",
|
||||
"@huggingface/transformers": "^3.6.0",
|
||||
"@moeru/std": "catalog:",
|
||||
"@proj-airi/audio": "workspace:^",
|
||||
"@proj-airi/ccc": "workspace:^",
|
||||
"@proj-airi/drizzle-duckdb-wasm": "catalog:",
|
||||
"@proj-airi/i18n": "workspace:*",
|
||||
|
||||
@@ -21,6 +21,7 @@
|
||||
"@huggingface/transformers": "^3.6.0",
|
||||
"@llama-flow/core": "^0.4.4",
|
||||
"@moeru/std": "catalog:",
|
||||
"@proj-airi/audio": "workspace:^",
|
||||
"@proj-airi/ccc": "workspace:^",
|
||||
"@proj-airi/drizzle-duckdb-wasm": "catalog:",
|
||||
"@proj-airi/i18n": "workspace:*",
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
{
|
||||
"name": "@proj-airi/audio",
|
||||
"type": "module",
|
||||
"private": true,
|
||||
"description": "Audio processing utilities for Project AIRI",
|
||||
"author": {
|
||||
"name": "Moeru AI Project AIRI Team",
|
||||
"email": "airi@moeru.ai",
|
||||
"url": "https://github.com/moeru-ai"
|
||||
},
|
||||
"license": "MIT",
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/moeru-ai/airi.git",
|
||||
"directory": "packages/audio"
|
||||
},
|
||||
"exports": {
|
||||
".": {
|
||||
"types": "./dist/index.d.mts",
|
||||
"default": "./dist/index.mjs"
|
||||
},
|
||||
"./audio-context": {
|
||||
"types": "./dist/audio-context/index.d.mts",
|
||||
"default": "./dist/audio-context/index.mjs"
|
||||
},
|
||||
"./audio-context/processor.worklet": {
|
||||
"types": "./dist/audio-context/processor.worklet.d.mts",
|
||||
"default": "./dist/audio-context/processor.worklet.mjs"
|
||||
},
|
||||
"./vue": {
|
||||
"types": "./dist/vue/index.d.mts",
|
||||
"default": "./dist/vue/index.mjs"
|
||||
}
|
||||
},
|
||||
"main": "./dist/index.mjs",
|
||||
"types": "./dist/index.d.mts",
|
||||
"files": [
|
||||
"README.md",
|
||||
"dist",
|
||||
"package.json"
|
||||
],
|
||||
"scripts": {
|
||||
"dev": "pnpm run build",
|
||||
"build": "tsdown",
|
||||
"typecheck": "tsc --noEmit"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"vue": ">=3"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"vue": {
|
||||
"optional": true
|
||||
}
|
||||
},
|
||||
"dependencies": {
|
||||
"@alexanderolsen/libsamplerate-js": "^2.1.2"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/audioworklet": "^0.0.77",
|
||||
"vite": "^7.0.0"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,316 @@
|
||||
/// <reference types="vite/client" />
|
||||
|
||||
import LibsamplerateWorkletURL from '@alexanderolsen/libsamplerate-js/dist/libsamplerate.worklet.js?worker&url'
|
||||
|
||||
import ProcessorWorkletURL from './processor.worklet?worker&url'
|
||||
|
||||
let context: AudioContext | undefined
|
||||
let sampleRate: number = 48000 // High quality base sample rate
|
||||
let isReady: boolean = false
|
||||
let error: string = ''
|
||||
let isInitializing: boolean = false
|
||||
let workletLoaded: boolean = false
|
||||
|
||||
const activeSources = new Set<MediaStreamAudioSourceNode>()
|
||||
const activeGainNodes = new Set<GainNode>()
|
||||
const activeAnalyzers = new Set<AnalyserNode>()
|
||||
const activeWorkletNodes = new Set<AudioWorkletNode>()
|
||||
|
||||
const listeners = new Set<(state: State) => void>()
|
||||
|
||||
export interface State {
|
||||
isReady: boolean
|
||||
sampleRate: number
|
||||
error: string
|
||||
isInitializing: boolean
|
||||
workletLoaded: boolean
|
||||
currentTime: number
|
||||
state: AudioContextState
|
||||
}
|
||||
|
||||
export interface WorkletOptions {
|
||||
inputSampleRate?: number
|
||||
outputSampleRate?: number
|
||||
channels?: number
|
||||
converterType?: number
|
||||
bufferSize?: number
|
||||
}
|
||||
|
||||
function notifyListeners() {
|
||||
const state: State = {
|
||||
isReady,
|
||||
sampleRate,
|
||||
error,
|
||||
isInitializing,
|
||||
workletLoaded,
|
||||
currentTime: context?.currentTime ?? 0,
|
||||
state: context?.state ?? 'closed',
|
||||
}
|
||||
listeners.forEach((listener) => {
|
||||
try {
|
||||
listener(state)
|
||||
}
|
||||
catch (err) {
|
||||
console.error('AudioContext state listener error:', err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
async function loadWorklets() {
|
||||
if (!context || workletLoaded)
|
||||
return
|
||||
|
||||
try {
|
||||
await context.audioWorklet.addModule(ProcessorWorkletURL)
|
||||
await context.audioWorklet.addModule(LibsamplerateWorkletURL)
|
||||
|
||||
workletLoaded = true
|
||||
}
|
||||
catch (err) {
|
||||
console.error('Failed to load AudioWorklets:', err)
|
||||
throw new Error(`Worklet loading failed: ${err}`)
|
||||
}
|
||||
}
|
||||
|
||||
export async function initializeAudioContext(requestedSampleRate: number = 48000): Promise<AudioContext> {
|
||||
// Use high quality base sample rate
|
||||
const baseSampleRate = Math.max(requestedSampleRate, 48000)
|
||||
|
||||
if (context && isReady && sampleRate === baseSampleRate && workletLoaded) {
|
||||
return context
|
||||
}
|
||||
|
||||
if (isInitializing) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const checkReady = () => {
|
||||
if (!isInitializing) {
|
||||
if (context && isReady && workletLoaded) {
|
||||
resolve(context)
|
||||
}
|
||||
else {
|
||||
reject(new Error(error || 'AudioContext initialization failed'))
|
||||
}
|
||||
}
|
||||
else {
|
||||
setTimeout(checkReady, 10)
|
||||
}
|
||||
}
|
||||
checkReady()
|
||||
})
|
||||
}
|
||||
|
||||
isInitializing = true
|
||||
error = ''
|
||||
notifyListeners()
|
||||
|
||||
try {
|
||||
// Close existing context if sample rate changed
|
||||
if (context && sampleRate !== baseSampleRate) {
|
||||
await cleanupAudioContext()
|
||||
}
|
||||
|
||||
// Create new context if needed
|
||||
if (!context) {
|
||||
context = new AudioContext({ sampleRate: baseSampleRate })
|
||||
sampleRate = baseSampleRate
|
||||
}
|
||||
|
||||
// Resume if suspended
|
||||
if (context.state === 'suspended') {
|
||||
await context.resume()
|
||||
}
|
||||
|
||||
// Load worklets
|
||||
await loadWorklets()
|
||||
|
||||
isReady = true
|
||||
notifyListeners()
|
||||
return context
|
||||
}
|
||||
catch (err) {
|
||||
error = err instanceof Error ? err.message : String(err)
|
||||
isReady = false
|
||||
workletLoaded = false
|
||||
notifyListeners()
|
||||
console.error('Failed to initialize AudioContext:', err)
|
||||
throw err
|
||||
}
|
||||
finally {
|
||||
isInitializing = false
|
||||
notifyListeners()
|
||||
}
|
||||
}
|
||||
|
||||
export function createAudioSource(mediaStream: MediaStream): MediaStreamAudioSourceNode {
|
||||
if (!context || !isReady) {
|
||||
throw new Error('AudioContext not initialized')
|
||||
}
|
||||
|
||||
const source = context.createMediaStreamSource(mediaStream)
|
||||
activeSources.add(source)
|
||||
return source
|
||||
}
|
||||
|
||||
export function createAudioAnalyser(options?: Partial<{
|
||||
fftSize: number
|
||||
smoothingTimeConstant: number
|
||||
minDecibels: number
|
||||
maxDecibels: number
|
||||
}>): AnalyserNode {
|
||||
if (!context || !isReady) {
|
||||
throw new Error('AudioContext not initialized')
|
||||
}
|
||||
|
||||
const analyser = context.createAnalyser()
|
||||
|
||||
if (options?.fftSize)
|
||||
analyser.fftSize = options.fftSize
|
||||
if (options?.smoothingTimeConstant !== undefined) {
|
||||
analyser.smoothingTimeConstant = options.smoothingTimeConstant
|
||||
}
|
||||
if (options?.minDecibels !== undefined)
|
||||
analyser.minDecibels = options.minDecibels
|
||||
if (options?.maxDecibels !== undefined)
|
||||
analyser.maxDecibels = options.maxDecibels
|
||||
|
||||
activeAnalyzers.add(analyser)
|
||||
return analyser
|
||||
}
|
||||
|
||||
export function createAudioGainNode(initialGain: number = 1): GainNode {
|
||||
if (!context || !isReady) {
|
||||
throw new Error('AudioContext not initialized')
|
||||
}
|
||||
|
||||
const gainNode = context.createGain()
|
||||
gainNode.gain.value = initialGain
|
||||
activeGainNodes.add(gainNode)
|
||||
return gainNode
|
||||
}
|
||||
|
||||
export function removeAudioSource(source: MediaStreamAudioSourceNode) {
|
||||
source.disconnect()
|
||||
activeSources.delete(source)
|
||||
}
|
||||
|
||||
export function removeAudioGainNode(gainNode: GainNode) {
|
||||
gainNode.disconnect()
|
||||
activeGainNodes.delete(gainNode)
|
||||
}
|
||||
|
||||
export function removeAudioAnalyser(analyser: AnalyserNode) {
|
||||
analyser.disconnect()
|
||||
activeAnalyzers.delete(analyser)
|
||||
}
|
||||
|
||||
export async function suspendAudioContext() {
|
||||
if (context && context.state === 'running') {
|
||||
await context.suspend()
|
||||
notifyListeners()
|
||||
}
|
||||
}
|
||||
|
||||
export async function resumeAudioContext() {
|
||||
if (context && context.state === 'suspended') {
|
||||
await context.resume()
|
||||
notifyListeners()
|
||||
}
|
||||
}
|
||||
|
||||
export function createResamplingWorkletNode(
|
||||
inputNode: AudioNode,
|
||||
options: WorkletOptions = {},
|
||||
): AudioWorkletNode {
|
||||
if (!context || !isReady || !workletLoaded) {
|
||||
throw new Error('AudioContext or worklets not ready')
|
||||
}
|
||||
|
||||
const workletOptions = {
|
||||
inputSampleRate: sampleRate,
|
||||
outputSampleRate: 16000,
|
||||
channels: 1,
|
||||
converterType: 2, // SRC_SINC_MEDIUM_QUALITY
|
||||
bufferSize: 4096,
|
||||
...options,
|
||||
}
|
||||
|
||||
const workletNode = new AudioWorkletNode(context, 'resampling-processor', {
|
||||
numberOfInputs: 1,
|
||||
numberOfOutputs: 1,
|
||||
channelCount: workletOptions.channels,
|
||||
processorOptions: workletOptions,
|
||||
})
|
||||
|
||||
// Connect input to worklet
|
||||
inputNode.connect(workletNode)
|
||||
|
||||
activeWorkletNodes.add(workletNode)
|
||||
|
||||
return workletNode
|
||||
}
|
||||
|
||||
export function removeWorkletNode(node: AudioWorkletNode) {
|
||||
node.disconnect()
|
||||
activeWorkletNodes.delete(node)
|
||||
}
|
||||
|
||||
export async function cleanupAudioContext() {
|
||||
// Disconnect all active nodes
|
||||
activeSources.forEach(source => source.disconnect())
|
||||
activeGainNodes.forEach(gainNode => gainNode.disconnect())
|
||||
activeAnalyzers.forEach(analyser => analyser.disconnect())
|
||||
activeWorkletNodes.forEach(worklet => worklet.disconnect())
|
||||
|
||||
// Clear sets
|
||||
activeSources.clear()
|
||||
activeGainNodes.clear()
|
||||
activeAnalyzers.clear()
|
||||
activeWorkletNodes.clear()
|
||||
|
||||
// Close context
|
||||
if (context && context.state !== 'closed') {
|
||||
await context.close()
|
||||
}
|
||||
|
||||
context = undefined
|
||||
isReady = false
|
||||
workletLoaded = false
|
||||
error = ''
|
||||
notifyListeners()
|
||||
}
|
||||
|
||||
export function getAudioContextState(): State {
|
||||
return {
|
||||
isReady,
|
||||
sampleRate,
|
||||
error,
|
||||
isInitializing,
|
||||
workletLoaded,
|
||||
currentTime: context?.currentTime ?? 0,
|
||||
state: context?.state ?? 'closed',
|
||||
}
|
||||
}
|
||||
|
||||
export function getAudioContext(): AudioContext | undefined {
|
||||
return context
|
||||
}
|
||||
|
||||
export function getCurrentTime(): number {
|
||||
return context?.currentTime ?? 0
|
||||
}
|
||||
|
||||
export function isAudioContextReady(): boolean {
|
||||
return isReady
|
||||
}
|
||||
|
||||
// Event subscription
|
||||
export function subscribeToAudioContext(listener: (state: State) => void): () => void {
|
||||
listeners.add(listener)
|
||||
return () => listeners.delete(listener)
|
||||
}
|
||||
|
||||
// Browser cleanup
|
||||
if ('window' in globalThis && globalThis.window != null) {
|
||||
globalThis.window.addEventListener('beforeunload', cleanupAudioContext)
|
||||
globalThis.window.addEventListener('pagehide', cleanupAudioContext)
|
||||
}
|
||||
@@ -0,0 +1,160 @@
|
||||
/// <reference types="@types/audioworklet" />
|
||||
|
||||
import type { ConverterTypeValue } from '@alexanderolsen/libsamplerate-js/dist/converter-type'
|
||||
|
||||
import { ConverterType, create } from '@alexanderolsen/libsamplerate-js'
|
||||
|
||||
interface ProcessorOptions {
|
||||
inputSampleRate: number
|
||||
outputSampleRate: number
|
||||
channels: number
|
||||
converterType: ConverterTypeValue
|
||||
bufferSize: number
|
||||
}
|
||||
|
||||
class ResamplingAudioWorkletProcessor extends AudioWorkletProcessor {
|
||||
private converter: Awaited<ReturnType<typeof create>> | null = null
|
||||
private isInitialized = false
|
||||
private options: ProcessorOptions
|
||||
private inputBuffer: Float32Array[] = []
|
||||
private outputBuffer: Float32Array[] = []
|
||||
private bufferSize: number
|
||||
|
||||
constructor(options: AudioWorkletNodeOptions) {
|
||||
super()
|
||||
|
||||
this.options = {
|
||||
inputSampleRate: options.processorOptions?.inputSampleRate || 44100,
|
||||
outputSampleRate: options.processorOptions?.outputSampleRate || 16000,
|
||||
channels: options.processorOptions?.channels || 1,
|
||||
converterType: options.processorOptions?.converterType || ConverterType.SRC_SINC_MEDIUM_QUALITY,
|
||||
bufferSize: options.processorOptions?.bufferSize || 4096,
|
||||
}
|
||||
|
||||
this.bufferSize = this.options.bufferSize
|
||||
|
||||
// Initialize input/output buffers for each channel
|
||||
for (let i = 0; i < this.options.channels; i++) {
|
||||
this.inputBuffer[i] = new Float32Array(this.bufferSize)
|
||||
this.outputBuffer[i] = new Float32Array(0)
|
||||
}
|
||||
|
||||
this.initializeConverter()
|
||||
|
||||
// Listen for messages from main thread
|
||||
this.port.onmessage = (event) => {
|
||||
if (event.data.type === 'updateOptions') {
|
||||
this.updateOptions(event.data.options)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async initializeConverter() {
|
||||
try {
|
||||
this.converter = await create(
|
||||
this.options.channels,
|
||||
this.options.inputSampleRate,
|
||||
this.options.outputSampleRate,
|
||||
{
|
||||
converterType: this.options.converterType,
|
||||
},
|
||||
)
|
||||
this.isInitialized = true
|
||||
this.port.postMessage({ type: 'initialized', success: true })
|
||||
}
|
||||
catch (error) {
|
||||
console.error('Failed to initialize sample rate converter:', error)
|
||||
|
||||
this.port.postMessage({
|
||||
type: 'initialized',
|
||||
success: false,
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
private async updateOptions(newOptions: Partial<ProcessorOptions>) {
|
||||
const needsReinitialize
|
||||
= newOptions.inputSampleRate !== this.options.inputSampleRate
|
||||
|| newOptions.outputSampleRate !== this.options.outputSampleRate
|
||||
|| newOptions.channels !== this.options.channels
|
||||
|| newOptions.converterType !== this.options.converterType
|
||||
|
||||
Object.assign(this.options, newOptions)
|
||||
|
||||
if (needsReinitialize && this.converter) {
|
||||
this.converter.destroy()
|
||||
this.converter = null
|
||||
this.isInitialized = false
|
||||
await this.initializeConverter()
|
||||
}
|
||||
}
|
||||
|
||||
process(inputs: Float32Array[][], outputs: Float32Array[][]): boolean {
|
||||
const input = inputs[0]
|
||||
const output = outputs[0]
|
||||
|
||||
if (!this.isInitialized || !this.converter || !input.length) {
|
||||
// Pass through if not ready
|
||||
for (let channel = 0; channel < output.length; channel++) {
|
||||
if (input[channel]) {
|
||||
output[channel].set(input[channel])
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
try {
|
||||
// Process each channel
|
||||
for (let channel = 0; channel < Math.min(input.length, this.options.channels); channel++) {
|
||||
const inputData = input[channel]
|
||||
|
||||
if (inputData && inputData.length > 0) {
|
||||
// Resample the input data
|
||||
const resampledData = this.converter.simple(inputData)
|
||||
|
||||
// Send resampled data to main thread
|
||||
this.port.postMessage({
|
||||
type: 'audioData',
|
||||
channel,
|
||||
data: resampledData,
|
||||
originalSampleRate: this.options.inputSampleRate,
|
||||
outputSampleRate: this.options.outputSampleRate,
|
||||
timestamp: currentTime,
|
||||
})
|
||||
|
||||
// Copy to output (you might want to buffer this properly for different sample rates)
|
||||
if (output[channel]) {
|
||||
const copyLength = Math.min(resampledData.length, output[channel].length)
|
||||
for (let i = 0; i < copyLength; i++) {
|
||||
output[channel][i] = resampledData[i]
|
||||
}
|
||||
// Zero-pad remaining
|
||||
for (let i = copyLength; i < output[channel].length; i++) {
|
||||
output[channel][i] = 0
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (error) {
|
||||
console.error('Resampling error in worklet:', error)
|
||||
|
||||
this.port.postMessage({
|
||||
type: 'error',
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
})
|
||||
|
||||
// Pass through original data on error
|
||||
for (let channel = 0; channel < output.length; channel++) {
|
||||
if (input[channel]) {
|
||||
output[channel].set(input[channel])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
registerProcessor('resampling-processor', ResamplingAudioWorkletProcessor)
|
||||
@@ -0,0 +1,94 @@
|
||||
import type { MaybeRefOrGetter } from 'vue'
|
||||
|
||||
import { computed, onUnmounted, ref, toRef, watch } from 'vue'
|
||||
|
||||
import { createAudioGainNode, createAudioSource, getAudioContext, getCurrentTime, initializeAudioContext, removeAudioGainNode, removeAudioSource } from '../audio-context'
|
||||
|
||||
export function useAudioPlayback(src: MaybeRefOrGetter<MediaStream | undefined>) {
|
||||
const audioContext = getAudioContext()
|
||||
const sourceStream = toRef(src)
|
||||
|
||||
const isEnabled = ref(false)
|
||||
const volume = ref(50) // 0-100
|
||||
const error = ref('')
|
||||
|
||||
// Audio nodes for playback
|
||||
const source = ref<MediaStreamAudioSourceNode>()
|
||||
const gainNode = ref<GainNode>()
|
||||
|
||||
const actualVolume = computed(() => volume.value / 100)
|
||||
|
||||
async function setupPlayback() {
|
||||
if (!sourceStream.value || !isEnabled.value)
|
||||
return
|
||||
|
||||
try {
|
||||
error.value = ''
|
||||
|
||||
// Ensure audio context is ready
|
||||
await initializeAudioContext()
|
||||
|
||||
// Clean up existing playback
|
||||
cleanupPlayback()
|
||||
|
||||
// Create new nodes
|
||||
source.value = createAudioSource(sourceStream.value)
|
||||
gainNode.value = createAudioGainNode(actualVolume.value)
|
||||
|
||||
// Connect: source -> gain -> destination
|
||||
source.value.connect(gainNode.value)
|
||||
gainNode.value.connect(audioContext!.destination)
|
||||
}
|
||||
catch (err) {
|
||||
error.value = err instanceof Error ? err.message : String(err)
|
||||
console.error('Failed to setup audio playback:', err)
|
||||
}
|
||||
}
|
||||
|
||||
function cleanupPlayback() {
|
||||
if (source.value) {
|
||||
removeAudioSource(source.value)
|
||||
source.value = undefined
|
||||
}
|
||||
|
||||
if (gainNode.value) {
|
||||
removeAudioGainNode(gainNode.value)
|
||||
gainNode.value = undefined
|
||||
}
|
||||
}
|
||||
|
||||
// Watch for changes
|
||||
watch([sourceStream, isEnabled], async () => {
|
||||
if (isEnabled.value && sourceStream.value) {
|
||||
await setupPlayback()
|
||||
}
|
||||
else {
|
||||
cleanupPlayback()
|
||||
}
|
||||
})
|
||||
|
||||
watch(actualVolume, (newVolume) => {
|
||||
if (gainNode.value) {
|
||||
// Smooth volume changes to prevent clicks
|
||||
const currentTime = getCurrentTime()
|
||||
gainNode.value.gain.cancelScheduledValues(currentTime)
|
||||
gainNode.value.gain.setValueAtTime(gainNode.value.gain.value, currentTime)
|
||||
gainNode.value.gain.linearRampToValueAtTime(newVolume, currentTime + 0.1)
|
||||
}
|
||||
})
|
||||
|
||||
onUnmounted(() => {
|
||||
cleanupPlayback()
|
||||
})
|
||||
|
||||
return {
|
||||
// State
|
||||
isEnabled,
|
||||
volume,
|
||||
error,
|
||||
|
||||
// Manual control
|
||||
setupPlayback,
|
||||
cleanupPlayback,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,194 @@
|
||||
import type { MaybeRefOrGetter } from 'vue'
|
||||
|
||||
import { onUnmounted, ref, toRef } from 'vue'
|
||||
|
||||
import { createAudioAnalyser, createAudioSource, getAudioContext, getCurrentTime, initializeAudioContext } from '../audio-context'
|
||||
|
||||
export interface AudioStreamConfig {
|
||||
deviceId: string
|
||||
sampleRate?: number
|
||||
echoCancellation?: boolean
|
||||
noiseSuppression?: boolean
|
||||
autoGainControl?: boolean
|
||||
}
|
||||
|
||||
export interface AudioAnalysisData {
|
||||
volumeLevel: number
|
||||
frequencyData: Uint8Array
|
||||
timeDomainData: Float32Array
|
||||
timestamp: number
|
||||
}
|
||||
|
||||
export type AudioAnalysisCallback = (data: AudioAnalysisData) => void
|
||||
export type AudioChunkCallback = (chunk: Float32Array, sampleRate: number) => void
|
||||
|
||||
export function useAudioStream(cfg: MaybeRefOrGetter<AudioStreamConfig | undefined>) {
|
||||
const config = toRef(cfg)
|
||||
|
||||
const isActive = ref(false)
|
||||
const error = ref<string>('')
|
||||
const isInitializing = ref(false) // Add loading state
|
||||
|
||||
// Stream-specific state
|
||||
const mediaStream = ref<MediaStream>()
|
||||
const source = ref<MediaStreamAudioSourceNode>()
|
||||
const analyser = ref<AnalyserNode>()
|
||||
|
||||
// Callbacks for different types of analysis
|
||||
const analysisCallbacks = new Set<AudioAnalysisCallback>()
|
||||
const chunkCallbacks = new Set<AudioChunkCallback>()
|
||||
|
||||
// Audio data arrays
|
||||
const dataArray = ref<Uint8Array>()
|
||||
const timeDataArray = ref<Float32Array>()
|
||||
|
||||
// Helper function to wait for audio context
|
||||
async function waitForAudioContext(maxRetries = 10, delay = 100) {
|
||||
for (let i = 0; i < maxRetries; i++) {
|
||||
const audioContext = getAudioContext()
|
||||
if (audioContext && audioContext.state !== 'closed') {
|
||||
return audioContext
|
||||
}
|
||||
await new Promise(resolve => setTimeout(resolve, delay))
|
||||
}
|
||||
throw new Error('Audio context failed to initialize within timeout')
|
||||
}
|
||||
|
||||
async function start() {
|
||||
if (isActive.value || isInitializing.value)
|
||||
return
|
||||
|
||||
try {
|
||||
isInitializing.value = true
|
||||
error.value = ''
|
||||
|
||||
// Initialize global audio context and wait for it
|
||||
await initializeAudioContext(config.value?.sampleRate || 16000)
|
||||
await waitForAudioContext()
|
||||
|
||||
// Get user media
|
||||
mediaStream.value = await navigator.mediaDevices.getUserMedia({
|
||||
audio: {
|
||||
deviceId: config.value?.deviceId,
|
||||
sampleRate: config.value?.sampleRate || 16000,
|
||||
echoCancellation: config.value?.echoCancellation ?? true,
|
||||
noiseSuppression: config.value?.noiseSuppression ?? true,
|
||||
autoGainControl: config.value?.autoGainControl ?? true,
|
||||
},
|
||||
})
|
||||
|
||||
// Ensure we still have a valid audio context after getUserMedia
|
||||
const currentAudioContext = getAudioContext()
|
||||
if (!currentAudioContext) {
|
||||
throw new Error('Audio context became unavailable')
|
||||
}
|
||||
|
||||
// Create audio nodes using global context
|
||||
source.value = createAudioSource(mediaStream.value)
|
||||
analyser.value = createAudioAnalyser({
|
||||
fftSize: 512,
|
||||
smoothingTimeConstant: 0.1,
|
||||
})
|
||||
|
||||
// Connect nodes
|
||||
source.value.connect(analyser.value)
|
||||
|
||||
// Setup data arrays
|
||||
const bufferLength = analyser.value.frequencyBinCount
|
||||
dataArray.value = new Uint8Array(bufferLength)
|
||||
timeDataArray.value = new Float32Array(analyser.value.fftSize)
|
||||
|
||||
isActive.value = true
|
||||
startAnalysis()
|
||||
}
|
||||
catch (err) {
|
||||
error.value = err instanceof Error ? err.message : String(err)
|
||||
console.error('Failed to start audio stream:', err)
|
||||
}
|
||||
finally {
|
||||
isInitializing.value = false
|
||||
}
|
||||
}
|
||||
|
||||
function startAnalysis() {
|
||||
const analyze = () => {
|
||||
const audioContext = getAudioContext()
|
||||
if (!audioContext || !analyser.value || !dataArray.value || !timeDataArray.value)
|
||||
return
|
||||
|
||||
// Get frequency and time domain data
|
||||
analyser.value.getByteFrequencyData(dataArray.value)
|
||||
analyser.value.getFloatTimeDomainData(timeDataArray.value)
|
||||
|
||||
// Calculate volume level
|
||||
let sum = 0
|
||||
for (let i = 0; i < dataArray.value.length; i++) {
|
||||
sum += dataArray.value[i] * dataArray.value[i]
|
||||
}
|
||||
const rms = Math.sqrt(sum / dataArray.value.length)
|
||||
const volumeLevel = Math.min(100, (rms / 255) * 100 * 3)
|
||||
|
||||
// Create analysis data
|
||||
const analysisData: AudioAnalysisData = {
|
||||
volumeLevel,
|
||||
frequencyData: new Uint8Array(dataArray.value),
|
||||
timeDomainData: new Float32Array(timeDataArray.value),
|
||||
timestamp: getCurrentTime(),
|
||||
}
|
||||
|
||||
// Notify analysis callbacks
|
||||
analysisCallbacks.forEach((callback) => {
|
||||
try {
|
||||
callback(analysisData)
|
||||
}
|
||||
catch (err) {
|
||||
console.error('Audio analysis callback error:', err)
|
||||
}
|
||||
})
|
||||
|
||||
// Notify chunk callbacks with fresh audioContext reference
|
||||
chunkCallbacks.forEach((callback) => {
|
||||
try {
|
||||
callback(
|
||||
new Float32Array(timeDataArray.value),
|
||||
audioContext.sampleRate,
|
||||
)
|
||||
}
|
||||
catch (err) {
|
||||
console.error('Audio chunk callback error:', err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
analyze()
|
||||
}
|
||||
|
||||
// Callback management
|
||||
function addAnalysisCallback(callback: AudioAnalysisCallback) {
|
||||
analysisCallbacks.add(callback)
|
||||
return () => analysisCallbacks.delete(callback)
|
||||
}
|
||||
|
||||
function addChunkCallback(callback: AudioChunkCallback) {
|
||||
chunkCallbacks.add(callback)
|
||||
return () => chunkCallbacks.delete(callback)
|
||||
}
|
||||
|
||||
// Cleanup
|
||||
onUnmounted(() => {
|
||||
stop()
|
||||
})
|
||||
|
||||
return {
|
||||
isActive,
|
||||
error,
|
||||
mediaStream,
|
||||
isInitializing,
|
||||
|
||||
start,
|
||||
stop,
|
||||
|
||||
addAnalysisCallback,
|
||||
addChunkCallback,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
export * from './audio-playback'
|
||||
export * from './audio-stream'
|
||||
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ESNext",
|
||||
"lib": [
|
||||
"ESNext",
|
||||
"DOM"
|
||||
],
|
||||
"module": "ESNext",
|
||||
"moduleResolution": "bundler",
|
||||
"types": [
|
||||
"@types/audioworklet"
|
||||
],
|
||||
"esModuleInterop": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"isolatedModules": true,
|
||||
"verbatimModuleSyntax": true,
|
||||
"skipLibCheck": true
|
||||
},
|
||||
"include": [
|
||||
"src/**/*.ts"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
import { defineConfig } from 'tsdown'
|
||||
|
||||
export default defineConfig({
|
||||
entry: {
|
||||
'index': 'src/index.ts',
|
||||
'audio-context/index': 'src/audio-context/index.ts',
|
||||
'audio-context/processor.worklet': 'src/audio-context/processor.worklet.ts',
|
||||
'vue/index': 'src/vue/index.ts',
|
||||
},
|
||||
unbundle: true,
|
||||
fixedExtension: true,
|
||||
external: [
|
||||
'@alexanderolsen/libsamplerate-js/dist/libsamplerate.worklet.js?worker&url',
|
||||
'./processor.worklet?worker&url',
|
||||
],
|
||||
})
|
||||
Generated
+88
-3
@@ -273,6 +273,9 @@ importers:
|
||||
'@moeru/std':
|
||||
specifier: 'catalog:'
|
||||
version: 0.1.0-beta.4
|
||||
'@proj-airi/audio':
|
||||
specifier: workspace:^
|
||||
version: link:../../packages/audio
|
||||
'@proj-airi/ccc':
|
||||
specifier: workspace:^
|
||||
version: link:../../packages/ccc
|
||||
@@ -562,6 +565,9 @@ importers:
|
||||
'@moeru/std':
|
||||
specifier: 'catalog:'
|
||||
version: 0.1.0-beta.4
|
||||
'@proj-airi/audio':
|
||||
specifier: workspace:^
|
||||
version: link:../../packages/audio
|
||||
'@proj-airi/ccc':
|
||||
specifier: workspace:^
|
||||
version: link:../../packages/ccc
|
||||
@@ -861,6 +867,22 @@ importers:
|
||||
specifier: ^3.5.17
|
||||
version: 3.5.17(typescript@5.8.3)
|
||||
|
||||
packages/audio:
|
||||
dependencies:
|
||||
'@alexanderolsen/libsamplerate-js':
|
||||
specifier: ^2.1.2
|
||||
version: 2.1.2
|
||||
vue:
|
||||
specifier: '>=3'
|
||||
version: 3.5.17(typescript@5.8.3)
|
||||
devDependencies:
|
||||
'@types/audioworklet':
|
||||
specifier: ^0.0.77
|
||||
version: 0.0.77
|
||||
vite:
|
||||
specifier: ^7.0.0
|
||||
version: 7.0.0(@types/node@24.0.7)(jiti@2.4.2)(less@4.3.0)(lightningcss@1.30.1)(terser@5.43.1)(tsx@4.20.3)(yaml@2.8.0)
|
||||
|
||||
packages/ccc:
|
||||
dependencies:
|
||||
meta-png:
|
||||
@@ -877,7 +899,7 @@ importers:
|
||||
devDependencies:
|
||||
unplugin-yaml:
|
||||
specifier: ^3.0.0
|
||||
version: 3.0.0(@nuxt/kit@3.14.1592(magicast@0.3.5)(rollup@4.44.1))(@nuxt/schema@3.14.1592(magicast@0.3.5)(rollup@4.44.1))(astro@5.10.1(@types/node@24.0.7)(encoding@0.1.13)(jiti@2.4.2)(less@4.3.0)(lightningcss@1.30.1)(rollup@4.44.1)(terser@5.43.1)(tsx@4.20.3)(typescript@5.8.3)(yaml@2.8.0))(esbuild@0.25.5)(rolldown@1.0.0-beta.20)(rollup@4.44.1)(vite@6.3.5(@types/node@24.0.7)(jiti@2.4.2)(less@4.3.0)(lightningcss@1.30.1)(terser@5.43.1)(tsx@4.20.3)(yaml@2.8.0))
|
||||
version: 3.0.0(@nuxt/kit@3.14.1592(magicast@0.3.5)(rollup@4.44.1))(@nuxt/schema@3.14.1592(magicast@0.3.5)(rollup@4.44.1))(astro@5.10.1(@types/node@24.0.7)(encoding@0.1.13)(jiti@2.4.2)(less@4.3.0)(lightningcss@1.30.1)(rollup@4.44.1)(terser@5.43.1)(tsx@4.20.3)(typescript@5.8.3)(yaml@2.8.0))(esbuild@0.25.5)(rolldown@1.0.0-beta.20)(rollup@4.44.1)(vite@7.0.0(@types/node@24.0.7)(jiti@2.4.2)(less@4.3.0)(lightningcss@1.30.1)(terser@5.43.1)(tsx@4.20.3)(yaml@2.8.0))
|
||||
|
||||
packages/memory-pgvector:
|
||||
dependencies:
|
||||
@@ -1653,6 +1675,9 @@ packages:
|
||||
resolution: {integrity: sha512-nznEC1ZA/m3hQDEnrGQ4c5gkaa9pcaVnw4LFJyzBAaR7E3nfiAPEHS3otnSafpZouVnoKeITl5D+2LsnwlnK8g==}
|
||||
engines: {node: '>=14.0.0'}
|
||||
|
||||
'@alexanderolsen/libsamplerate-js@2.1.2':
|
||||
resolution: {integrity: sha512-pIXQDX/DZIgz6pKInUddDd6Tnq/s2E9g4ZITkKX60kxM/nAuZcxYa/z0y/jwJbjyp/oKe+8/qHLSqMzQ18xSiQ==}
|
||||
|
||||
'@alvarosabu/utils@3.2.0':
|
||||
resolution: {integrity: sha512-aoGWRfaQjOo9TUwrBA6W0zwTHktgrXy69GIFNILT4gHsqscw6+X8P6uoSlZVQFr887SPm8x3aDin5EBVq8y4pw==}
|
||||
|
||||
@@ -11501,6 +11526,46 @@ packages:
|
||||
yaml:
|
||||
optional: true
|
||||
|
||||
vite@7.0.0:
|
||||
resolution: {integrity: sha512-ixXJB1YRgDIw2OszKQS9WxGHKwLdCsbQNkpJN171udl6szi/rIySHL6/Os3s2+oE4P/FLD4dxg4mD7Wust+u5g==}
|
||||
engines: {node: ^20.19.0 || >=22.12.0}
|
||||
hasBin: true
|
||||
peerDependencies:
|
||||
'@types/node': ^20.19.0 || >=22.12.0
|
||||
jiti: '>=1.21.0'
|
||||
less: ^4.0.0
|
||||
lightningcss: ^1.21.0
|
||||
sass: ^1.70.0
|
||||
sass-embedded: ^1.70.0
|
||||
stylus: '>=0.54.8'
|
||||
sugarss: ^5.0.0
|
||||
terser: ^5.16.0
|
||||
tsx: ^4.8.1
|
||||
yaml: ^2.4.2
|
||||
peerDependenciesMeta:
|
||||
'@types/node':
|
||||
optional: true
|
||||
jiti:
|
||||
optional: true
|
||||
less:
|
||||
optional: true
|
||||
lightningcss:
|
||||
optional: true
|
||||
sass:
|
||||
optional: true
|
||||
sass-embedded:
|
||||
optional: true
|
||||
stylus:
|
||||
optional: true
|
||||
sugarss:
|
||||
optional: true
|
||||
terser:
|
||||
optional: true
|
||||
tsx:
|
||||
optional: true
|
||||
yaml:
|
||||
optional: true
|
||||
|
||||
vitefu@1.0.7:
|
||||
resolution: {integrity: sha512-eRWXLBbJjW3X5z5P5IHcSm2yYbYRPb2kQuc+oqsbAl99WB5kVsPbiiox+cymo8twTzifA6itvhr2CmjnaZZp0Q==}
|
||||
peerDependencies:
|
||||
@@ -12049,6 +12114,8 @@ snapshots:
|
||||
|
||||
'@akryum/tinypool@0.3.1': {}
|
||||
|
||||
'@alexanderolsen/libsamplerate-js@2.1.2': {}
|
||||
|
||||
'@alvarosabu/utils@3.2.0': {}
|
||||
|
||||
'@ampproject/remapping@2.3.0':
|
||||
@@ -23982,7 +24049,7 @@ snapshots:
|
||||
rollup: 4.44.1
|
||||
vite: rolldown-vite@6.3.21(@types/node@24.0.7)(esbuild@0.25.5)(jiti@2.4.2)(less@4.3.0)(terser@5.43.1)(tsx@4.20.3)(yaml@2.8.0)
|
||||
|
||||
unplugin-yaml@3.0.0(@nuxt/kit@3.14.1592(magicast@0.3.5)(rollup@4.44.1))(@nuxt/schema@3.14.1592(magicast@0.3.5)(rollup@4.44.1))(astro@5.10.1(@types/node@24.0.7)(encoding@0.1.13)(jiti@2.4.2)(less@4.3.0)(lightningcss@1.30.1)(rollup@4.44.1)(terser@5.43.1)(tsx@4.20.3)(typescript@5.8.3)(yaml@2.8.0))(esbuild@0.25.5)(rolldown@1.0.0-beta.20)(rollup@4.44.1)(vite@6.3.5(@types/node@24.0.7)(jiti@2.4.2)(less@4.3.0)(lightningcss@1.30.1)(terser@5.43.1)(tsx@4.20.3)(yaml@2.8.0)):
|
||||
unplugin-yaml@3.0.0(@nuxt/kit@3.14.1592(magicast@0.3.5)(rollup@4.44.1))(@nuxt/schema@3.14.1592(magicast@0.3.5)(rollup@4.44.1))(astro@5.10.1(@types/node@24.0.7)(encoding@0.1.13)(jiti@2.4.2)(less@4.3.0)(lightningcss@1.30.1)(rollup@4.44.1)(terser@5.43.1)(tsx@4.20.3)(typescript@5.8.3)(yaml@2.8.0))(esbuild@0.25.5)(rolldown@1.0.0-beta.20)(rollup@4.44.1)(vite@7.0.0(@types/node@24.0.7)(jiti@2.4.2)(less@4.3.0)(lightningcss@1.30.1)(terser@5.43.1)(tsx@4.20.3)(yaml@2.8.0)):
|
||||
dependencies:
|
||||
'@rollup/pluginutils': 5.2.0(rollup@4.44.1)
|
||||
unplugin: 2.3.5
|
||||
@@ -23994,7 +24061,7 @@ snapshots:
|
||||
esbuild: 0.25.5
|
||||
rolldown: 1.0.0-beta.20
|
||||
rollup: 4.44.1
|
||||
vite: 6.3.5(@types/node@24.0.7)(jiti@2.4.2)(less@4.3.0)(lightningcss@1.30.1)(terser@5.43.1)(tsx@4.20.3)(yaml@2.8.0)
|
||||
vite: 7.0.0(@types/node@24.0.7)(jiti@2.4.2)(less@4.3.0)(lightningcss@1.30.1)(terser@5.43.1)(tsx@4.20.3)(yaml@2.8.0)
|
||||
|
||||
unplugin@1.16.1:
|
||||
dependencies:
|
||||
@@ -24349,6 +24416,24 @@ snapshots:
|
||||
tsx: 4.20.3
|
||||
yaml: 2.8.0
|
||||
|
||||
vite@7.0.0(@types/node@24.0.7)(jiti@2.4.2)(less@4.3.0)(lightningcss@1.30.1)(terser@5.43.1)(tsx@4.20.3)(yaml@2.8.0):
|
||||
dependencies:
|
||||
esbuild: 0.25.5
|
||||
fdir: 6.4.6(picomatch@4.0.2)
|
||||
picomatch: 4.0.2
|
||||
postcss: 8.5.6
|
||||
rollup: 4.44.1
|
||||
tinyglobby: 0.2.14
|
||||
optionalDependencies:
|
||||
'@types/node': 24.0.7
|
||||
fsevents: 2.3.3
|
||||
jiti: 2.4.2
|
||||
less: 4.3.0
|
||||
lightningcss: 1.30.1
|
||||
terser: 5.43.1
|
||||
tsx: 4.20.3
|
||||
yaml: 2.8.0
|
||||
|
||||
vitefu@1.0.7(vite@6.3.5(@types/node@24.0.7)(jiti@2.4.2)(less@4.3.0)(lightningcss@1.30.1)(terser@5.43.1)(tsx@4.20.3)(yaml@2.8.0)):
|
||||
optionalDependencies:
|
||||
vite: 6.3.5(@types/node@24.0.7)(jiti@2.4.2)(less@4.3.0)(lightningcss@1.30.1)(terser@5.43.1)(tsx@4.20.3)(yaml@2.8.0)
|
||||
|
||||
Reference in New Issue
Block a user