nanakura/moeka-project
1
0
Fork 0
Code Issues Pull Requests 1 Actions 2 Packages Projects Releases Wiki Activity

fix(stage-ui): audio recording

Browse Source
This commit is contained in:
Neko Ayaka
2025-08-01 02:34:56 +08:00
parent c6fcf942a9
commit e1e0606bb8
5 changed files with 121 additions and 156 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/stage-ui/package.json
+1
View File
@@ -58,6 +58,7 @@
"@pixiv/three-vrm": "^3.4.2",
"@pixiv/three-vrm-animation": "^3.4.2",
"@pixiv/three-vrm-core": "^3.4.2",
"@proj-airi/audio": "workspace:^",
"@proj-airi/ccc": "workspace:^",
"@proj-airi/chromatic": "^1.0.0",
"@proj-airi/drizzle-duckdb-wasm": "catalog:",
packages/stage-ui/src/components/Scenarios/Providers/TranscriptionPlayground.vue
+6 -7
View File
@@ -147,13 +147,6 @@ watch(audioInputs, () => {
// Monitoring toggle
async function toggleMonitoring() {
if (!isMonitoring.value) {
await setupAudioMonitoring()
await startRecord()
isMonitoring.value = true
}
else {
await stopAudioMonitoring()
onStopRecord(async (recording) => {
try {
if (recording && recording.size > 0) {
@@ -168,6 +161,12 @@ async function toggleMonitoring() {
}
})
await setupAudioMonitoring()
await startRecord()
isMonitoring.value = true
}
else {
await stopAudioMonitoring()
await stopRecord()
isMonitoring.value = false
packages/stage-ui/src/composables/audio/audio-recorder.ts
+85 -41
View File
@@ -1,77 +1,121 @@
import type { MaybeRefOrGetter } from 'vue'
import { toWav } from '@proj-airi/audio/encoding'
import { until } from '@vueuse/core'
import { ref, shallowRef, toRef, watch } from 'vue'
export function useAudioRecorder(
media: MaybeRefOrGetter<MediaStream | undefined>,
) {
const audioRecorder = ref<MediaRecorder>()
const mediaRef = toRef(media)
const recordingChunk = shallowRef<Blob[]>([])
const recording = shallowRef<Blob>()
const isRecording = ref(false)
// Audio processing variables
const audioContext = ref<AudioContext>()
const sourceNode = ref<MediaStreamAudioSourceNode>()
const processorNode = ref<ScriptProcessorNode>() // Using ScriptProcessorNode for wider compatibility
const audioData = ref<Float32Array<ArrayBufferLike>[]>([])
const onStopRecordHooks = ref<Array<(recording: Blob | undefined) => Promise<void>>>([])
function onStopRecord(callback: (recording: Blob | undefined) => Promise<void>) {
onStopRecordHooks.value.push(callback)
}
// Convert audio buffer to WAV format
function encodeWAV(samples: Float32Array[], sampleRate: number): Blob {
const view = toWav(samples[0].buffer, sampleRate)
return new Blob([view], { type: 'audio/wav' })
}
async function startRecord() {
await until(mediaRef).toBeTruthy()
if (!mediaRef.value) {
return
}
// Reset recording state
recording.value = undefined
recordingChunk.value = []
audioData.value = []
isRecording.value = true
audioRecorder.value = new MediaRecorder(mediaRef.value)
// Whisper: problem with audio/mp4 blobs from Safari
// https://community.openai.com/t/whisper-problem-with-audio-mp4-blobs-from-safari/322252
audioRecorder.value.start(1000)
try {
// Create audio context and nodes
audioContext.value = new AudioContext()
sourceNode.value = audioContext.value.createMediaStreamSource(mediaRef.value)
audioRecorder.value.onerror = (event) => {
console.error('Error recording audio:', event)
// Create script processor for audio processing
processorNode.value = audioContext.value.createScriptProcessor(4096, 1, 1)
// Process audio data
processorNode.value.onaudioprocess = (e) => {
if (isRecording.value) {
const channelData = new Float32Array(e.inputBuffer.getChannelData(0))
audioData.value.push(channelData)
}
}
// Connect nodes
sourceNode.value.connect(processorNode.value)
processorNode.value.connect(audioContext.value.destination)
}
catch (error) {
console.error('Error starting audio recording:', error)
isRecording.value = false
}
}
async function stopRecord() {
if (!isRecording.value || !audioContext.value) {
return
}
audioRecorder.value.onstop = () => {
if (recordingChunk.value.length > 0) {
const blob = new Blob(recordingChunk.value, { type: audioRecorder.value?.mimeType })
recording.value = blob
isRecording.value = false
try {
// Disconnect and clean up audio nodes
if (processorNode.value) {
processorNode.value.disconnect()
processorNode.value = undefined
}
if (sourceNode.value) {
sourceNode.value.disconnect()
sourceNode.value = undefined
}
// Create WAV from recorded data
if (audioData.value.length > 0) {
recording.value = encodeWAV(audioData.value, audioContext.value.sampleRate)
// Call hooks with the recording
for (const hook of onStopRecordHooks.value) {
await hook(recording.value)
}
}
else {
recording.value = undefined
}
for (const hook of onStopRecordHooks.value) {
hook(recording.value)
}
// Close audio context
await audioContext.value.close()
audioContext.value = undefined
}
catch (error) {
console.error('Error stopping audio recording:', error)
}
audioRecorder.value.ondataavailable = (event) => {
recordingChunk.value.push(event.data)
}
}
function onStopRecord(callback: (recording: Blob | undefined) => Promise<void>) {
onStopRecordHooks.value.push(callback)
}
async function stopRecord() {
if (!audioRecorder.value) {
return []
}
audioRecorder.value?.stop()
return recordingChunk.value
return audioData.value
}
watch(mediaRef, () => {
if (audioRecorder.value && audioRecorder.value.state === 'recording') {
audioRecorder.value.stop()
}
audioRecorder.value = undefined
if (mediaRef.value && mediaRef.value.active) {
startRecord()
if (isRecording.value) {
stopRecord().then(() => {
if (mediaRef.value && mediaRef.value.active) {
startRecord()
}
})
}
})
@@ -79,7 +123,7 @@ export function useAudioRecorder(
startRecord,
stopRecord,
onStopRecord,
recordingChunk,
recording,
isRecording,
}
}
packages/stage-ui/src/composables/download.ts
+19
View File
@@ -0,0 +1,19 @@
export function useDownload(data: Blob, filename: string) {
function download() {
const url = URL.createObjectURL(data)
const downloadLink = document.createElement('a')
downloadLink.href = url
downloadLink.download = filename
downloadLink.click()
// Clean up the URL object after the download
setTimeout(() => {
URL.revokeObjectURL(url)
}, 5000)
}
return {
download,
}
}
pnpm-lock.yaml
Generated
+10 -108
View File
@@ -1113,7 +1113,7 @@ importers:
version: 0.1.3
unplugin-yaml:
specifier: ^3.0.2
version: 3.0.2(@nuxt/kit@3.14.1592(magicast@0.3.5)(rollup@4.45.1))(@nuxt/schema@3.14.1592(magicast@0.3.5)(rollup@4.45.1))(astro@5.10.1(@types/node@24.0.15)(jiti@2.4.2)(less@4.4.0)(lightningcss@1.30.1)(rollup@4.45.1)(terser@5.43.1)(tsx@4.20.3)(typescript@5.8.3)(yaml@2.8.0))(esbuild@0.25.5)(rolldown@1.0.0-beta.29)(rollup@4.45.1)(vite@7.0.5(@types/node@24.0.15)(jiti@2.4.2)(less@4.4.0)(lightningcss@1.30.1)(terser@5.43.1)(tsx@4.20.3)(yaml@2.8.0))
version: 3.0.2(@nuxt/kit@3.14.1592(magicast@0.3.5)(rollup@4.45.1))(@nuxt/schema@3.14.1592(magicast@0.3.5)(rollup@4.45.1))(astro@5.10.1(@types/node@24.0.15)(encoding@0.1.13)(jiti@2.4.2)(less@4.4.0)(lightningcss@1.30.1)(rollup@4.45.1)(terser@5.43.1)(tsx@4.20.3)(typescript@5.8.3)(yaml@2.8.0))(esbuild@0.25.5)(rolldown@1.0.0-beta.29)(rollup@4.45.1)(vite@7.0.5(@types/node@24.0.15)(jiti@2.4.2)(less@4.4.0)(lightningcss@1.30.1)(terser@5.43.1)(tsx@4.20.3)(yaml@2.8.0))
vitepress:
specifier: ^2.0.0-alpha.8
version: 2.0.0-alpha.8(@algolia/client-search@5.32.0)(@types/node@24.0.15)(change-case@5.4.4)(fuse.js@7.1.0)(jiti@2.4.2)(less@4.4.0)(lightningcss@1.30.1)(nprogress@0.2.0)(postcss@8.5.6)(react@18.3.1)(search-insights@2.17.3)(terser@5.43.1)(tsx@4.20.3)(typescript@5.8.3)(yaml@2.8.0)
@@ -1156,7 +1156,7 @@ importers:
devDependencies:
unplugin-yaml:
specifier: ^3.0.2
version: 3.0.2(@nuxt/kit@3.14.1592(magicast@0.3.5)(rollup@4.45.1))(@nuxt/schema@3.14.1592(magicast@0.3.5)(rollup@4.45.1))(astro@5.10.1(@types/node@24.0.15)(jiti@2.4.2)(less@4.4.0)(lightningcss@1.30.1)(rollup@4.45.1)(terser@5.43.1)(tsx@4.20.3)(typescript@5.8.3)(yaml@2.8.0))(esbuild@0.25.5)(rolldown@1.0.0-beta.29)(rollup@4.45.1)(vite@7.0.5(@types/node@24.0.15)(jiti@2.4.2)(less@4.4.0)(lightningcss@1.30.1)(terser@5.43.1)(tsx@4.20.3)(yaml@2.8.0))
version: 3.0.2(@nuxt/kit@3.14.1592(magicast@0.3.5)(rollup@4.45.1))(@nuxt/schema@3.14.1592(magicast@0.3.5)(rollup@4.45.1))(astro@5.10.1(@types/node@24.0.15)(encoding@0.1.13)(jiti@2.4.2)(less@4.4.0)(lightningcss@1.30.1)(rollup@4.45.1)(terser@5.43.1)(tsx@4.20.3)(typescript@5.8.3)(yaml@2.8.0))(esbuild@0.25.5)(rolldown@1.0.0-beta.29)(rollup@4.45.1)(vite@7.0.5(@types/node@24.0.15)(jiti@2.4.2)(less@4.4.0)(lightningcss@1.30.1)(terser@5.43.1)(tsx@4.20.3)(yaml@2.8.0))
packages/memory-pgvector:
dependencies:
@@ -1272,6 +1272,9 @@ importers:
'@pixiv/three-vrm-core':
specifier: ^3.4.2
version: 3.4.2(three@0.178.0)
'@proj-airi/audio':
specifier: workspace:^
version: link:../audio
'@proj-airi/ccc':
specifier: workspace:^
version: link:../ccc
@@ -1530,7 +1533,7 @@ importers:
version: 1.2.2(@nuxt/kit@3.14.1592(magicast@0.3.5)(rollup@4.45.1))(@nuxt/schema@3.14.1592(magicast@0.3.5)(rollup@4.45.1))(esbuild@0.25.5)(rollup@4.45.1)(vite@6.3.5(@types/node@24.0.15)(jiti@2.4.2)(less@4.4.0)(lightningcss@1.30.1)(terser@5.43.1)(tsx@4.20.3)(yaml@2.8.0))
unplugin-yaml:
specifier: ^3.0.2
version: 3.0.2(@nuxt/kit@3.14.1592(magicast@0.3.5)(rollup@4.45.1))(@nuxt/schema@3.14.1592(magicast@0.3.5)(rollup@4.45.1))(astro@5.10.1(@types/node@24.0.15)(jiti@2.4.2)(less@4.4.0)(lightningcss@1.30.1)(rollup@4.45.1)(terser@5.43.1)(tsx@4.20.3)(typescript@5.8.3)(yaml@2.8.0))(esbuild@0.25.5)(rolldown@1.0.0-beta.29)(rollup@4.45.1)(vite@6.3.5(@types/node@24.0.15)(jiti@2.4.2)(less@4.4.0)(lightningcss@1.30.1)(terser@5.43.1)(tsx@4.20.3)(yaml@2.8.0))
version: 3.0.2(@nuxt/kit@3.14.1592(magicast@0.3.5)(rollup@4.45.1))(@nuxt/schema@3.14.1592(magicast@0.3.5)(rollup@4.45.1))(astro@5.10.1(@types/node@24.0.15)(encoding@0.1.13)(jiti@2.4.2)(less@4.4.0)(lightningcss@1.30.1)(rollup@4.45.1)(terser@5.43.1)(tsx@4.20.3)(typescript@5.8.3)(yaml@2.8.0))(esbuild@0.25.5)(rolldown@1.0.0-beta.29)(rollup@4.45.1)(vite@6.3.5(@types/node@24.0.15)(jiti@2.4.2)(less@4.4.0)(lightningcss@1.30.1)(terser@5.43.1)(tsx@4.20.3)(yaml@2.8.0))
vite:
specifier: ^6.3.5
version: 6.3.5(@types/node@24.0.15)(jiti@2.4.2)(less@4.4.0)(lightningcss@1.30.1)(terser@5.43.1)(tsx@4.20.3)(yaml@2.8.0)
@@ -18737,107 +18740,6 @@ snapshots:
- yaml
optional: true
astro@5.10.1(@types/node@24.0.15)(jiti@2.4.2)(less@4.4.0)(lightningcss@1.30.1)(rollup@4.45.1)(terser@5.43.1)(tsx@4.20.3)(typescript@5.8.3)(yaml@2.8.0):
dependencies:
'@astrojs/compiler': 2.12.2
'@astrojs/internal-helpers': 0.6.1
'@astrojs/markdown-remark': 6.3.2
'@astrojs/telemetry': 3.3.0
'@capsizecss/unpack': 2.4.0(encoding@0.1.13)
'@oslojs/encoding': 1.1.0
'@rollup/pluginutils': 5.2.0(rollup@4.45.1)
acorn: 8.15.0
aria-query: 5.3.2
axobject-query: 4.1.0
boxen: 8.0.1
ci-info: 4.2.0
clsx: 2.1.1
common-ancestor-path: 1.0.1
cookie: 1.0.2
cssesc: 3.0.0
debug: 4.4.1
deterministic-object-hash: 2.0.2
devalue: 5.1.1
diff: 5.2.0
dlv: 1.1.3
dset: 3.1.4
es-module-lexer: 1.7.0
esbuild: 0.25.5
estree-walker: 3.0.3
flattie: 1.1.1
fontace: 0.3.0
github-slugger: 2.0.0
html-escaper: 3.0.3
http-cache-semantics: 4.2.0
import-meta-resolve: 4.1.0
js-yaml: 4.1.0
kleur: 4.1.5
magic-string: 0.30.17
magicast: 0.3.5
mrmime: 2.0.1
neotraverse: 0.6.18
p-limit: 6.2.0
p-queue: 8.1.0
package-manager-detector: 1.3.0
picomatch: 4.0.2
prompts: 2.4.2
rehype: 13.0.2
semver: 7.7.2
shiki: 3.8.1
tinyexec: 0.3.2
tinyglobby: 0.2.14
tsconfck: 3.1.6(typescript@5.8.3)
ultrahtml: 1.6.0
unifont: 0.5.2
unist-util-visit: 5.0.0
unstorage: 1.16.0
vfile: 6.0.3
vite: 6.3.5(@types/node@24.0.15)(jiti@2.4.2)(less@4.4.0)(lightningcss@1.30.1)(terser@5.43.1)(tsx@4.20.3)(yaml@2.8.0)
vitefu: 1.0.7(vite@6.3.5(@types/node@24.0.15)(jiti@2.4.2)(less@4.4.0)(lightningcss@1.30.1)(terser@5.43.1)(tsx@4.20.3)(yaml@2.8.0))
xxhash-wasm: 1.1.0
yargs-parser: 21.1.1
yocto-spinner: 0.2.3
zod: 3.25.76
zod-to-json-schema: 3.24.6(zod@3.25.76)
zod-to-ts: 1.2.0(typescript@5.8.3)(zod@3.25.76)
optionalDependencies:
sharp: 0.33.5
transitivePeerDependencies:
- '@azure/app-configuration'
- '@azure/cosmos'
- '@azure/data-tables'
- '@azure/identity'
- '@azure/keyvault-secrets'
- '@azure/storage-blob'
- '@capacitor/preferences'
- '@deno/kv'
- '@netlify/blobs'
- '@planetscale/database'
- '@types/node'
- '@upstash/redis'
- '@vercel/blob'
- '@vercel/kv'
- aws4fetch
- db0
- encoding
- idb-keyval
- ioredis
- jiti
- less
- lightningcss
- rollup
- sass
- sass-embedded
- stylus
- sugarss
- supports-color
- terser
- tsx
- typescript
- uploadthing
- yaml
optional: true
async-mutex@0.3.2:
dependencies:
tslib: 2.8.1
@@ -25706,7 +25608,7 @@ snapshots:
rollup: 4.45.1
vite: rolldown-vite@7.0.9(@types/node@24.0.15)(esbuild@0.25.5)(jiti@2.4.2)(less@4.4.0)(terser@5.43.1)(tsx@4.20.3)(yaml@2.8.0)
unplugin-yaml@3.0.2(@nuxt/kit@3.14.1592(magicast@0.3.5)(rollup@4.45.1))(@nuxt/schema@3.14.1592(magicast@0.3.5)(rollup@4.45.1))(astro@5.10.1(@types/node@24.0.15)(jiti@2.4.2)(less@4.4.0)(lightningcss@1.30.1)(rollup@4.45.1)(terser@5.43.1)(tsx@4.20.3)(typescript@5.8.3)(yaml@2.8.0))(esbuild@0.25.5)(rolldown@1.0.0-beta.29)(rollup@4.45.1)(vite@6.3.5(@types/node@24.0.15)(jiti@2.4.2)(less@4.4.0)(lightningcss@1.30.1)(terser@5.43.1)(tsx@4.20.3)(yaml@2.8.0)):
unplugin-yaml@3.0.2(@nuxt/kit@3.14.1592(magicast@0.3.5)(rollup@4.45.1))(@nuxt/schema@3.14.1592(magicast@0.3.5)(rollup@4.45.1))(astro@5.10.1(@types/node@24.0.15)(encoding@0.1.13)(jiti@2.4.2)(less@4.4.0)(lightningcss@1.30.1)(rollup@4.45.1)(terser@5.43.1)(tsx@4.20.3)(typescript@5.8.3)(yaml@2.8.0))(esbuild@0.25.5)(rolldown@1.0.0-beta.29)(rollup@4.45.1)(vite@6.3.5(@types/node@24.0.15)(jiti@2.4.2)(less@4.4.0)(lightningcss@1.30.1)(terser@5.43.1)(tsx@4.20.3)(yaml@2.8.0)):
dependencies:
'@rollup/pluginutils': 5.2.0(rollup@4.45.1)
unplugin: 2.3.5
@@ -25714,13 +25616,13 @@ snapshots:
optionalDependencies:
'@nuxt/kit': 3.14.1592(magicast@0.3.5)(rollup@4.45.1)
'@nuxt/schema': 3.14.1592(magicast@0.3.5)(rollup@4.45.1)
astro: 5.10.1(@types/node@24.0.15)(jiti@2.4.2)(less@4.4.0)(lightningcss@1.30.1)(rollup@4.45.1)(terser@5.43.1)(tsx@4.20.3)(typescript@5.8.3)(yaml@2.8.0)
astro: 5.10.1(@types/node@24.0.15)(encoding@0.1.13)(jiti@2.4.2)(less@4.4.0)(lightningcss@1.30.1)(rollup@4.45.1)(terser@5.43.1)(tsx@4.20.3)(typescript@5.8.3)(yaml@2.8.0)
esbuild: 0.25.5
rolldown: 1.0.0-beta.29
rollup: 4.45.1
vite: 6.3.5(@types/node@24.0.15)(jiti@2.4.2)(less@4.4.0)(lightningcss@1.30.1)(terser@5.43.1)(tsx@4.20.3)(yaml@2.8.0)
unplugin-yaml@3.0.2(@nuxt/kit@3.14.1592(magicast@0.3.5)(rollup@4.45.1))(@nuxt/schema@3.14.1592(magicast@0.3.5)(rollup@4.45.1))(astro@5.10.1(@types/node@24.0.15)(jiti@2.4.2)(less@4.4.0)(lightningcss@1.30.1)(rollup@4.45.1)(terser@5.43.1)(tsx@4.20.3)(typescript@5.8.3)(yaml@2.8.0))(esbuild@0.25.5)(rolldown@1.0.0-beta.29)(rollup@4.45.1)(vite@7.0.5(@types/node@24.0.15)(jiti@2.4.2)(less@4.4.0)(lightningcss@1.30.1)(terser@5.43.1)(tsx@4.20.3)(yaml@2.8.0)):
unplugin-yaml@3.0.2(@nuxt/kit@3.14.1592(magicast@0.3.5)(rollup@4.45.1))(@nuxt/schema@3.14.1592(magicast@0.3.5)(rollup@4.45.1))(astro@5.10.1(@types/node@24.0.15)(encoding@0.1.13)(jiti@2.4.2)(less@4.4.0)(lightningcss@1.30.1)(rollup@4.45.1)(terser@5.43.1)(tsx@4.20.3)(typescript@5.8.3)(yaml@2.8.0))(esbuild@0.25.5)(rolldown@1.0.0-beta.29)(rollup@4.45.1)(vite@7.0.5(@types/node@24.0.15)(jiti@2.4.2)(less@4.4.0)(lightningcss@1.30.1)(terser@5.43.1)(tsx@4.20.3)(yaml@2.8.0)):
dependencies:
'@rollup/pluginutils': 5.2.0(rollup@4.45.1)
unplugin: 2.3.5
@@ -25728,7 +25630,7 @@ snapshots:
optionalDependencies:
'@nuxt/kit': 3.14.1592(magicast@0.3.5)(rollup@4.45.1)
'@nuxt/schema': 3.14.1592(magicast@0.3.5)(rollup@4.45.1)
astro: 5.10.1(@types/node@24.0.15)(jiti@2.4.2)(less@4.4.0)(lightningcss@1.30.1)(rollup@4.45.1)(terser@5.43.1)(tsx@4.20.3)(typescript@5.8.3)(yaml@2.8.0)
astro: 5.10.1(@types/node@24.0.15)(encoding@0.1.13)(jiti@2.4.2)(less@4.4.0)(lightningcss@1.30.1)(rollup@4.45.1)(terser@5.43.1)(tsx@4.20.3)(typescript@5.8.3)(yaml@2.8.0)
esbuild: 0.25.5
rolldown: 1.0.0-beta.29
rollup: 4.45.1