diff --git a/packages/stage-ui/package.json b/packages/stage-ui/package.json index 8e70178ba..79336b23f 100644 --- a/packages/stage-ui/package.json +++ b/packages/stage-ui/package.json @@ -58,6 +58,7 @@ "@pixiv/three-vrm": "^3.4.2", "@pixiv/three-vrm-animation": "^3.4.2", "@pixiv/three-vrm-core": "^3.4.2", + "@proj-airi/audio": "workspace:^", "@proj-airi/ccc": "workspace:^", "@proj-airi/chromatic": "^1.0.0", "@proj-airi/drizzle-duckdb-wasm": "catalog:", diff --git a/packages/stage-ui/src/components/Scenarios/Providers/TranscriptionPlayground.vue b/packages/stage-ui/src/components/Scenarios/Providers/TranscriptionPlayground.vue index a60117fb8..901ec581a 100644 --- a/packages/stage-ui/src/components/Scenarios/Providers/TranscriptionPlayground.vue +++ b/packages/stage-ui/src/components/Scenarios/Providers/TranscriptionPlayground.vue @@ -147,13 +147,6 @@ watch(audioInputs, () => { // Monitoring toggle async function toggleMonitoring() { if (!isMonitoring.value) { - await setupAudioMonitoring() - await startRecord() - isMonitoring.value = true - } - else { - await stopAudioMonitoring() - onStopRecord(async (recording) => { try { if (recording && recording.size > 0) { @@ -168,6 +161,12 @@ async function toggleMonitoring() { } }) + await setupAudioMonitoring() + await startRecord() + isMonitoring.value = true + } + else { + await stopAudioMonitoring() await stopRecord() isMonitoring.value = false diff --git a/packages/stage-ui/src/composables/audio/audio-recorder.ts b/packages/stage-ui/src/composables/audio/audio-recorder.ts index c3b26b369..68a662a66 100644 --- a/packages/stage-ui/src/composables/audio/audio-recorder.ts +++ b/packages/stage-ui/src/composables/audio/audio-recorder.ts @@ -1,77 +1,121 @@ import type { MaybeRefOrGetter } from 'vue' +import { toWav } from '@proj-airi/audio/encoding' import { until } from '@vueuse/core' import { ref, shallowRef, toRef, watch } from 'vue' export function useAudioRecorder( media: MaybeRefOrGetter, ) { - const audioRecorder = ref() const mediaRef = toRef(media) - const recordingChunk = shallowRef([]) const recording = shallowRef() + const isRecording = ref(false) + + // Audio processing variables + const audioContext = ref() + const sourceNode = ref() + const processorNode = ref() // Using ScriptProcessorNode for wider compatibility + const audioData = ref[]>([]) const onStopRecordHooks = ref Promise>>([]) + function onStopRecord(callback: (recording: Blob | undefined) => Promise) { + onStopRecordHooks.value.push(callback) + } + + // Convert audio buffer to WAV format + function encodeWAV(samples: Float32Array[], sampleRate: number): Blob { + const view = toWav(samples[0].buffer, sampleRate) + return new Blob([view], { type: 'audio/wav' }) + } + async function startRecord() { await until(mediaRef).toBeTruthy() if (!mediaRef.value) { return } + // Reset recording state recording.value = undefined - recordingChunk.value = [] + audioData.value = [] + isRecording.value = true - audioRecorder.value = new MediaRecorder(mediaRef.value) - // Whisper: problem with audio/mp4 blobs from Safari - // https://community.openai.com/t/whisper-problem-with-audio-mp4-blobs-from-safari/322252 - audioRecorder.value.start(1000) + try { + // Create audio context and nodes + audioContext.value = new AudioContext() + sourceNode.value = audioContext.value.createMediaStreamSource(mediaRef.value) - audioRecorder.value.onerror = (event) => { - console.error('Error recording audio:', event) + // Create script processor for audio processing + processorNode.value = audioContext.value.createScriptProcessor(4096, 1, 1) + + // Process audio data + processorNode.value.onaudioprocess = (e) => { + if (isRecording.value) { + const channelData = new Float32Array(e.inputBuffer.getChannelData(0)) + audioData.value.push(channelData) + } + } + + // Connect nodes + sourceNode.value.connect(processorNode.value) + processorNode.value.connect(audioContext.value.destination) + } + catch (error) { + console.error('Error starting audio recording:', error) + isRecording.value = false + } + } + + async function stopRecord() { + if (!isRecording.value || !audioContext.value) { + return } - audioRecorder.value.onstop = () => { - if (recordingChunk.value.length > 0) { - const blob = new Blob(recordingChunk.value, { type: audioRecorder.value?.mimeType }) - recording.value = blob + isRecording.value = false + + try { + // Disconnect and clean up audio nodes + if (processorNode.value) { + processorNode.value.disconnect() + processorNode.value = undefined + } + + if (sourceNode.value) { + sourceNode.value.disconnect() + sourceNode.value = undefined + } + + // Create WAV from recorded data + if (audioData.value.length > 0) { + recording.value = encodeWAV(audioData.value, audioContext.value.sampleRate) + + // Call hooks with the recording + for (const hook of onStopRecordHooks.value) { + await hook(recording.value) + } } else { recording.value = undefined } - for (const hook of onStopRecordHooks.value) { - hook(recording.value) - } + // Close audio context + await audioContext.value.close() + audioContext.value = undefined + } + catch (error) { + console.error('Error stopping audio recording:', error) } - audioRecorder.value.ondataavailable = (event) => { - recordingChunk.value.push(event.data) - } - } - - function onStopRecord(callback: (recording: Blob | undefined) => Promise) { - onStopRecordHooks.value.push(callback) - } - - async function stopRecord() { - if (!audioRecorder.value) { - return [] - } - - audioRecorder.value?.stop() - return recordingChunk.value + return audioData.value } watch(mediaRef, () => { - if (audioRecorder.value && audioRecorder.value.state === 'recording') { - audioRecorder.value.stop() - } - - audioRecorder.value = undefined - - if (mediaRef.value && mediaRef.value.active) { - startRecord() + if (isRecording.value) { + stopRecord().then(() => { + if (mediaRef.value && mediaRef.value.active) { + startRecord() + } + }) } }) @@ -79,7 +123,7 @@ export function useAudioRecorder( startRecord, stopRecord, onStopRecord, - recordingChunk, recording, + isRecording, } } diff --git a/packages/stage-ui/src/composables/download.ts b/packages/stage-ui/src/composables/download.ts new file mode 100644 index 000000000..146d7d729 --- /dev/null +++ b/packages/stage-ui/src/composables/download.ts @@ -0,0 +1,19 @@ +export function useDownload(data: Blob, filename: string) { + function download() { + const url = URL.createObjectURL(data) + + const downloadLink = document.createElement('a') + downloadLink.href = url + downloadLink.download = filename + downloadLink.click() + + // Clean up the URL object after the download + setTimeout(() => { + URL.revokeObjectURL(url) + }, 5000) + } + + return { + download, + } +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a019052e5..206804d25 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1113,7 +1113,7 @@ importers: version: 0.1.3 unplugin-yaml: specifier: ^3.0.2 - version: 3.0.2(@nuxt/kit@3.14.1592(magicast@0.3.5)(rollup@4.45.1))(@nuxt/schema@3.14.1592(magicast@0.3.5)(rollup@4.45.1))(astro@5.10.1(@types/node@24.0.15)(jiti@2.4.2)(less@4.4.0)(lightningcss@1.30.1)(rollup@4.45.1)(terser@5.43.1)(tsx@4.20.3)(typescript@5.8.3)(yaml@2.8.0))(esbuild@0.25.5)(rolldown@1.0.0-beta.29)(rollup@4.45.1)(vite@7.0.5(@types/node@24.0.15)(jiti@2.4.2)(less@4.4.0)(lightningcss@1.30.1)(terser@5.43.1)(tsx@4.20.3)(yaml@2.8.0)) + version: 3.0.2(@nuxt/kit@3.14.1592(magicast@0.3.5)(rollup@4.45.1))(@nuxt/schema@3.14.1592(magicast@0.3.5)(rollup@4.45.1))(astro@5.10.1(@types/node@24.0.15)(encoding@0.1.13)(jiti@2.4.2)(less@4.4.0)(lightningcss@1.30.1)(rollup@4.45.1)(terser@5.43.1)(tsx@4.20.3)(typescript@5.8.3)(yaml@2.8.0))(esbuild@0.25.5)(rolldown@1.0.0-beta.29)(rollup@4.45.1)(vite@7.0.5(@types/node@24.0.15)(jiti@2.4.2)(less@4.4.0)(lightningcss@1.30.1)(terser@5.43.1)(tsx@4.20.3)(yaml@2.8.0)) vitepress: specifier: ^2.0.0-alpha.8 version: 2.0.0-alpha.8(@algolia/client-search@5.32.0)(@types/node@24.0.15)(change-case@5.4.4)(fuse.js@7.1.0)(jiti@2.4.2)(less@4.4.0)(lightningcss@1.30.1)(nprogress@0.2.0)(postcss@8.5.6)(react@18.3.1)(search-insights@2.17.3)(terser@5.43.1)(tsx@4.20.3)(typescript@5.8.3)(yaml@2.8.0) @@ -1156,7 +1156,7 @@ importers: devDependencies: unplugin-yaml: specifier: ^3.0.2 - version: 3.0.2(@nuxt/kit@3.14.1592(magicast@0.3.5)(rollup@4.45.1))(@nuxt/schema@3.14.1592(magicast@0.3.5)(rollup@4.45.1))(astro@5.10.1(@types/node@24.0.15)(jiti@2.4.2)(less@4.4.0)(lightningcss@1.30.1)(rollup@4.45.1)(terser@5.43.1)(tsx@4.20.3)(typescript@5.8.3)(yaml@2.8.0))(esbuild@0.25.5)(rolldown@1.0.0-beta.29)(rollup@4.45.1)(vite@7.0.5(@types/node@24.0.15)(jiti@2.4.2)(less@4.4.0)(lightningcss@1.30.1)(terser@5.43.1)(tsx@4.20.3)(yaml@2.8.0)) + version: 3.0.2(@nuxt/kit@3.14.1592(magicast@0.3.5)(rollup@4.45.1))(@nuxt/schema@3.14.1592(magicast@0.3.5)(rollup@4.45.1))(astro@5.10.1(@types/node@24.0.15)(encoding@0.1.13)(jiti@2.4.2)(less@4.4.0)(lightningcss@1.30.1)(rollup@4.45.1)(terser@5.43.1)(tsx@4.20.3)(typescript@5.8.3)(yaml@2.8.0))(esbuild@0.25.5)(rolldown@1.0.0-beta.29)(rollup@4.45.1)(vite@7.0.5(@types/node@24.0.15)(jiti@2.4.2)(less@4.4.0)(lightningcss@1.30.1)(terser@5.43.1)(tsx@4.20.3)(yaml@2.8.0)) packages/memory-pgvector: dependencies: @@ -1272,6 +1272,9 @@ importers: '@pixiv/three-vrm-core': specifier: ^3.4.2 version: 3.4.2(three@0.178.0) + '@proj-airi/audio': + specifier: workspace:^ + version: link:../audio '@proj-airi/ccc': specifier: workspace:^ version: link:../ccc @@ -1530,7 +1533,7 @@ importers: version: 1.2.2(@nuxt/kit@3.14.1592(magicast@0.3.5)(rollup@4.45.1))(@nuxt/schema@3.14.1592(magicast@0.3.5)(rollup@4.45.1))(esbuild@0.25.5)(rollup@4.45.1)(vite@6.3.5(@types/node@24.0.15)(jiti@2.4.2)(less@4.4.0)(lightningcss@1.30.1)(terser@5.43.1)(tsx@4.20.3)(yaml@2.8.0)) unplugin-yaml: specifier: ^3.0.2 - version: 3.0.2(@nuxt/kit@3.14.1592(magicast@0.3.5)(rollup@4.45.1))(@nuxt/schema@3.14.1592(magicast@0.3.5)(rollup@4.45.1))(astro@5.10.1(@types/node@24.0.15)(jiti@2.4.2)(less@4.4.0)(lightningcss@1.30.1)(rollup@4.45.1)(terser@5.43.1)(tsx@4.20.3)(typescript@5.8.3)(yaml@2.8.0))(esbuild@0.25.5)(rolldown@1.0.0-beta.29)(rollup@4.45.1)(vite@6.3.5(@types/node@24.0.15)(jiti@2.4.2)(less@4.4.0)(lightningcss@1.30.1)(terser@5.43.1)(tsx@4.20.3)(yaml@2.8.0)) + version: 3.0.2(@nuxt/kit@3.14.1592(magicast@0.3.5)(rollup@4.45.1))(@nuxt/schema@3.14.1592(magicast@0.3.5)(rollup@4.45.1))(astro@5.10.1(@types/node@24.0.15)(encoding@0.1.13)(jiti@2.4.2)(less@4.4.0)(lightningcss@1.30.1)(rollup@4.45.1)(terser@5.43.1)(tsx@4.20.3)(typescript@5.8.3)(yaml@2.8.0))(esbuild@0.25.5)(rolldown@1.0.0-beta.29)(rollup@4.45.1)(vite@6.3.5(@types/node@24.0.15)(jiti@2.4.2)(less@4.4.0)(lightningcss@1.30.1)(terser@5.43.1)(tsx@4.20.3)(yaml@2.8.0)) vite: specifier: ^6.3.5 version: 6.3.5(@types/node@24.0.15)(jiti@2.4.2)(less@4.4.0)(lightningcss@1.30.1)(terser@5.43.1)(tsx@4.20.3)(yaml@2.8.0) @@ -18737,107 +18740,6 @@ snapshots: - yaml optional: true - astro@5.10.1(@types/node@24.0.15)(jiti@2.4.2)(less@4.4.0)(lightningcss@1.30.1)(rollup@4.45.1)(terser@5.43.1)(tsx@4.20.3)(typescript@5.8.3)(yaml@2.8.0): - dependencies: - '@astrojs/compiler': 2.12.2 - '@astrojs/internal-helpers': 0.6.1 - '@astrojs/markdown-remark': 6.3.2 - '@astrojs/telemetry': 3.3.0 - '@capsizecss/unpack': 2.4.0(encoding@0.1.13) - '@oslojs/encoding': 1.1.0 - '@rollup/pluginutils': 5.2.0(rollup@4.45.1) - acorn: 8.15.0 - aria-query: 5.3.2 - axobject-query: 4.1.0 - boxen: 8.0.1 - ci-info: 4.2.0 - clsx: 2.1.1 - common-ancestor-path: 1.0.1 - cookie: 1.0.2 - cssesc: 3.0.0 - debug: 4.4.1 - deterministic-object-hash: 2.0.2 - devalue: 5.1.1 - diff: 5.2.0 - dlv: 1.1.3 - dset: 3.1.4 - es-module-lexer: 1.7.0 - esbuild: 0.25.5 - estree-walker: 3.0.3 - flattie: 1.1.1 - fontace: 0.3.0 - github-slugger: 2.0.0 - html-escaper: 3.0.3 - http-cache-semantics: 4.2.0 - import-meta-resolve: 4.1.0 - js-yaml: 4.1.0 - kleur: 4.1.5 - magic-string: 0.30.17 - magicast: 0.3.5 - mrmime: 2.0.1 - neotraverse: 0.6.18 - p-limit: 6.2.0 - p-queue: 8.1.0 - package-manager-detector: 1.3.0 - picomatch: 4.0.2 - prompts: 2.4.2 - rehype: 13.0.2 - semver: 7.7.2 - shiki: 3.8.1 - tinyexec: 0.3.2 - tinyglobby: 0.2.14 - tsconfck: 3.1.6(typescript@5.8.3) - ultrahtml: 1.6.0 - unifont: 0.5.2 - unist-util-visit: 5.0.0 - unstorage: 1.16.0 - vfile: 6.0.3 - vite: 6.3.5(@types/node@24.0.15)(jiti@2.4.2)(less@4.4.0)(lightningcss@1.30.1)(terser@5.43.1)(tsx@4.20.3)(yaml@2.8.0) - vitefu: 1.0.7(vite@6.3.5(@types/node@24.0.15)(jiti@2.4.2)(less@4.4.0)(lightningcss@1.30.1)(terser@5.43.1)(tsx@4.20.3)(yaml@2.8.0)) - xxhash-wasm: 1.1.0 - yargs-parser: 21.1.1 - yocto-spinner: 0.2.3 - zod: 3.25.76 - zod-to-json-schema: 3.24.6(zod@3.25.76) - zod-to-ts: 1.2.0(typescript@5.8.3)(zod@3.25.76) - optionalDependencies: - sharp: 0.33.5 - transitivePeerDependencies: - - '@azure/app-configuration' - - '@azure/cosmos' - - '@azure/data-tables' - - '@azure/identity' - - '@azure/keyvault-secrets' - - '@azure/storage-blob' - - '@capacitor/preferences' - - '@deno/kv' - - '@netlify/blobs' - - '@planetscale/database' - - '@types/node' - - '@upstash/redis' - - '@vercel/blob' - - '@vercel/kv' - - aws4fetch - - db0 - - encoding - - idb-keyval - - ioredis - - jiti - - less - - lightningcss - - rollup - - sass - - sass-embedded - - stylus - - sugarss - - supports-color - - terser - - tsx - - typescript - - uploadthing - - yaml - optional: true - async-mutex@0.3.2: dependencies: tslib: 2.8.1 @@ -25706,7 +25608,7 @@ snapshots: rollup: 4.45.1 vite: rolldown-vite@7.0.9(@types/node@24.0.15)(esbuild@0.25.5)(jiti@2.4.2)(less@4.4.0)(terser@5.43.1)(tsx@4.20.3)(yaml@2.8.0) - unplugin-yaml@3.0.2(@nuxt/kit@3.14.1592(magicast@0.3.5)(rollup@4.45.1))(@nuxt/schema@3.14.1592(magicast@0.3.5)(rollup@4.45.1))(astro@5.10.1(@types/node@24.0.15)(jiti@2.4.2)(less@4.4.0)(lightningcss@1.30.1)(rollup@4.45.1)(terser@5.43.1)(tsx@4.20.3)(typescript@5.8.3)(yaml@2.8.0))(esbuild@0.25.5)(rolldown@1.0.0-beta.29)(rollup@4.45.1)(vite@6.3.5(@types/node@24.0.15)(jiti@2.4.2)(less@4.4.0)(lightningcss@1.30.1)(terser@5.43.1)(tsx@4.20.3)(yaml@2.8.0)): + unplugin-yaml@3.0.2(@nuxt/kit@3.14.1592(magicast@0.3.5)(rollup@4.45.1))(@nuxt/schema@3.14.1592(magicast@0.3.5)(rollup@4.45.1))(astro@5.10.1(@types/node@24.0.15)(encoding@0.1.13)(jiti@2.4.2)(less@4.4.0)(lightningcss@1.30.1)(rollup@4.45.1)(terser@5.43.1)(tsx@4.20.3)(typescript@5.8.3)(yaml@2.8.0))(esbuild@0.25.5)(rolldown@1.0.0-beta.29)(rollup@4.45.1)(vite@6.3.5(@types/node@24.0.15)(jiti@2.4.2)(less@4.4.0)(lightningcss@1.30.1)(terser@5.43.1)(tsx@4.20.3)(yaml@2.8.0)): dependencies: '@rollup/pluginutils': 5.2.0(rollup@4.45.1) unplugin: 2.3.5 @@ -25714,13 +25616,13 @@ snapshots: optionalDependencies: '@nuxt/kit': 3.14.1592(magicast@0.3.5)(rollup@4.45.1) '@nuxt/schema': 3.14.1592(magicast@0.3.5)(rollup@4.45.1) - astro: 5.10.1(@types/node@24.0.15)(jiti@2.4.2)(less@4.4.0)(lightningcss@1.30.1)(rollup@4.45.1)(terser@5.43.1)(tsx@4.20.3)(typescript@5.8.3)(yaml@2.8.0) + astro: 5.10.1(@types/node@24.0.15)(encoding@0.1.13)(jiti@2.4.2)(less@4.4.0)(lightningcss@1.30.1)(rollup@4.45.1)(terser@5.43.1)(tsx@4.20.3)(typescript@5.8.3)(yaml@2.8.0) esbuild: 0.25.5 rolldown: 1.0.0-beta.29 rollup: 4.45.1 vite: 6.3.5(@types/node@24.0.15)(jiti@2.4.2)(less@4.4.0)(lightningcss@1.30.1)(terser@5.43.1)(tsx@4.20.3)(yaml@2.8.0) - unplugin-yaml@3.0.2(@nuxt/kit@3.14.1592(magicast@0.3.5)(rollup@4.45.1))(@nuxt/schema@3.14.1592(magicast@0.3.5)(rollup@4.45.1))(astro@5.10.1(@types/node@24.0.15)(jiti@2.4.2)(less@4.4.0)(lightningcss@1.30.1)(rollup@4.45.1)(terser@5.43.1)(tsx@4.20.3)(typescript@5.8.3)(yaml@2.8.0))(esbuild@0.25.5)(rolldown@1.0.0-beta.29)(rollup@4.45.1)(vite@7.0.5(@types/node@24.0.15)(jiti@2.4.2)(less@4.4.0)(lightningcss@1.30.1)(terser@5.43.1)(tsx@4.20.3)(yaml@2.8.0)): + unplugin-yaml@3.0.2(@nuxt/kit@3.14.1592(magicast@0.3.5)(rollup@4.45.1))(@nuxt/schema@3.14.1592(magicast@0.3.5)(rollup@4.45.1))(astro@5.10.1(@types/node@24.0.15)(encoding@0.1.13)(jiti@2.4.2)(less@4.4.0)(lightningcss@1.30.1)(rollup@4.45.1)(terser@5.43.1)(tsx@4.20.3)(typescript@5.8.3)(yaml@2.8.0))(esbuild@0.25.5)(rolldown@1.0.0-beta.29)(rollup@4.45.1)(vite@7.0.5(@types/node@24.0.15)(jiti@2.4.2)(less@4.4.0)(lightningcss@1.30.1)(terser@5.43.1)(tsx@4.20.3)(yaml@2.8.0)): dependencies: '@rollup/pluginutils': 5.2.0(rollup@4.45.1) unplugin: 2.3.5 @@ -25728,7 +25630,7 @@ snapshots: optionalDependencies: '@nuxt/kit': 3.14.1592(magicast@0.3.5)(rollup@4.45.1) '@nuxt/schema': 3.14.1592(magicast@0.3.5)(rollup@4.45.1) - astro: 5.10.1(@types/node@24.0.15)(jiti@2.4.2)(less@4.4.0)(lightningcss@1.30.1)(rollup@4.45.1)(terser@5.43.1)(tsx@4.20.3)(typescript@5.8.3)(yaml@2.8.0) + astro: 5.10.1(@types/node@24.0.15)(encoding@0.1.13)(jiti@2.4.2)(less@4.4.0)(lightningcss@1.30.1)(rollup@4.45.1)(terser@5.43.1)(tsx@4.20.3)(typescript@5.8.3)(yaml@2.8.0) esbuild: 0.25.5 rolldown: 1.0.0-beta.29 rollup: 4.45.1