diff --git a/apps/stage-tamagotchi/src/main/windows/beat-sync/index.ts b/apps/stage-tamagotchi/src/main/windows/beat-sync/index.ts index bde0f3fb3..f1b19c107 100644 --- a/apps/stage-tamagotchi/src/main/windows/beat-sync/index.ts +++ b/apps/stage-tamagotchi/src/main/windows/beat-sync/index.ts @@ -5,6 +5,7 @@ import { defineInvoke, defineInvokeHandler } from '@moeru/eventa' import { createContext } from '@moeru/eventa/adapters/electron/main' import { beatSyncBeatSignaledInvokeEventa, + beatSyncGetInputByteFrequencyDataInvokeEventa, beatSyncGetStateInvokeEventa, beatSyncStateChangedInvokeEventa, beatSyncToggleInvokeEventa, @@ -14,6 +15,7 @@ import { BrowserWindow, ipcMain } from 'electron' import { beatSyncElectronChangeState, + beatSyncElectronGetInputByteFrequencyData, beatSyncElectronGetState, beatSyncElectronSignalBeat, beatSyncElectronToggle, @@ -36,6 +38,7 @@ export async function setupBeatSync() { const toggle = defineInvoke(context, beatSyncElectronToggle) as (enabled: boolean) => Promise // TODO: Better type const getState = defineInvoke(context, beatSyncElectronGetState) const updateParameters = defineInvoke(context, beatSyncElectronUpdateParameters) + const getInputByteFrequencyData = defineInvoke(context, beatSyncElectronGetInputByteFrequencyData) await load(window, baseUrl(resolve(getElectronMainDirname(), '..', 'renderer'), 'beat-sync.html')) return { @@ -54,6 +57,7 @@ export async function setupBeatSync() { defineInvokeHandler(context, beatSyncToggleInvokeEventa, async enabled => toggle(enabled)), defineInvokeHandler(context, beatSyncGetStateInvokeEventa, async () => getState()), defineInvokeHandler(context, beatSyncUpdateParametersInvokeEventa, async params => updateParameters(params)), + defineInvokeHandler(context, beatSyncGetInputByteFrequencyDataInvokeEventa, async () => getInputByteFrequencyData()), ] const removeHandlers = () => removeHandlerFns.forEach(fn => fn()) window.on('closed', () => removeHandlers()) diff --git a/apps/stage-tamagotchi/src/renderer/beat-sync.main.ts b/apps/stage-tamagotchi/src/renderer/beat-sync.main.ts index 6f6a9603c..00e2e5c8a 100644 --- a/apps/stage-tamagotchi/src/renderer/beat-sync.main.ts +++ b/apps/stage-tamagotchi/src/renderer/beat-sync.main.ts @@ -5,6 +5,7 @@ import { createBeatSyncDetector } from '@proj-airi/stage-shared/beat-sync/browse import { beatSyncElectronChangeState, + beatSyncElectronGetInputByteFrequencyData, beatSyncElectronGetState, beatSyncElectronSignalBeat, beatSyncElectronToggle, @@ -58,3 +59,8 @@ defineInvokeHandler(context, beatSyncElectronUpdateParameters, async (params) => console.log('[update-params]', params) detector.updateParameters(params) }) +defineInvokeHandler(context, beatSyncElectronGetInputByteFrequencyData, async () => { + // eslint-disable-next-line no-console + console.debug('[get-input-byte-frequency-data]') // This could be noisy. + return detector.getInputByteFrequencyData() +}) diff --git a/apps/stage-tamagotchi/src/shared/eventa.ts b/apps/stage-tamagotchi/src/shared/eventa.ts index 055d0aa7f..d5b469e08 100644 --- a/apps/stage-tamagotchi/src/shared/eventa.ts +++ b/apps/stage-tamagotchi/src/shared/eventa.ts @@ -79,5 +79,6 @@ export const beatSyncElectronChangeState = defineInvokeEventa('eventa:event:electron:beat-sync:electron:get-state') export const beatSyncElectronToggle = defineInvokeEventa('eventa:event:electron:beat-sync:electron:toggle') export const beatSyncElectronUpdateParameters = defineInvokeEventa>('eventa:event:electron:beat-sync:electron:update-parameters') +export const beatSyncElectronGetInputByteFrequencyData = defineInvokeEventa>('eventa:event:electron:beat-sync:electron:get-input-byte-frequency-data') export { electron } from './electron' diff --git a/packages/stage-pages/src/pages/settings/modules/beat-sync.vue b/packages/stage-pages/src/pages/settings/modules/beat-sync.vue index 609d4fc96..1b2eac999 100644 --- a/packages/stage-pages/src/pages/settings/modules/beat-sync.vue +++ b/packages/stage-pages/src/pages/settings/modules/beat-sync.vue @@ -4,12 +4,14 @@ import type { BeatSyncDetectorState } from '@proj-airi/stage-shared/beat-sync' import { DEFAULT_ANALYSER_WORKLET_PARAMS } from '@nekopaw/tempora' import { + getBeatSyncInputByteFrequencyData, getBeatSyncState, listenBeatSyncBeatSignal, listenBeatSyncStateChange, toggleBeatSync, updateBeatSyncParameters, } from '@proj-airi/stage-shared/beat-sync/browser' +import { AudioSpectrumVisualizer } from '@proj-airi/stage-ui/components' import { Button, FieldCheckbox, FieldRange } from '@proj-airi/ui' import { createTimeline } from 'animejs' import { nanoid } from 'nanoid' @@ -17,6 +19,8 @@ import { onMounted, onUnmounted, ref, toRaw, watch } from 'vue' import { useI18n } from 'vue-i18n' const state = ref() +const frequencies = ref([]) +const isUpdatingFrequencies = ref(false) const { t } = useI18n() @@ -36,26 +40,6 @@ function normalizeEnergy(energy: number) { return ((base ** energy - 1) / (base - 1)) ** a } -onMounted(() => { - getBeatSyncState().then(initialState => state.value = initialState) - - const removeHandlerFns = [ - listenBeatSyncStateChange((newState) => { - state.value = { ...newState } - }), - listenBeatSyncBeatSignal(({ energy }) => { - beatsHistory.value.unshift({ - id: nanoid(), - energy, - normalizedEnergy: normalizeEnergy(energy), - }) - }), - ] - - const removeHandlers = () => removeHandlerFns.forEach(fn => fn()) - onUnmounted(() => removeHandlers()) -}) - function onRippleEnter(el: Element, done: () => void) { const beatId = (el as HTMLElement).dataset.beatId createTimeline() @@ -85,11 +69,56 @@ function onRippleEnter(el: Element, done: () => void) { function resetDefaultParameters() { parameters.value = { ...DEFAULT_ANALYSER_WORKLET_PARAMS } } + +async function updateFrequencies() { + frequencies.value = Array.from(await getBeatSyncInputByteFrequencyData()) + + if (isUpdatingFrequencies.value) + requestAnimationFrame(updateFrequencies) + else + frequencies.value = [0] +} + +watch(state, async (newState) => { + if (newState?.isActive) { + if (!isUpdatingFrequencies.value) { + isUpdatingFrequencies.value = true + updateFrequencies() + } + } + else { + isUpdatingFrequencies.value = false + } +}, { immediate: true, deep: true }) + +onMounted(() => { + getBeatSyncState().then(initialState => state.value = initialState) + + const removeHandlerFns = [ + listenBeatSyncStateChange((newState) => { + state.value = { ...newState } + }), + listenBeatSyncBeatSignal(({ energy }) => { + beatsHistory.value.unshift({ + id: nanoid(), + energy, + normalizedEnergy: normalizeEnergy(energy), + }) + }), + ] + + const removeHandlers = () => removeHandlerFns.forEach(fn => fn()) + onUnmounted(() => removeHandlers()) +}) + +onUnmounted(() => { + isUpdatingFrequencies.value = false +}) diff --git a/packages/stage-shared/src/beat-sync/browser/detector.ts b/packages/stage-shared/src/beat-sync/browser/detector.ts index d74562617..aca150a26 100644 --- a/packages/stage-shared/src/beat-sync/browser/detector.ts +++ b/packages/stage-shared/src/beat-sync/browser/detector.ts @@ -11,6 +11,7 @@ import { startAnalyser as startTemporaAnalyser } from '@nekopaw/tempora' import { isStageTamagotchi, isStageWeb, StageEnvironment } from '../../environment' import { beatSyncBeatSignaledInvokeEventa, + beatSyncGetInputByteFrequencyDataInvokeEventa, beatSyncGetStateInvokeEventa, beatSyncStateChangedInvokeEventa, beatSyncToggleInvokeEventa, @@ -20,6 +21,8 @@ import { createContext, } from './eventa' +export const inputAnalyserFFTSize = 1024 + export interface BeatSyncDetector { start: (createSource: (context: AudioContext) => Promise) => Promise updateParameters: (params: Partial) => void @@ -27,6 +30,7 @@ export interface BeatSyncDetector { stop: () => void on: (event: E, listener: BeatSyncDetectorEventMap[E]) => () => void off: (event: E, listener: BeatSyncDetectorEventMap[E]) => void + getInputByteFrequencyData: () => Uint8Array readonly state: BeatSyncDetectorState readonly context: AudioContext | undefined readonly analyser: Analyser | undefined @@ -52,7 +56,9 @@ export function createBeatSyncDetector(options: CreateBeatSyncDetectorOptions): } let stopSource: (() => void) | undefined - let tapNode: ScriptProcessorNode | undefined // Dev + + let inputAnalyserNode: AnalyserNode | undefined + let inputAnalyserBuffer: Uint8Array | undefined const listeners: { [K in keyof BeatSyncDetectorEventMap]: Array<(...args: any) => void> } = { stateChange: [], @@ -72,11 +78,10 @@ export function createBeatSyncDetector(options: CreateBeatSyncDetectorOptions): stopSource?.() stopSource = undefined - // Dev - if (tapNode) { - tapNode.onaudioprocess = null - tapNode.disconnect() - tapNode = undefined + if (inputAnalyserNode) { + inputAnalyserNode.disconnect() + inputAnalyserNode = undefined + inputAnalyserBuffer = undefined } source?.disconnect() @@ -103,24 +108,14 @@ export function createBeatSyncDetector(options: CreateBeatSyncDetectorOptions): const node = await createSource(context) - if (import.meta.env.DEV) { - // Dev: Use a tap node to verify there're audio samples flowing in the pipeline. - tapNode = context.createScriptProcessor(256, 1, 1) - tapNode.addEventListener('audioprocess', (e) => { - const sample = e.inputBuffer.getChannelData(0) - // Avoid log flooding - if (e.timeStamp % 10 === 0) { - // eslint-disable-next-line no-console - console.debug('[debug:tap] sample.length =', sample.length, 'timeStamp =', e.timeStamp) - } - e.outputBuffer.copyToChannel(sample, 0) - }) - node.connect(tapNode) - tapNode.connect(analyser.workletNode) - } - else { - node.connect(analyser.workletNode) - } + inputAnalyserNode = context.createAnalyser() + inputAnalyserNode.fftSize = inputAnalyserFFTSize // Fast Fourier Transform size (power of 2, 32-32768) + // A smaller fftSize gives better time resolution but worse frequency resolution. + inputAnalyserNode.smoothingTimeConstant = 0.8 // A value between 0 and 1. Higher value smooths out changes. + inputAnalyserBuffer = new Uint8Array(inputAnalyserNode.frequencyBinCount) + + node.connect(inputAnalyserNode) + inputAnalyserNode.connect(analyser?.workletNode) source = node @@ -214,6 +209,11 @@ export function createBeatSyncDetector(options: CreateBeatSyncDetectorOptions): return () => off(event, listener) } + const getInputByteFrequencyData = () => { + inputAnalyserNode?.getByteFrequencyData(inputAnalyserBuffer!) + return inputAnalyserBuffer! + } + return { start, updateParameters, @@ -221,6 +221,7 @@ export function createBeatSyncDetector(options: CreateBeatSyncDetectorOptions): stop, on, off, + getInputByteFrequencyData, get state() { return state }, get context() { return context }, @@ -313,3 +314,15 @@ export function listenBeatSyncBeatSignal(listener: (e: AnalyserBeatEvent) => voi throw new Error('Unknown environment for listenBeatSyncBeatSignal()') } + +export async function getBeatSyncInputByteFrequencyData() { + if (isStageWeb()) { + return getDetector().getInputByteFrequencyData() + } + + if (isStageTamagotchi()) { + return defineInvoke(getContext(), beatSyncGetInputByteFrequencyDataInvokeEventa)() + } + + throw new Error('Unknown environment for getBeatSyncInputByteFrequencyData()') +} diff --git a/packages/stage-shared/src/beat-sync/eventa.ts b/packages/stage-shared/src/beat-sync/eventa.ts index 34a56420a..5f6022a69 100644 --- a/packages/stage-shared/src/beat-sync/eventa.ts +++ b/packages/stage-shared/src/beat-sync/eventa.ts @@ -8,6 +8,7 @@ import { defineInvokeEventa } from '@moeru/eventa' export const beatSyncToggleInvokeEventa = defineInvokeEventa('eventa:invoke:electron:beat-sync:toggle') export const beatSyncGetStateInvokeEventa = defineInvokeEventa('eventa:invoke:electron:beat-sync:get-state') export const beatSyncUpdateParametersInvokeEventa = defineInvokeEventa>('eventa:event:electron:beat-sync:update-parameters') +export const beatSyncGetInputByteFrequencyDataInvokeEventa = defineInvokeEventa>('eventa:invoke:electron:beat-sync:get-input-byte-frequency-data') // Events export const beatSyncStateChangedInvokeEventa = defineInvokeEventa('eventa:event:electron:beat-sync:state-changed')