nanakura/moeka-project
1
0
Fork 0
Code Issues Pull Requests 1 Actions 1 Packages Projects Releases Wiki Activity

feat(stage-tamagotchi,stage-pages,stage-shared): show audio spectrum for better input monitoring

Browse Source
This commit is contained in:
Makito
2025-12-03 00:15:52 +09:00
parent f6c5ca6026
commit 88428e7a67
6 changed files with 129 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/stage-tamagotchi/src/main/windows/beat-sync/index.ts
+4
View File
@@ -5,6 +5,7 @@ import { defineInvoke, defineInvokeHandler } from '@moeru/eventa'
import { createContext } from '@moeru/eventa/adapters/electron/main'
import {
beatSyncBeatSignaledInvokeEventa,
beatSyncGetInputByteFrequencyDataInvokeEventa,
beatSyncGetStateInvokeEventa,
beatSyncStateChangedInvokeEventa,
beatSyncToggleInvokeEventa,
@@ -14,6 +15,7 @@ import { BrowserWindow, ipcMain } from 'electron'
import {
beatSyncElectronChangeState,
beatSyncElectronGetInputByteFrequencyData,
beatSyncElectronGetState,
beatSyncElectronSignalBeat,
beatSyncElectronToggle,
@@ -36,6 +38,7 @@ export async function setupBeatSync() {
const toggle = defineInvoke(context, beatSyncElectronToggle) as (enabled: boolean) => Promise<void> // TODO: Better type
const getState = defineInvoke(context, beatSyncElectronGetState)
const updateParameters = defineInvoke(context, beatSyncElectronUpdateParameters)
const getInputByteFrequencyData = defineInvoke(context, beatSyncElectronGetInputByteFrequencyData)
await load(window, baseUrl(resolve(getElectronMainDirname(), '..', 'renderer'), 'beat-sync.html'))
return {
@@ -54,6 +57,7 @@ export async function setupBeatSync() {
defineInvokeHandler(context, beatSyncToggleInvokeEventa, async enabled => toggle(enabled)),
defineInvokeHandler(context, beatSyncGetStateInvokeEventa, async () => getState()),
defineInvokeHandler(context, beatSyncUpdateParametersInvokeEventa, async params => updateParameters(params)),
defineInvokeHandler(context, beatSyncGetInputByteFrequencyDataInvokeEventa, async () => getInputByteFrequencyData()),
]
const removeHandlers = () => removeHandlerFns.forEach(fn => fn())
window.on('closed', () => removeHandlers())
apps/stage-tamagotchi/src/renderer/beat-sync.main.ts
+6
View File
@@ -5,6 +5,7 @@ import { createBeatSyncDetector } from '@proj-airi/stage-shared/beat-sync/browse
import {
beatSyncElectronChangeState,
beatSyncElectronGetInputByteFrequencyData,
beatSyncElectronGetState,
beatSyncElectronSignalBeat,
beatSyncElectronToggle,
@@ -58,3 +59,8 @@ defineInvokeHandler(context, beatSyncElectronUpdateParameters, async (params) =>
console.log('[update-params]', params)
detector.updateParameters(params)
})
defineInvokeHandler(context, beatSyncElectronGetInputByteFrequencyData, async () => {
// eslint-disable-next-line no-console
console.debug('[get-input-byte-frequency-data]') // This could be noisy.
return detector.getInputByteFrequencyData()
})
apps/stage-tamagotchi/src/shared/eventa.ts
+1
View File
@@ -79,5 +79,6 @@ export const beatSyncElectronChangeState = defineInvokeEventa<void, BeatSyncDete
export const beatSyncElectronGetState = defineInvokeEventa<BeatSyncDetectorState>('eventa:event:electron:beat-sync:electron:get-state')
export const beatSyncElectronToggle = defineInvokeEventa<void, boolean>('eventa:event:electron:beat-sync:electron:toggle')
export const beatSyncElectronUpdateParameters = defineInvokeEventa<void, Partial<AnalyserWorkletParameters>>('eventa:event:electron:beat-sync:electron:update-parameters')
export const beatSyncElectronGetInputByteFrequencyData = defineInvokeEventa<Uint8Array<ArrayBuffer>>('eventa:event:electron:beat-sync:electron:get-input-byte-frequency-data')
export { electron } from './electron'
packages/stage-pages/src/pages/settings/modules/beat-sync.vue
+80 -46
View File
@@ -4,12 +4,14 @@ import type { BeatSyncDetectorState } from '@proj-airi/stage-shared/beat-sync'
import { DEFAULT_ANALYSER_WORKLET_PARAMS } from '@nekopaw/tempora'
import {
getBeatSyncInputByteFrequencyData,
getBeatSyncState,
listenBeatSyncBeatSignal,
listenBeatSyncStateChange,
toggleBeatSync,
updateBeatSyncParameters,
} from '@proj-airi/stage-shared/beat-sync/browser'
import { AudioSpectrumVisualizer } from '@proj-airi/stage-ui/components'
import { Button, FieldCheckbox, FieldRange } from '@proj-airi/ui'
import { createTimeline } from 'animejs'
import { nanoid } from 'nanoid'
@@ -17,6 +19,8 @@ import { onMounted, onUnmounted, ref, toRaw, watch } from 'vue'
import { useI18n } from 'vue-i18n'
const state = ref<BeatSyncDetectorState>()
const frequencies = ref<number[]>([])
const isUpdatingFrequencies = ref(false)
const { t } = useI18n()
@@ -36,26 +40,6 @@ function normalizeEnergy(energy: number) {
return ((base ** energy - 1) / (base - 1)) ** a
}
onMounted(() => {
getBeatSyncState().then(initialState => state.value = initialState)
const removeHandlerFns = [
listenBeatSyncStateChange((newState) => {
state.value = { ...newState }
}),
listenBeatSyncBeatSignal(({ energy }) => {
beatsHistory.value.unshift({
id: nanoid(),
energy,
normalizedEnergy: normalizeEnergy(energy),
})
}),
]
const removeHandlers = () => removeHandlerFns.forEach(fn => fn())
onUnmounted(() => removeHandlers())
})
function onRippleEnter(el: Element, done: () => void) {
const beatId = (el as HTMLElement).dataset.beatId
createTimeline()
@@ -85,11 +69,56 @@ function onRippleEnter(el: Element, done: () => void) {
function resetDefaultParameters() {
parameters.value = { ...DEFAULT_ANALYSER_WORKLET_PARAMS }
}
async function updateFrequencies() {
frequencies.value = Array.from(await getBeatSyncInputByteFrequencyData())
if (isUpdatingFrequencies.value)
requestAnimationFrame(updateFrequencies)
else
frequencies.value = [0]
}
watch(state, async (newState) => {
if (newState?.isActive) {
if (!isUpdatingFrequencies.value) {
isUpdatingFrequencies.value = true
updateFrequencies()
}
}
else {
isUpdatingFrequencies.value = false
}
}, { immediate: true, deep: true })
onMounted(() => {
getBeatSyncState().then(initialState => state.value = initialState)
const removeHandlerFns = [
listenBeatSyncStateChange((newState) => {
state.value = { ...newState }
}),
listenBeatSyncBeatSignal(({ energy }) => {
beatsHistory.value.unshift({
id: nanoid(),
energy,
normalizedEnergy: normalizeEnergy(energy),
})
}),
]
const removeHandlers = () => removeHandlerFns.forEach(fn => fn())
onUnmounted(() => removeHandlers())
})
onUnmounted(() => {
isUpdatingFrequencies.value = false
})
</script>
<template>
<div flex="~ col md:row gap-6">
<div bg="neutral-100 dark:[rgba(0,0,0,0.3)]" rounded-xl p-4 flex="~ col gap-4" class="h-fit w-full md:w-[40%]">
<div bg="neutral-100 dark:[rgba(0,0,0,0.3)]" rounded-xl p-4 flex="~ col gap-4" class="h-fit w-full md:w-[60%]">
<div flex="~ col gap-6">
<div flex="~ col gap-4">
<div>
@@ -227,33 +256,38 @@ function resetDefaultParameters() {
</div>
</div>
<div flex="~ col gap-6" class="w-full md:w-[60%]">
<div w-full rounded-xl flex="~ col gap-4">
<h2 class="mb-4 text-lg text-neutral-500 md:text-2xl dark:text-neutral-400" w-full>
<div class="inline-flex items-center gap-4">
{{ t('settings.pages.modules.beat_sync.sections.beat_visualizer.title') }}
</div>
</h2>
<div flex="~ col gap-4 items-center">
<TransitionGroup
tag="div"
bg="neutral/10"
relative box-border aspect-square h-full max-h-400px max-w-400px w-full rounded-2xl
flex="~ row gap-2 wrap items-center"
:css="false"
@enter="onRippleEnter"
>
<div
v-for="beat in beatsHistory"
:key="beat.id"
:data-beat-id="beat.id"
absolute h-full w-full
rounded-full bg="primary/50"
/>
</TransitionGroup>
<div flex="~ col gap-6 items-center" class="w-full md:w-[40%]">
<h2 class="mb-4 text-lg text-neutral-500 md:text-2xl dark:text-neutral-400" w-full>
<div class="inline-flex items-center gap-4">
{{ t('settings.pages.modules.beat_sync.sections.beat_visualizer.title') }}
</div>
</h2>
<div bg="neutral/10" h-64px max-w-400px w-full overflow-hidden rounded-2xl>
<AudioSpectrumVisualizer
v-if="isUpdatingFrequencies"
:frequencies="frequencies"
h-full w-full gap-0
bars-class="bg-primary-400/50 dark:bg-primary-500/50 rounded-none"
/>
</div>
<TransitionGroup
tag="div"
bg="neutral/10"
relative box-border aspect-square h-full max-h-400px max-w-400px w-full rounded-2xl
flex="~ row gap-2 wrap items-center"
:css="false"
@enter="onRippleEnter"
>
<div
v-for="beat in beatsHistory"
:key="beat.id"
:data-beat-id="beat.id"
absolute h-full w-full
rounded-full bg="primary/50"
/>
</TransitionGroup>
</div>
</div>
</template>
packages/stage-shared/src/beat-sync/browser/detector.ts
+37 -24
View File
@@ -11,6 +11,7 @@ import { startAnalyser as startTemporaAnalyser } from '@nekopaw/tempora'
import { isStageTamagotchi, isStageWeb, StageEnvironment } from '../../environment'
import {
beatSyncBeatSignaledInvokeEventa,
beatSyncGetInputByteFrequencyDataInvokeEventa,
beatSyncGetStateInvokeEventa,
beatSyncStateChangedInvokeEventa,
beatSyncToggleInvokeEventa,
@@ -20,6 +21,8 @@ import {
createContext,
} from './eventa'
export const inputAnalyserFFTSize = 1024
export interface BeatSyncDetector {
start: (createSource: (context: AudioContext) => Promise<AudioNode>) => Promise<void>
updateParameters: (params: Partial<AnalyserWorkletParameters>) => void
@@ -27,6 +30,7 @@ export interface BeatSyncDetector {
stop: () => void
on: <E extends keyof BeatSyncDetectorEventMap>(event: E, listener: BeatSyncDetectorEventMap[E]) => () => void
off: <E extends keyof BeatSyncDetectorEventMap>(event: E, listener: BeatSyncDetectorEventMap[E]) => void
getInputByteFrequencyData: () => Uint8Array<ArrayBuffer>
readonly state: BeatSyncDetectorState
readonly context: AudioContext | undefined
readonly analyser: Analyser | undefined
@@ -52,7 +56,9 @@ export function createBeatSyncDetector(options: CreateBeatSyncDetectorOptions):
}
let stopSource: (() => void) | undefined
let tapNode: ScriptProcessorNode | undefined // Dev
let inputAnalyserNode: AnalyserNode | undefined
let inputAnalyserBuffer: Uint8Array<ArrayBuffer> | undefined
const listeners: { [K in keyof BeatSyncDetectorEventMap]: Array<(...args: any) => void> } = {
stateChange: [],
@@ -72,11 +78,10 @@ export function createBeatSyncDetector(options: CreateBeatSyncDetectorOptions):
stopSource?.()
stopSource = undefined
// Dev
if (tapNode) {
tapNode.onaudioprocess = null
tapNode.disconnect()
tapNode = undefined
if (inputAnalyserNode) {
inputAnalyserNode.disconnect()
inputAnalyserNode = undefined
inputAnalyserBuffer = undefined
}
source?.disconnect()
@@ -103,24 +108,14 @@ export function createBeatSyncDetector(options: CreateBeatSyncDetectorOptions):
const node = await createSource(context)
if (import.meta.env.DEV) {
// Dev: Use a tap node to verify there're audio samples flowing in the pipeline.
tapNode = context.createScriptProcessor(256, 1, 1)
tapNode.addEventListener('audioprocess', (e) => {
const sample = e.inputBuffer.getChannelData(0)
// Avoid log flooding
if (e.timeStamp % 10 === 0) {
// eslint-disable-next-line no-console
console.debug('[debug:tap] sample.length =', sample.length, 'timeStamp =', e.timeStamp)
}
e.outputBuffer.copyToChannel(sample, 0)
})
node.connect(tapNode)
tapNode.connect(analyser.workletNode)
}
else {
node.connect(analyser.workletNode)
}
inputAnalyserNode = context.createAnalyser()
inputAnalyserNode.fftSize = inputAnalyserFFTSize // Fast Fourier Transform size (power of 2, 32-32768)
// A smaller fftSize gives better time resolution but worse frequency resolution.
inputAnalyserNode.smoothingTimeConstant = 0.8 // A value between 0 and 1. Higher value smooths out changes.
inputAnalyserBuffer = new Uint8Array(inputAnalyserNode.frequencyBinCount)
node.connect(inputAnalyserNode)
inputAnalyserNode.connect(analyser?.workletNode)
source = node
@@ -214,6 +209,11 @@ export function createBeatSyncDetector(options: CreateBeatSyncDetectorOptions):
return () => off(event, listener)
}
const getInputByteFrequencyData = () => {
inputAnalyserNode?.getByteFrequencyData(inputAnalyserBuffer!)
return inputAnalyserBuffer!
}
return {
start,
updateParameters,
@@ -221,6 +221,7 @@ export function createBeatSyncDetector(options: CreateBeatSyncDetectorOptions):
stop,
on,
off,
getInputByteFrequencyData,
get state() { return state },
get context() { return context },
@@ -313,3 +314,15 @@ export function listenBeatSyncBeatSignal(listener: (e: AnalyserBeatEvent) => voi
throw new Error('Unknown environment for listenBeatSyncBeatSignal()')
}
export async function getBeatSyncInputByteFrequencyData() {
if (isStageWeb()) {
return getDetector().getInputByteFrequencyData()
}
if (isStageTamagotchi()) {
return defineInvoke(getContext(), beatSyncGetInputByteFrequencyDataInvokeEventa)()
}
throw new Error('Unknown environment for getBeatSyncInputByteFrequencyData()')
}
packages/stage-shared/src/beat-sync/eventa.ts
+1
View File
@@ -8,6 +8,7 @@ import { defineInvokeEventa } from '@moeru/eventa'
export const beatSyncToggleInvokeEventa = defineInvokeEventa<void, boolean>('eventa:invoke:electron:beat-sync:toggle')
export const beatSyncGetStateInvokeEventa = defineInvokeEventa<BeatSyncDetectorState>('eventa:invoke:electron:beat-sync:get-state')
export const beatSyncUpdateParametersInvokeEventa = defineInvokeEventa<void, Partial<AnalyserWorkletParameters>>('eventa:event:electron:beat-sync:update-parameters')
export const beatSyncGetInputByteFrequencyDataInvokeEventa = defineInvokeEventa<Uint8Array<ArrayBuffer>>('eventa:invoke:electron:beat-sync:get-input-byte-frequency-data')
// Events
export const beatSyncStateChangedInvokeEventa = defineInvokeEventa<void, BeatSyncDetectorState>('eventa:event:electron:beat-sync:state-changed')