diff --git a/cspell.config.yaml b/cspell.config.yaml index 89cd5f0be..4f45a3e1e 100644 --- a/cspell.config.yaml +++ b/cspell.config.yaml @@ -176,6 +176,7 @@ words: - ndarray - Neko - nekomeowww + - nekopaw - neuri - Neuro - Neuro-sama diff --git a/packages/stage-shared/package.json b/packages/stage-shared/package.json index 440c06183..4d21951d4 100644 --- a/packages/stage-shared/package.json +++ b/packages/stage-shared/package.json @@ -22,6 +22,7 @@ "typecheck": "tsc --noEmit" }, "devDependencies": { - "@electron-toolkit/preload": "^3.0.2" + "@electron-toolkit/preload": "^3.0.2", + "@nekopaw/tempora": "0.3.1-alpha.1" } } diff --git a/packages/stage-shared/src/beat-sync.ts b/packages/stage-shared/src/beat-sync.ts new file mode 100644 index 000000000..77aacc777 --- /dev/null +++ b/packages/stage-shared/src/beat-sync.ts @@ -0,0 +1,191 @@ +import type { Analyser, AnalyserBeatEvent, AnalyserWorkletParameters } from '@nekopaw/tempora' + +import analyserWorklet from '@nekopaw/tempora/worklet?url' + +import { startAnalyser as startTemporaAnalyser } from '@nekopaw/tempora' + +import { StageEnvironment } from './environment' + +export interface DetectorEventMap { + stateChange: (isActive: boolean) => void + beat: (e: AnalyserBeatEvent) => void +} + +interface Detector { + start: (createSource: (context: AudioContext) => Promise) => Promise + updateParameters: (params: Partial) => void + startScreenCapture: () => Promise + stop: () => void + on: (event: E, listener: DetectorEventMap[E]) => void + off: (event: E, listener: DetectorEventMap[E]) => void + readonly isActive: boolean + readonly context: AudioContext | undefined + readonly analyser: Analyser | undefined + readonly source: AudioNode | undefined +} + +type CreateDetectorOptions = | + { + env: StageEnvironment.Tamagotchi + enableLoopbackAudio: () => Promise + disableLoopbackAudio: () => Promise + } + | { + env: StageEnvironment.Web + } + +export function createDetector(options: CreateDetectorOptions): Detector { + let context: AudioContext | undefined + let analyser: Analyser | undefined + let source: AudioNode | undefined + let isActive = false + + let stopSource: (() => void) | undefined + + const listeners: { [K in keyof DetectorEventMap]: Array<(...args: any) => void> } = { + stateChange: [], + beat: [], + } + + const emit = (event: E, ...args: Parameters) => { + listeners[event].forEach(listener => listener(...args)) + } + + const stop = () => { + if (!isActive) + return + + isActive = false + emit('stateChange', isActive) + stopSource?.() + stopSource = undefined + + source?.disconnect() + source = undefined + + analyser?.stop() + analyser = undefined + + context?.close() + context = undefined + } + + const start = async (createSource: (context: AudioContext) => Promise) => { + stop() + + context = new AudioContext() + analyser = await startTemporaAnalyser({ + context, + worklet: analyserWorklet, + listeners: { + onBeat: e => emit('beat', e), + }, + }) + + const node = await createSource(context) + node.connect(analyser.workletNode) + source = node + + isActive = true + emit('stateChange', isActive) + } + + const updateParameters = (params: Partial) => { + analyser?.updateParameters(params) + } + + const startScreenCapture = async () => start(async (ctx) => { + switch (options.env) { + case StageEnvironment.Web: { + const stream = await navigator.mediaDevices.getDisplayMedia({ + audio: { + echoCancellation: false, + noiseSuppression: false, + autoGainControl: false, + }, + video: true, + }) + + if (stream.getAudioTracks().length === 0) { + throw new Error('No audio track available in the stream') + } + + stream.getAudioTracks().forEach((track) => { + let stopCalled = false + track.addEventListener('ended', () => { + if (stopCalled) + return + stopCalled = true + stop() + }) + }) + + const node = ctx.createMediaStreamSource(stream) + stopSource = () => { + stream.getTracks().forEach(track => track.stop()) + } + + return node + } + case StageEnvironment.Tamagotchi: { + await options.enableLoopbackAudio() + + const stream = await navigator.mediaDevices.getDisplayMedia({ + video: true, + audio: true, + }) + + const videoTracks = stream.getVideoTracks() + + videoTracks.forEach((track) => { + track.stop() + stream.removeTrack(track) + }) + + const node = ctx.createMediaStreamSource(stream) + stopSource = () => { + stream.getTracks().forEach(track => track.stop()) + options.disableLoopbackAudio() + } + await options.disableLoopbackAudio() + + return node + } + default: + throw new Error('Failed to start screen capture: Unsupported environment') + } + }) + + return { + start, + updateParameters, + startScreenCapture, + stop, + on: (event: E, listener: DetectorEventMap[E]) => { + switch (event) { + case 'beat': + listeners.beat.push(listener) + break + default: + throw new Error(`Unknown event: ${event}`) + } + }, + off: (event, listener) => { + switch (event) { + case 'beat': { + const index = listeners.beat.indexOf(listener) + if (index !== -1) + listeners.beat.splice(index, 1) + break + } + default: + throw new Error(`Unknown event: ${event}`) + } + }, + + get isActive() { return isActive }, + get context() { return context }, + get analyser() { return analyser }, + get source() { return source }, + } +} diff --git a/packages/stage-shared/src/environment.ts b/packages/stage-shared/src/environment.ts index 61149b1c7..8269eea76 100644 --- a/packages/stage-shared/src/environment.ts +++ b/packages/stage-shared/src/environment.ts @@ -1,3 +1,8 @@ +export enum StageEnvironment { + Web = 'web', + Tamagotchi = 'tamagotchi', +} + export function isStageWeb(): boolean { return !import.meta.env.RUNTIME_ENVIRONMENT || import.meta.env.RUNTIME_ENVIRONMENT === 'browser' } diff --git a/packages/stage-shared/src/index.ts b/packages/stage-shared/src/index.ts index a7f7eafbc..a10527f6d 100644 --- a/packages/stage-shared/src/index.ts +++ b/packages/stage-shared/src/index.ts @@ -1,3 +1,4 @@ +export * from './beat-sync' export * from './environment' export * from './url' export * from './window'