From c74f3352531bcb64e4ebfe3c1a3762fdb79c2127 Mon Sep 17 00:00:00 2001 From: Lilia_Chen Date: Wed, 17 Dec 2025 02:50:06 +0000 Subject: [PATCH] feat(stage-ui-three-performance-runtime): init new experimental three performance runtime design (#809) --- apps/stage-web/package.json | 1 + .../pages/devtools/performance-playground.vue | 311 ++++++++++++++++++ .../src/pages/settings/system/developer.vue | 6 + .../package.json | 30 ++ .../src/blackboard.ts | 137 ++++++++ .../src/index.ts | 2 + .../src/types.ts | 71 ++++ .../tsconfig.json | 11 + packages/stage-ui/package.json | 3 +- pnpm-lock.yaml | 16 + 10 files changed, 587 insertions(+), 1 deletion(-) create mode 100644 apps/stage-web/src/pages/devtools/performance-playground.vue create mode 100644 packages/stage-ui-three-performance-runtime/package.json create mode 100644 packages/stage-ui-three-performance-runtime/src/blackboard.ts create mode 100644 packages/stage-ui-three-performance-runtime/src/index.ts create mode 100644 packages/stage-ui-three-performance-runtime/src/types.ts create mode 100644 packages/stage-ui-three-performance-runtime/tsconfig.json diff --git a/apps/stage-web/package.json b/apps/stage-web/package.json index a98a17099..54909eb32 100644 --- a/apps/stage-web/package.json +++ b/apps/stage-web/package.json @@ -32,6 +32,7 @@ "@proj-airi/server-sdk": "workspace:^", "@proj-airi/stage-ui": "workspace:^", "@proj-airi/stage-ui-three": "workspace:^", + "@proj-airi/stage-ui-three-performance-runtime": "workspace:^", "@proj-airi/ui": "workspace:^", "@proj-airi/ui-transitions": "workspace:^", "@standard-schema/spec": "^1.0.0", diff --git a/apps/stage-web/src/pages/devtools/performance-playground.vue b/apps/stage-web/src/pages/devtools/performance-playground.vue new file mode 100644 index 000000000..4d84f13f2 --- /dev/null +++ b/apps/stage-web/src/pages/devtools/performance-playground.vue @@ -0,0 +1,311 @@ + + + + + +meta: + layout: settings + diff --git a/apps/stage-web/src/pages/settings/system/developer.vue b/apps/stage-web/src/pages/settings/system/developer.vue index aa5628de6..a104e4736 100644 --- a/apps/stage-web/src/pages/settings/system/developer.vue +++ b/apps/stage-web/src/pages/settings/system/developer.vue @@ -68,6 +68,12 @@ const menu = computed(() => [ icon: 'i-solar:sledgehammer-bold-duotone', to: '/devtools/providers-transcription-realtime-aliyun-nls', }, + { + title: 'Performance Playground', + description: 'Test performance runtime: VRM expressions + TTS lip sync', + icon: 'i-solar:sledgehammer-bold-duotone', + to: '/devtools/performance-playground', + }, ]) diff --git a/packages/stage-ui-three-performance-runtime/package.json b/packages/stage-ui-three-performance-runtime/package.json new file mode 100644 index 000000000..dff0a4e74 --- /dev/null +++ b/packages/stage-ui-three-performance-runtime/package.json @@ -0,0 +1,30 @@ +{ + "name": "@proj-airi/stage-ui-three-performance-runtime", + "type": "module", + "private": true, + "description": "Runtime timeline + blackboard utilities for stage-ui-three performances", + "author": { + "name": "Moeru AI Project AIRI Team", + "email": "airi@moeru.ai", + "url": "https://github.com/moeru-ai" + }, + "license": "MIT", + "repository": { + "type": "git", + "url": "https://github.com/moeru-ai/airi.git", + "directory": "packages/stage-ui-three-performance-runtime" + }, + "exports": { + ".": "./src/index.ts" + }, + "scripts": { + "typecheck": "tsc --noEmit -p tsconfig.json" + }, + "dependencies": { + "pinia": "^3.0.4", + "vue": "^3.5.25" + }, + "devDependencies": { + "typescript": "^5.6.3" + } +} diff --git a/packages/stage-ui-three-performance-runtime/src/blackboard.ts b/packages/stage-ui-three-performance-runtime/src/blackboard.ts new file mode 100644 index 000000000..1069dc7bb --- /dev/null +++ b/packages/stage-ui-three-performance-runtime/src/blackboard.ts @@ -0,0 +1,137 @@ +import { defineStore } from 'pinia' +import { computed, ref } from 'vue' + +export interface BlackboardGazeTarget { + type: 'screen' | 'world' + position?: { x: number, y: number, z?: number } +} + +export interface BlackboardSpeechState { + speaking: boolean + lastText?: string + lastSpecial?: string + playheadMs?: number +} + +export interface BlackboardEmotionState { + tag: string | null + intensity: number +} + +export interface BlackboardPoseState { + current: string | null + weight?: number + layer?: string +} + +export interface BlackboardActorState { + emotion: BlackboardEmotionState + expression: string | null + gaze: BlackboardGazeTarget | null + pose: BlackboardPoseState + speech: BlackboardSpeechState + flags: Record + markers: Record +} + +function createDefaultActorState(): BlackboardActorState { + return { + emotion: { tag: null, intensity: 0 }, + expression: null, + gaze: null, + pose: { current: null, weight: 0 }, + speech: { speaking: false }, + flags: {}, + markers: {}, + } +} + +export const useBlackboardStore = defineStore('stage-performance-blackboard', () => { + const actors = ref>({ + default: createDefaultActorState(), + }) + + const activeActorId = ref('default') + + const activeActor = computed(() => actors.value[activeActorId.value] ?? actors.value.default) + + function ensureActor(id: string) { + if (!actors.value[id]) + actors.value[id] = createDefaultActorState() + } + + function setActiveActor(id: string) { + ensureActor(id) + activeActorId.value = id + } + + function updateEmotion(id: string, emotion: Partial) { + ensureActor(id) + actors.value[id].emotion = { ...actors.value[id].emotion, ...emotion } + } + + function updateExpression(id: string, expression: string | null) { + ensureActor(id) + actors.value[id].expression = expression + } + + function updateGaze(id: string, gaze: BlackboardGazeTarget | null) { + ensureActor(id) + actors.value[id].gaze = gaze + } + + function updatePose(id: string, pose: Partial) { + ensureActor(id) + actors.value[id].pose = { ...actors.value[id].pose, ...pose } + } + + function updateSpeech(id: string, speech: Partial) { + ensureActor(id) + actors.value[id].speech = { ...actors.value[id].speech, ...speech } + } + + function setFlag(id: string, key: string, value: string | number | boolean) { + ensureActor(id) + actors.value[id].flags[key] = value + } + + function clearFlag(id: string, key: string) { + if (!actors.value[id]) + return + delete actors.value[id].flags[key] + } + + function setMarker(id: string, key: string, timestampMs: number) { + ensureActor(id) + actors.value[id].markers[key] = timestampMs + } + + function clearActor(id: string) { + if (!actors.value[id]) + return + actors.value[id] = createDefaultActorState() + } + + function resetAll() { + actors.value = { default: createDefaultActorState() } + activeActorId.value = 'default' + } + + return { + actors, + activeActorId, + activeActor, + + setActiveActor, + updateEmotion, + updateExpression, + updateGaze, + updatePose, + updateSpeech, + setFlag, + clearFlag, + setMarker, + clearActor, + resetAll, + } +}) diff --git a/packages/stage-ui-three-performance-runtime/src/index.ts b/packages/stage-ui-three-performance-runtime/src/index.ts new file mode 100644 index 000000000..0c084b30a --- /dev/null +++ b/packages/stage-ui-three-performance-runtime/src/index.ts @@ -0,0 +1,2 @@ +export * from './blackboard' +export * from './types' diff --git a/packages/stage-ui-three-performance-runtime/src/types.ts b/packages/stage-ui-three-performance-runtime/src/types.ts new file mode 100644 index 000000000..d308a9026 --- /dev/null +++ b/packages/stage-ui-three-performance-runtime/src/types.ts @@ -0,0 +1,71 @@ +// Core event/timeline contracts for the performance runtime; kept UI-agnostic so +// adapters (VRM, audio, devtools) can build on top without pulling stage-ui in. + +export type PerformanceEventKind = 'speech' | 'viseme' | 'expression' | 'body' | 'marker' + +export interface SpeechEventPayload { + text: string + buffer?: AudioBuffer + durationMs?: number + special?: string | null +} + +export interface VisemeEventPayload { + id: string + weight?: number + durationMs?: number +} + +export interface ExpressionEventPayload { + name: string + weight?: number + durationMs?: number +} + +export interface BodyEventPayload { + clip: string + fadeMs?: number + durationMs?: number +} + +export interface MarkerEventPayload { + key: string +} + +interface PerformanceEventPayloadMap { + speech: SpeechEventPayload + viseme: VisemeEventPayload + expression: ExpressionEventPayload + body: BodyEventPayload + marker: MarkerEventPayload +} + +export type PerformanceEventPayload = PerformanceEventPayloadMap[keyof PerformanceEventPayloadMap] + +export interface PerformanceEvent { + id: string + kind: T + t: number // milliseconds since timeline start + payload: PerformanceEventPayloadMap[T] +} + +export interface PerformanceTrack { + id: string + type: PerformanceEventKind + events: PerformanceEvent[] +} + +export interface PerformanceTimeline { + id: string + lengthMs?: number + tracks: PerformanceTrack[] +} + +export interface SchedulerHandlers { + onSpeechStart?: (event: PerformanceEvent<'speech'>) => void + onSpeechEnd?: (event: PerformanceEvent<'speech'>) => void + onViseme?: (event: PerformanceEvent<'viseme'>) => void + onExpression?: (event: PerformanceEvent<'expression'>) => void + onBody?: (event: PerformanceEvent<'body'>) => void + onMarker?: (event: PerformanceEvent<'marker'>) => void +} diff --git a/packages/stage-ui-three-performance-runtime/tsconfig.json b/packages/stage-ui-three-performance-runtime/tsconfig.json new file mode 100644 index 000000000..b67e12719 --- /dev/null +++ b/packages/stage-ui-three-performance-runtime/tsconfig.json @@ -0,0 +1,11 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "composite": false, + "rootDir": "src", + "outDir": "dist" + }, + "include": [ + "src" + ] +} diff --git a/packages/stage-ui/package.json b/packages/stage-ui/package.json index 01e2ca0cf..379cd9c99 100644 --- a/packages/stage-ui/package.json +++ b/packages/stage-ui/package.json @@ -32,7 +32,8 @@ "./workers/vad": "./src/workers/vad/index.ts", "./workers/*": "./src/workers/*.ts", "./workers": "./src/workers/index.ts", - "./utils": "./src/utils/index.ts" + "./utils": "./src/utils/index.ts", + "./utils/tts": "./src/utils/tts.ts" }, "scripts": { "typecheck": "vue-tsc --noEmit", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 23e10caf3..80a04f597 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -720,6 +720,9 @@ importers: '@proj-airi/stage-ui-three': specifier: workspace:^ version: link:../../packages/stage-ui-three + '@proj-airi/stage-ui-three-performance-runtime': + specifier: workspace:^ + version: link:../../packages/stage-ui-three-performance-runtime '@proj-airi/ui': specifier: workspace:^ version: link:../../packages/ui @@ -1811,6 +1814,19 @@ importers: specifier: ^3.1.8 version: 3.1.8(typescript@5.9.3) + packages/stage-ui-three-performance-runtime: + dependencies: + pinia: + specifier: ^3.0.4 + version: 3.0.4(typescript@5.9.3)(vue@3.5.25(typescript@5.9.3)) + vue: + specifier: ^3.5.25 + version: 3.5.25(typescript@5.9.3) + devDependencies: + typescript: + specifier: ^5.6.3 + version: 5.9.3 + packages/tauri-plugin-mcp: dependencies: '@tauri-apps/api':