diff --git a/apps/server/vitest.config.ts b/apps/server/vitest.config.ts index 2caa75aa8..7b4008b72 100644 --- a/apps/server/vitest.config.ts +++ b/apps/server/vitest.config.ts @@ -3,7 +3,10 @@ import { defineConfig } from 'vitest/config' export default defineConfig({ test: { environment: 'node', + fileParallelism: false, globals: true, + hookTimeout: 60_000, + maxWorkers: 1, coverage: { provider: 'v8', include: [ diff --git a/apps/stage-tamagotchi/package.json b/apps/stage-tamagotchi/package.json index 075d54719..32383ea2a 100644 --- a/apps/stage-tamagotchi/package.json +++ b/apps/stage-tamagotchi/package.json @@ -69,6 +69,7 @@ "@proj-airi/stage-pages": "workspace:^", "@proj-airi/stage-ui": "workspace:^", "@proj-airi/stage-ui-live2d": "workspace:^", + "@proj-airi/stage-ui-mmd": "workspace:^", "@proj-airi/stage-ui-spine": "workspace:^", "@proj-airi/stage-ui-three": "workspace:^", "@proj-airi/ui": "workspace:^", diff --git a/apps/stage-tamagotchi/src/renderer/pages/index.vue b/apps/stage-tamagotchi/src/renderer/pages/index.vue index 5f6268969..d7a2889c0 100644 --- a/apps/stage-tamagotchi/src/renderer/pages/index.vue +++ b/apps/stage-tamagotchi/src/renderer/pages/index.vue @@ -190,6 +190,20 @@ const modelSettingsRuntimeSnapshot = computed(() = }) } + if (stageModelRenderer.value === 'mmd') { + const phase = resolveComponentStateToRuntimePhase(componentStateStage.value, { hasModel }) + + return createEmptyModelSettingsRuntimeSnapshot({ + ownerInstanceId: modelSettingsRuntimeOwnerInstanceId, + renderer: 'mmd', + phase, + controlsLocked: hasModel ? phase !== 'mounted' : false, + previewAvailable: hasModel, + canCapturePreview: false, + updatedAt: Date.now(), + }) + } + if (stageModelRenderer.value === 'godot') { return createEmptyModelSettingsRuntimeSnapshot({ ownerInstanceId: modelSettingsRuntimeOwnerInstanceId, diff --git a/apps/stage-tamagotchi/vitest.config.ts b/apps/stage-tamagotchi/vitest.config.ts index e7abe03e3..850f622a9 100644 --- a/apps/stage-tamagotchi/vitest.config.ts +++ b/apps/stage-tamagotchi/vitest.config.ts @@ -13,5 +13,7 @@ export default defineConfig({ env: loadEnv('test', cwd(), ''), include: ['src/**/*.test.ts', 'scripts/**/*.test.ts'], exclude: ['**/node_modules/**', '**/.git/**'], + fileParallelism: false, + maxWorkers: 1, }, }) diff --git a/packages/i18n/src/locales/en/settings.yaml b/packages/i18n/src/locales/en/settings.yaml index 6ef389755..f58dbb9d1 100644 --- a/packages/i18n/src/locales/en/settings.yaml +++ b/packages/i18n/src/locales/en/settings.yaml @@ -119,7 +119,8 @@ model-select: support-status-header: We support both 2D and 3D models support-status: > Click {select-button} to select or import models. - Currently, {zip} (for Live2D and Spine models) and {vrm} (for VRM models) are supported. + Currently, {zip} (for Live2D and Spine models), {vrm} (for VRM models), + and {mmd} (for MMD models, zipped with their textures) are supported. model-type-example: > Neuro-sama uses 2D model driven by framework developed by Live2D Inc, while Grok Ani (in Grok Companion) uses 3D model driven by VRM / MMD open formats. @@ -432,7 +433,7 @@ pages: delete: Delete all chat sessions models: title: Models - description: Remove imported Live2D/VRM models. + description: Remove imported Live2D, VRM, Spine, and MMD models. delete: Delete all models modules: title: Modules @@ -473,7 +474,7 @@ pages: desktop_window_centered: Desktop window moved to screen center. desktop_reset: Desktop data reset. models: - description: Live2D, VRM, Spine, etc. + description: Live2D, VRM, Spine, MMD, etc. title: Models sections: section: @@ -1757,6 +1758,55 @@ spine: fps: options: unlimited: Unlimited +mmd: + title: MMD Settings + scale-and-position: + title: Scale And Position + scale: Scale + x: X + 'y': 'Y' + rotation: Rotation + theme-color-from-model: + title: Extract colors from model + button-extract: + title: Extract + physics: + title: Physics & Rig + enabled: Enable physics (hair, skirt, accessories) + ik: Enable IK (inverse kinematics) + grant: Enable append-bone propagation + gravity: Gravity strength + gaze: + title: Gaze + tracking: Looking at + options: + cursor: Mouse + camera: Camera + none: Disabled + animation: + title: Animation + idle-motion: Idle Motion + import-motion: Import Motion (.vmd) + play-once: Preview Animation + remove: Remove motion + description: >- + Import VMD motion files, set one as the looping idle, or play any as a + one-shot gesture. Map emotions to motions to play gestures alongside + facial expressions. + morphs: + title: Morphs + auto: (auto) + revert: Revert to auto + no-morphs: This model exposes no morphs. + description: >- + The renderer auto-detects standard Japanese morph names (あいうえお, + まばたき). Override a slot here if your model uses non-standard names. + materials: + title: Materials + no-materials: This model exposes no materials. + description: >- + Adjust the opacity of individual model parts (face, hair, clothing, + accessories). Set a part to 0 to hide it. websocket-secure-enabled: title: Enable Secure WebSocket (WSS) description: >- diff --git a/packages/i18n/src/locales/zh-Hans/settings.yaml b/packages/i18n/src/locales/zh-Hans/settings.yaml index b2b2cb729..04d5d269d 100644 --- a/packages/i18n/src/locales/zh-Hans/settings.yaml +++ b/packages/i18n/src/locales/zh-Hans/settings.yaml @@ -454,7 +454,7 @@ pages: desktop_window_centered: Desktop window moved to screen center. desktop_reset: 桌面数据重置。 models: - description: Live2D, VRM, Spine, etc. + description: 切换角色的 Live2D、VRM、Spine 模型 title: 角色模型 sections: section: @@ -1674,7 +1674,7 @@ spine: mix-duration: Mix Duration (seconds) speed: Animation Speed appearance: - title: Appearance + title: 外观 variant: title: Variant current-variant: Active Variant diff --git a/packages/i18n/src/locales/zh-Hant/settings.yaml b/packages/i18n/src/locales/zh-Hant/settings.yaml index 63f24dd92..ae42279cb 100644 --- a/packages/i18n/src/locales/zh-Hant/settings.yaml +++ b/packages/i18n/src/locales/zh-Hant/settings.yaml @@ -454,7 +454,7 @@ pages: desktop_window_centered: Desktop window moved to screen center. desktop_reset: 桌面資料已重設 models: - description: Live2D, VRM, Spine, etc. + description: 切換角色的 Live2D、VRM、Spine 模型 title: 角色模型 sections: section: @@ -1674,7 +1674,7 @@ spine: mix-duration: Mix Duration (seconds) speed: Animation Speed appearance: - title: Appearance + title: 外觀 variant: title: Variant current-variant: Active Variant diff --git a/packages/stage-pages/package.json b/packages/stage-pages/package.json index 9aa302346..e70e036fd 100644 --- a/packages/stage-pages/package.json +++ b/packages/stage-pages/package.json @@ -29,6 +29,7 @@ "@proj-airi/server-sdk": "workspace:*", "@proj-airi/stage-ui": "workspace:*", "@proj-airi/stage-ui-live2d": "workspace:*", + "@proj-airi/stage-ui-mmd": "workspace:*", "@proj-airi/stage-ui-spine": "workspace:*", "@proj-airi/stage-ui-three": "workspace:*", "@proj-airi/ui": "workspace:*", diff --git a/packages/stage-ui-mmd/README.md b/packages/stage-ui-mmd/README.md new file mode 100644 index 000000000..74edb7358 --- /dev/null +++ b/packages/stage-ui-mmd/README.md @@ -0,0 +1,72 @@ +# @proj-airi/stage-ui-mmd + +MMD (MikuMikuDance) scene components, composables, and stores for Project AIRI. + +## What it does + +Renders PMX/PMD models with VMD/VPD motion support inside AIRI's stage, +reaching feature parity with the Live2D and VRM renderers: + +- **Model loading** for `.pmx`/`.pmd`, from a packaged ZIP (model + textures) + or a bare model URL. +- **Physics** for hair, skirts, and custom rigs via Bullet (Ammo.js) — rigid + bodies and 6-DOF spring joints, toggleable at runtime. +- **IK** (CCD solver) and **append-bone ("grant")** propagation for standard + and custom rigs. +- **Importable, mappable motions**: import VMD clips (persisted to IndexedDB, + synced across windows), set one as the looping idle, preview any as a + one-shot, remove them, and map AIRI emotions to gesture motions. +- **Morph-based lip-sync** driven by the shared wLipSync profile, mapping + phonemes to the canonical あいうえお mouth morphs, with per-slot morph + remapping for non-standard models. +- **Emotion expressions** via morph cross-fades. +- **Looking-at modes** (camera / cursor / disabled) with idle saccades, plus + procedural blink. +- **Scene settings**: model transform, camera FOV, ambient + directional + lights, albedo glow, render scale, physics gravity, and per-material opacity + — all live, persisted, and synced across windows. + +It builds on [`three-stdlib`](https://github.com/pmndrs/three-stdlib) (the +maintained TypeScript port of three.js' `examples/jsm`) for `MMDLoader`, +`MMDAnimationHelper`, `MMDPhysics`, and `CCDIKSolver`, because upstream three +removed the first-party MMD modules in r168. Physics uses `ammojs-typed`, +loaded lazily so the WASM binary only ships once an MMD model is mounted. + +## How to use + +The package exposes the same scene contract as the other renderers, so it is +wired through `@proj-airi/stage-ui`'s `Stage.vue` automatically when the +selected model resolves to the `mmd` renderer: + +```ts +import { MMDScene, useMMD } from '@proj-airi/stage-ui-mmd' +``` + +```vue + +``` + +The component exposes `canvasElement()`, `captureFrame()`, +`setEmotion(name, intensity)`, `listMorphs()`, and `listMotions()`. + +Runtime configuration (physics/IK/grant toggles, gaze tracking, scale, +morph overrides, emotion→motion mapping) lives in the `useMMD` Pinia store and +is edited through the model settings panel. + +## When to use it + +- Displaying MMD/MikuMikuDance characters (`.pmx`/`.pmd`) on the AIRI stage. +- When you need physics-driven secondary motion, IK, and VMD playback. + +## When not to use it + +- For VRM avatars use `@proj-airi/stage-ui-three`; for Live2D use + `@proj-airi/stage-ui-live2d`; for Spine use `@proj-airi/stage-ui-spine`. +- In environments without WebGL/WASM support (MMD physics requires Ammo WASM). diff --git a/packages/stage-ui-mmd/package.json b/packages/stage-ui-mmd/package.json new file mode 100644 index 000000000..44de8c795 --- /dev/null +++ b/packages/stage-ui-mmd/package.json @@ -0,0 +1,57 @@ +{ + "name": "@proj-airi/stage-ui-mmd", + "type": "module", + "private": true, + "description": "MMD (MikuMikuDance) scene components and stores for Project AIRI", + "author": { + "name": "Moeru AI Project AIRI Team", + "email": "airi@moeru.ai", + "url": "https://github.com/moeru-ai" + }, + "license": "MIT", + "repository": { + "type": "git", + "url": "https://github.com/moeru-ai/airi.git", + "directory": "packages/stage-ui-mmd" + }, + "exports": { + ".": "./src/index.ts", + "./components": "./src/components/index.ts", + "./components/scenes": "./src/components/scenes/index.ts", + "./components/scenes/MMD.vue": "./src/components/scenes/MMD.vue", + "./composables/mmd": "./src/composables/mmd/index.ts", + "./constants/emotions": "./src/constants/emotions.ts", + "./stores": "./src/stores/index.ts", + "./stores/mmd": "./src/stores/mmd.ts", + "./utils/mmd-loader": "./src/utils/mmd-loader.ts", + "./utils/mmd-preview": "./src/utils/mmd-preview.ts", + "./utils/mmd-validator": "./src/utils/mmd-validator.ts", + "./utils/mmd-zip-loader": "./src/utils/mmd-zip-loader.ts" + }, + "scripts": { + "typecheck": "vue-tsc --noEmit" + }, + "dependencies": { + "@moeru/std": "catalog:", + "@proj-airi/stage-shared": "workspace:^", + "@proj-airi/ui": "workspace:^", + "@vueuse/core": "catalog:", + "@xsai/tool": "catalog:", + "ammojs-typed": "catalog:", + "culori": "catalog:", + "es-toolkit": "catalog:", + "jszip": "catalog:", + "localforage": "catalog:", + "pinia": "catalog:", + "three": "catalog:", + "three-stdlib": "catalog:", + "vue": "catalog:", + "wlipsync": "catalog:" + }, + "devDependencies": { + "@types/culori": "catalog:", + "@types/three": "catalog:", + "vitest": "catalog:vitest", + "vue-tsc": "catalog:" + } +} diff --git a/packages/stage-ui-mmd/src/assets/lip-sync-profile.json b/packages/stage-ui-mmd/src/assets/lip-sync-profile.json new file mode 100644 index 000000000..879765368 --- /dev/null +++ b/packages/stage-ui-mmd/src/assets/lip-sync-profile.json @@ -0,0 +1 @@ +{ "jsonPath": "/home/steamvr/projects/wLipSync/www/profile.json", "mfccNum": 12, "mfccDataCount": 12, "melFilterBankChannels": 30, "targetSampleRate": 16000, "sampleCount": 1024, "useStandardization": false, "compareMethod": 2, "mfccs": [{ "name": "A", "mfccCalibrationDataList": [{ "array": [94.40318298339844, 0.32245922088623049, -65.5116195678711, -29.537851333618165, 4.888294219970703, 14.523965835571289, -32.6411247253418, 1.6505765914916993, -9.960077285766602, -5.7025322914123539, -11.886154174804688, -24.35236358642578] }, { "array": [94.40318298339844, 0.32245922088623049, -65.5116195678711, -29.537851333618165, 4.888294219970703, 14.523965835571289, -32.6411247253418, 1.6505765914916993, -9.960077285766602, -5.7025322914123539, -11.886154174804688, -24.35236358642578] }, { "array": [102.16287231445313, -3.3587560653686525, -65.58428192138672, -25.24440574645996, 3.224522590637207, 12.005892753601075, -29.293079376220704, 0.6378564834594727, -10.817683219909668, -1.3263540267944337, -14.543159484863282, -24.169780731201173] }, { "array": [99.35592651367188, -3.681424140930176, -65.20439910888672, -23.45950698852539, 6.205645561218262, 14.96288013458252, -29.882709503173829, 0.6733551025390625, -7.077619552612305, -3.5570802688598635, -14.427347183227539, -23.340003967285158] }, { "array": [99.35592651367188, -3.681424140930176, -65.20439910888672, -23.45950698852539, 6.205645561218262, 14.96288013458252, -29.882709503173829, 0.6733551025390625, -7.077619552612305, -3.5570802688598635, -14.427347183227539, -23.340003967285158] }, { "array": [104.24951171875, -2.8328847885131838, -66.59016418457031, -22.962886810302736, 5.519782066345215, 16.50394058227539, -32.338768005371097, 5.820473670959473, -10.59586238861084, -2.7398462295532228, -12.5281400680542, -24.459365844726564] }, { "array": [104.24951171875, -2.8328847885131838, -66.59016418457031, -22.962886810302736, 5.519782066345215, 16.50394058227539, -32.338768005371097, 5.820473670959473, -10.59586238861084, -2.7398462295532228, -12.5281400680542, -24.459365844726564] }, { "array": [95.58644104003906, -5.775191307067871, -61.220008850097659, -24.658382415771486, 4.4112701416015629, 13.673284530639649, -25.223039627075197, -2.0546646118164064, -6.887641906738281, -5.683987617492676, -11.20918083190918, -23.215322494506837] }, { "array": [98.24864196777344, -3.8367862701416017, -62.34006118774414, -24.563793182373048, 4.608433723449707, 16.228965759277345, -28.992279052734376, 2.1237001419067385, -9.07174015045166, -4.581008434295654, -10.662440299987793, -26.19581413269043] }, { "array": [102.7921142578125, -4.580304145812988, -62.531837463378909, -26.292770385742189, 7.911410331726074, 17.136384963989259, -31.118263244628908, 5.196089744567871, -10.010396957397461, -0.8527965545654297, -12.346561431884766, -23.580944061279298] }, { "array": [102.12345123291016, -1.5254135131835938, -62.21220397949219, -26.728734970092775, 10.62057876586914, 16.918357849121095, -28.815664291381837, 3.6714258193969728, -9.673786163330079, -0.7385025024414063, -9.717185020446778, -27.09702491760254] }, { "array": [102.12345123291016, -1.5254135131835938, -62.21220397949219, -26.728734970092775, 10.62057876586914, 16.918357849121095, -28.815664291381837, 3.6714258193969728, -9.673786163330079, -0.7385025024414063, -9.717185020446778, -27.09702491760254] }] }, { "name": "I", "mfccCalibrationDataList": [{ "array": [14.441835403442383, 52.400115966796878, 57.377838134765628, -33.70046615600586, -12.751934051513672, -15.709930419921875, -41.381065368652347, -12.190519332885743, -2.863154411315918, -8.727733612060547, 2.6656012535095217, 1.4855976104736329] }, { "array": [14.441835403442383, 52.400115966796878, 57.377838134765628, -33.70046615600586, -12.751934051513672, -15.709930419921875, -41.381065368652347, -12.190519332885743, -2.863154411315918, -8.727733612060547, 2.6656012535095217, 1.4855976104736329] }, { "array": [15.294279098510743, 50.07628631591797, 57.262847900390628, -31.748844146728517, -13.642471313476563, -13.48408031463623, -41.535011291503909, -16.863862991333009, -1.739903450012207, -9.32723331451416, 8.31618881225586, -1.779850959777832] }, { "array": [15.685098648071289, 49.68647766113281, 58.447113037109378, -32.513519287109378, -16.664287567138673, -13.78364372253418, -40.48309326171875, -16.04582405090332, -3.5356569290161135, -8.654275894165039, 10.645575523376465, -2.556441307067871] }, { "array": [15.685098648071289, 49.68647766113281, 58.447113037109378, -32.513519287109378, -16.664287567138673, -13.78364372253418, -40.48309326171875, -16.04582405090332, -3.5356569290161135, -8.654275894165039, 10.645575523376465, -2.556441307067871] }, { "array": [15.685098648071289, 49.68647766113281, 58.447113037109378, -32.513519287109378, -16.664287567138673, -13.78364372253418, -40.48309326171875, -16.04582405090332, -3.5356569290161135, -8.654275894165039, 10.645575523376465, -2.556441307067871] }, { "array": [15.225826263427735, 43.872196197509769, 52.844512939453128, -32.51786804199219, -17.806241989135743, -10.609650611877442, -40.13084411621094, -11.58648681640625, -5.082568168640137, -14.396997451782227, 6.896979331970215, -0.785430908203125] }, { "array": [15.571629524230957, 48.634883880615237, 59.5339469909668, -34.59955596923828, -21.959871292114259, -10.298498153686524, -39.11286926269531, -11.998537063598633, -8.433327674865723, -10.80599594116211, 8.789299011230469, -1.884697437286377] }, { "array": [15.571629524230957, 48.634883880615237, 59.5339469909668, -34.59955596923828, -21.959871292114259, -10.298498153686524, -39.11286926269531, -11.998537063598633, -8.433327674865723, -10.80599594116211, 8.789299011230469, -1.884697437286377] }, { "array": [15.571629524230957, 48.634883880615237, 59.5339469909668, -34.59955596923828, -21.959871292114259, -10.298498153686524, -39.11286926269531, -11.998537063598633, -8.433327674865723, -10.80599594116211, 8.789299011230469, -1.884697437286377] }, { "array": [19.23290252685547, 47.433998107910159, 54.90937423706055, -33.7783203125, -13.836353302001954, -5.141571044921875, -39.34584045410156, -13.409493446350098, -4.945652008056641, -12.960502624511719, 12.210061073303223, 0.5807018280029297] }, { "array": [19.124774932861329, 46.46723937988281, 53.41281509399414, -34.65093994140625, -18.181049346923829, -7.733134746551514, -45.67931365966797, -10.64135456085205, -2.624391555786133, -15.708955764770508, 7.3649187088012699, -5.627689361572266] }] }, { "name": "U", "mfccCalibrationDataList": [{ "array": [83.38372802734375, 42.39790725708008, 27.812450408935548, 10.696150779724121, -13.612553596496582, -32.487091064453128, -35.2574348449707, -6.425739288330078, -4.214997291564941, -6.896385669708252, -3.49631404876709, 4.997060775756836] }, { "array": [103.65653228759766, 38.661563873291019, 30.985050201416017, 17.432451248168947, -14.383820533752442, -39.810001373291019, -39.63761901855469, 1.2333955764770508, -4.217883110046387, -3.005303382873535, -6.272947311401367, 4.751875877380371] }, { "array": [103.44242095947266, 45.238216400146487, 27.622669219970704, 18.682138442993165, -16.854982376098634, -39.85029602050781, -34.15940856933594, -2.7482595443725588, -3.7410573959350588, -1.0625238418579102, -4.215768814086914, 6.514510154724121] }, { "array": [103.44242095947266, 45.238216400146487, 27.622669219970704, 18.682138442993165, -16.854982376098634, -39.85029602050781, -34.15940856933594, -2.7482595443725588, -3.7410573959350588, -1.0625238418579102, -4.215768814086914, 6.514510154724121] }, { "array": [99.99739837646485, 39.48998260498047, 26.057384490966798, 23.26814079284668, -16.25522804260254, -38.15496063232422, -35.70051574707031, -1.2821111679077149, -2.3941946029663088, -0.15543842315673829, -4.757769584655762, 2.198577880859375] }, { "array": [99.99739837646485, 39.48998260498047, 26.057384490966798, 23.26814079284668, -16.25522804260254, -38.15496063232422, -35.70051574707031, -1.2821111679077149, -2.3941946029663088, -0.15543842315673829, -4.757769584655762, 2.198577880859375] }, { "array": [107.09538269042969, 37.704010009765628, 17.795482635498048, 21.882326126098634, -14.739266395568848, -36.407527923583987, -37.95854949951172, -1.4393510818481446, -1.9593324661254883, -0.7294750213623047, -7.93386173248291, 3.9560585021972658] }, { "array": [103.5069351196289, 35.08988952636719, 23.21630859375, 23.947580337524415, -14.157055854797364, -38.546836853027347, -39.75208282470703, 0.516876220703125, -2.715259552001953, -4.0768208503723148, -4.716378211975098, 4.662134170532227] }, { "array": [103.5069351196289, 35.08988952636719, 23.21630859375, 23.947580337524415, -14.157055854797364, -38.546836853027347, -39.75208282470703, 0.516876220703125, -2.715259552001953, -4.0768208503723148, -4.716378211975098, 4.662134170532227] }, { "array": [103.5069351196289, 35.08988952636719, 23.21630859375, 23.947580337524415, -14.157055854797364, -38.546836853027347, -39.75208282470703, 0.516876220703125, -2.715259552001953, -4.0768208503723148, -4.716378211975098, 4.662134170532227] }, { "array": [98.43344116210938, 35.42580032348633, 29.2958984375, 24.73729133605957, -15.485936164855957, -44.676483154296878, -39.978858947753909, 0.5548343658447266, -2.2034664154052736, -3.485844612121582, -7.421210289001465, 6.30616569519043] }, { "array": [94.0390625, 36.81925582885742, 24.73573875427246, 22.579418182373048, -14.354126930236817, -37.92849349975586, -44.69046401977539, 0.7474861145019531, -3.3195743560791017, -3.9850082397460939, -5.991059303283691, 6.134122848510742] }] }, { "name": "E", "mfccCalibrationDataList": [{ "array": [60.52040481567383, 14.444153785705567, 50.91899108886719, 6.730878829956055, -58.12107467651367, -16.403745651245118, -25.244909286499025, 5.399906158447266, -7.63681697845459, -2.4964828491210939, 7.271292209625244, 1.7322711944580079] }, { "array": [59.327247619628909, 14.82846450805664, 51.402244567871097, 5.413976669311523, -55.603240966796878, -15.348665237426758, -25.923606872558595, 3.0006580352783205, -4.183259963989258, -3.4587841033935549, 7.941498756408691, 3.4499120712280275] }, { "array": [59.327247619628909, 14.82846450805664, 51.402244567871097, 5.413976669311523, -55.603240966796878, -15.348665237426758, -25.923606872558595, 3.0006580352783205, -4.183259963989258, -3.4587841033935549, 7.941498756408691, 3.4499120712280275] }, { "array": [64.61061096191406, 7.8438310623168949, 54.753726959228519, 11.154451370239258, -62.99680709838867, -10.397377967834473, -36.124359130859378, 12.57413387298584, -6.086113452911377, -3.032306671142578, 10.453157424926758, -0.00012826919555664063] }, { "array": [59.8906135559082, 13.646936416625977, 53.14240646362305, 11.346290588378907, -60.17724609375, -15.942718505859375, -29.547088623046876, 8.241331100463868, -6.8904523849487309, -3.6554131507873537, 14.714229583740235, -2.811859607696533] }, { "array": [53.77520751953125, 6.071747779846191, 47.870723724365237, 5.943275451660156, -51.11442184448242, -16.625276565551759, -24.842336654663087, 4.076478004455566, -8.835965156555176, -4.306196689605713, 13.907751083374024, -0.6555676460266113] }, { "array": [53.77520751953125, 6.071747779846191, 47.870723724365237, 5.943275451660156, -51.11442184448242, -16.625276565551759, -24.842336654663087, 4.076478004455566, -8.835965156555176, -4.306196689605713, 13.907751083374024, -0.6555676460266113] }, { "array": [53.77520751953125, 6.071747779846191, 47.870723724365237, 5.943275451660156, -51.11442184448242, -16.625276565551759, -24.842336654663087, 4.076478004455566, -8.835965156555176, -4.306196689605713, 13.907751083374024, -0.6555676460266113] }, { "array": [62.81553649902344, 6.203365325927734, 48.45057678222656, 8.571174621582032, -53.907508850097659, -16.376169204711915, -25.989578247070314, 5.736949920654297, -8.150140762329102, -5.895424842834473, 13.745902061462403, -4.22935676574707] }, { "array": [53.43303680419922, 7.019550323486328, 43.32084655761719, 7.639513969421387, -49.81471633911133, -18.708377838134767, -21.690540313720704, 0.34458446502685549, -8.689970970153809, -3.96992826461792, 11.29841423034668, -4.165286540985107] }, { "array": [56.29218673706055, 7.115049362182617, 47.741546630859378, 4.102975845336914, -50.46143341064453, -18.235626220703126, -22.557659149169923, 6.925202369689941, -13.170380592346192, -1.0300326347351075, 8.813325881958008, -3.2347850799560549] }, { "array": [56.29218673706055, 7.115049362182617, 47.741546630859378, 4.102975845336914, -50.46143341064453, -18.235626220703126, -22.557659149169923, 6.925202369689941, -13.170380592346192, -1.0300326347351075, 8.813325881958008, -3.2347850799560549] }] }, { "name": "O", "mfccCalibrationDataList": [{ "array": [108.12348937988281, 57.48288345336914, 0.23154354095458985, -31.144771575927736, -38.093109130859378, -18.33026885986328, -11.250101089477539, -0.7636222839355469, -1.371236801147461, -9.181392669677735, -1.6202507019042969, -7.501105308532715] }, { "array": [108.12348937988281, 57.48288345336914, 0.23154354095458985, -31.144771575927736, -38.093109130859378, -18.33026885986328, -11.250101089477539, -0.7636222839355469, -1.371236801147461, -9.181392669677735, -1.6202507019042969, -7.501105308532715] }, { "array": [120.48326873779297, 61.383270263671878, -2.6785545349121095, -32.3900146484375, -40.94635772705078, -12.681024551391602, -10.979912757873536, -0.7160100936889648, -2.9078426361083986, -12.300739288330079, 1.8719825744628907, -6.2853875160217289] }, { "array": [109.58128356933594, 58.46321105957031, 0.1479501724243164, -31.080230712890626, -34.73053741455078, -17.423336029052736, -10.221624374389649, 0.3863086700439453, -1.2579708099365235, -7.037452220916748, -2.894855499267578, -7.0971550941467289] }, { "array": [109.58128356933594, 58.46321105957031, 0.1479501724243164, -31.080230712890626, -34.73053741455078, -17.423336029052736, -10.221624374389649, 0.3863086700439453, -1.2579708099365235, -7.037452220916748, -2.894855499267578, -7.0971550941467289] }, { "array": [116.19681549072266, 60.3573112487793, 0.07350921630859375, -32.384727478027347, -34.47290802001953, -16.373615264892579, -14.695085525512696, 0.3102083206176758, 0.28844451904296877, -8.948881149291993, -3.104994773864746, -9.900266647338868] }, { "array": [116.19681549072266, 60.3573112487793, 0.07350921630859375, -32.384727478027347, -34.47290802001953, -16.373615264892579, -14.695085525512696, 0.3102083206176758, 0.28844451904296877, -8.948881149291993, -3.104994773864746, -9.900266647338868] }, { "array": [98.63448333740235, 48.661781311035159, 2.394181251525879, -28.785797119140626, -31.548860549926759, -18.37759017944336, -14.998208999633789, -1.8050260543823243, -2.018402099609375, -4.584748268127441, -5.160560607910156, -7.968695163726807] }, { "array": [124.25032043457031, 59.27610397338867, 2.7454710006713869, -36.72577667236328, -38.65552520751953, -3.116687774658203, -24.24558448791504, 0.5085678100585938, 2.3633852005004885, -10.51361083984375, 1.7447805404663087, -13.22685432434082] }, { "array": [87.89213562011719, 45.08750534057617, 4.292821884155273, -26.482845306396486, -30.386096954345704, -20.410654067993165, -11.817208290100098, -3.1270408630371095, -1.1370172500610352, -6.159217357635498, -3.454045295715332, -5.7265400886535648] }, { "array": [87.89213562011719, 45.08750534057617, 4.292821884155273, -26.482845306396486, -30.386096954345704, -20.410654067993165, -11.817208290100098, -3.1270408630371095, -1.1370172500610352, -6.159217357635498, -3.454045295715332, -5.7265400886535648] }, { "array": [87.89213562011719, 45.08750534057617, 4.292821884155273, -26.482845306396486, -30.386096954345704, -20.410654067993165, -11.817208290100098, -3.1270408630371095, -1.1370172500610352, -6.159217357635498, -3.454045295715332, -5.7265400886535648] }] }, { "name": "S", "mfccCalibrationDataList": [{ "array": [-94.29214477539063, 9.299236297607422, -21.6169376373291, 4.123956203460693, 1.3645498752593995, -2.339733839035034, 12.92388916015625, -7.490560531616211, 10.520170211791993, -5.4832611083984379, 2.1621110439300539, 2.6961774826049806] }, { "array": [-94.29214477539063, 9.299236297607422, -21.6169376373291, 4.123956203460693, 1.3645498752593995, -2.339733839035034, 12.92388916015625, -7.490560531616211, 10.520170211791993, -5.4832611083984379, 2.1621110439300539, 2.6961774826049806] }, { "array": [-92.13811492919922, 8.712703704833985, -17.194181442260743, 4.387561798095703, 2.288078784942627, -9.562786102294922, 9.854814529418946, -7.843216896057129, 8.969221115112305, -7.9670305252075199, 0.38271141052246096, 2.731431007385254] }, { "array": [-92.13811492919922, 8.712703704833985, -17.194181442260743, 4.387561798095703, 2.288078784942627, -9.562786102294922, 9.854814529418946, -7.843216896057129, 8.969221115112305, -7.9670305252075199, 0.38271141052246096, 2.731431007385254] }, { "array": [-90.87933349609375, 12.622742652893067, -12.25172233581543, 6.775156497955322, 6.892632007598877, -4.708589553833008, 10.558273315429688, -10.194192886352539, 10.907587051391602, 0.7259445190429688, 6.631969451904297, 1.2803831100463868] }, { "array": [-90.87933349609375, 12.622742652893067, -12.25172233581543, 6.775156497955322, 6.892632007598877, -4.708589553833008, 10.558273315429688, -10.194192886352539, 10.907587051391602, 0.7259445190429688, 6.631969451904297, 1.2803831100463868] }, { "array": [-94.86066436767578, 18.40726089477539, -5.981902599334717, 7.446142196655273, 11.998884201049805, 1.1316719055175782, 5.726372718811035, -9.411809921264649, 9.966836929321289, 0.6692547798156738, 3.0949947834014894, -0.5439543724060059] }, { "array": [-94.86066436767578, 18.40726089477539, -5.981902599334717, 7.446142196655273, 11.998884201049805, 1.1316719055175782, 5.726372718811035, -9.411809921264649, 9.966836929321289, 0.6692547798156738, 3.0949947834014894, -0.5439543724060059] }, { "array": [-94.86066436767578, 18.40726089477539, -5.981902599334717, 7.446142196655273, 11.998884201049805, 1.1316719055175782, 5.726372718811035, -9.411809921264649, 9.966836929321289, 0.6692547798156738, 3.0949947834014894, -0.5439543724060059] }, { "array": [-100.9681396484375, 15.002283096313477, -4.994745254516602, 0.22259855270385743, -0.15606117248535157, -1.8661277294158936, 1.5652005672454835, -13.30648422241211, 12.554527282714844, -2.990779399871826, 3.5510034561157228, 5.119507312774658] }, { "array": [-100.9681396484375, 15.002283096313477, -4.994745254516602, 0.22259855270385743, -0.15606117248535157, -1.8661277294158936, 1.5652005672454835, -13.30648422241211, 12.554527282714844, -2.990779399871826, 3.5510034561157228, 5.119507312774658] }, { "array": [-101.28047943115235, 14.962509155273438, -10.988410949707032, 3.6384878158569338, -1.4698257446289063, -4.758091449737549, -1.3547701835632325, -12.941855430603028, 3.3519961833953859, -5.5131611824035648, 9.386914253234864, 3.8310816287994386] }] }, { "name": "A", "mfccCalibrationDataList": [{ "array": [4.20286750793457, -73.493896484375, -24.746726989746095, -41.51460266113281, 36.48657989501953, -18.2531795501709, -42.99116516113281, 25.612823486328126, -18.336681365966798, -15.366691589355469, -4.867555618286133, -8.545194625854493] }, { "array": [6.645953178405762, -70.99688720703125, -23.49920082092285, -40.70307922363281, 35.09113311767578, -19.63579750061035, -41.851219177246097, 26.548370361328126, -20.361244201660158, -15.091900825500489, -5.332237243652344, -7.199653625488281] }, { "array": [6.645953178405762, -70.99688720703125, -23.49920082092285, -40.70307922363281, 35.09113311767578, -19.63579750061035, -41.851219177246097, 26.548370361328126, -20.361244201660158, -15.091900825500489, -5.332237243652344, -7.199653625488281] }, { "array": [3.4860363006591799, -73.30689239501953, -21.244325637817384, -38.725765228271487, 41.22803497314453, -15.975458145141602, -42.09079360961914, 27.367385864257814, -19.52243995666504, -16.397735595703126, -3.7795209884643556, -4.958502292633057] }, { "array": [0.09921550750732422, -70.22235107421875, -20.888980865478517, -33.53620910644531, 48.01523208618164, -13.365336418151856, -42.88488006591797, 26.451934814453126, -13.952343940734864, -16.65743064880371, -1.894343376159668, -1.8812918663024903] }, { "array": [0.09921550750732422, -70.22235107421875, -20.888980865478517, -33.53620910644531, 48.01523208618164, -13.365336418151856, -42.88488006591797, 26.451934814453126, -13.952343940734864, -16.65743064880371, -1.894343376159668, -1.8812918663024903] }, { "array": [0.09921550750732422, -70.22235107421875, -20.888980865478517, -33.53620910644531, 48.01523208618164, -13.365336418151856, -42.88488006591797, 26.451934814453126, -13.952343940734864, -16.65743064880371, -1.894343376159668, -1.8812918663024903] }, { "array": [-4.117016792297363, -72.00682830810547, -19.490331649780275, -34.48163986206055, 51.383995056152347, -9.989368438720704, -41.8690185546875, 23.824867248535158, -11.14547061920166, -14.500547409057618, -2.504335403442383, 1.0616645812988282] }, { "array": [-3.0772790908813478, -77.95940399169922, -21.169992446899415, -43.19057846069336, 47.594181060791019, -14.239681243896485, -43.05297088623047, 23.796558380126954, -16.033828735351564, -14.308491706848145, -4.90071964263916, -0.6597251892089844] }, { "array": [-3.0772790908813478, -77.95940399169922, -21.169992446899415, -43.19057846069336, 47.594181060791019, -14.239681243896485, -43.05297088623047, 23.796558380126954, -16.033828735351564, -14.308491706848145, -4.90071964263916, -0.6597251892089844] }, { "array": [0.27674388885498049, -74.16876220703125, -19.241043090820314, -43.69765853881836, 45.94886779785156, -15.777923583984375, -40.226318359375, 25.209468841552736, -19.91909408569336, -14.123311042785645, -6.749327659606934, -2.186051368713379] }, { "array": [1.313084602355957, -70.17343139648438, -20.149150848388673, -40.1507568359375, 43.281288146972659, -17.598236083984376, -39.989742279052737, 19.475574493408204, -18.73434066772461, -15.377893447875977, -3.6761083602905275, -2.3720903396606447] }] }, { "name": "I", "mfccCalibrationDataList": [{ "array": [-43.23860549926758, 15.659211158752442, 48.96119689941406, -84.8009033203125, -2.832998275756836, -25.101383209228517, -19.283388137817384, -1.3310365676879883, -3.9067935943603517, -5.5547566413879398, -24.206161499023439, 19.690078735351564] }, { "array": [-43.23860549926758, 15.659211158752442, 48.96119689941406, -84.8009033203125, -2.832998275756836, -25.101383209228517, -19.283388137817384, -1.3310365676879883, -3.9067935943603517, -5.5547566413879398, -24.206161499023439, 19.690078735351564] }, { "array": [-43.23860549926758, 15.659211158752442, 48.96119689941406, -84.8009033203125, -2.832998275756836, -25.101383209228517, -19.283388137817384, -1.3310365676879883, -3.9067935943603517, -5.5547566413879398, -24.206161499023439, 19.690078735351564] }, { "array": [-35.28649139404297, 11.94938850402832, 46.56689453125, -80.19161987304688, -5.332358360290527, -16.747013092041017, -17.014617919921876, -6.106320381164551, -3.5759763717651369, -11.297148704528809, -19.73563575744629, 19.95285415649414] }, { "array": [-35.28649139404297, 11.94938850402832, 46.56689453125, -80.19161987304688, -5.332358360290527, -16.747013092041017, -17.014617919921876, -6.106320381164551, -3.5759763717651369, -11.297148704528809, -19.73563575744629, 19.95285415649414] }, { "array": [-33.85117721557617, 12.748700141906739, 48.141944885253909, -75.80270385742188, -0.9702749252319336, -14.077031135559082, -15.911153793334961, -4.433165073394775, -4.00740909576416, -9.756240844726563, -20.910476684570314, 18.42697525024414] }, { "array": [-29.72378921508789, 13.039468765258789, 47.7448616027832, -74.11089324951172, 0.8275318145751953, -16.012189865112306, -17.36796760559082, -1.0376081466674805, -4.8292741775512699, -6.667880058288574, -23.82168960571289, 16.032718658447267] }, { "array": [-29.259418487548829, 11.963765144348145, 41.54622268676758, -75.67298889160156, -2.0329294204711916, -20.586360931396486, -21.125045776367189, 0.5504961013793945, -5.419882774353027, -6.314876556396484, -25.0130615234375, 14.262750625610352] }, { "array": [-29.259418487548829, 11.963765144348145, 41.54622268676758, -75.67298889160156, -2.0329294204711916, -20.586360931396486, -21.125045776367189, 0.5504961013793945, -5.419882774353027, -6.314876556396484, -25.0130615234375, 14.262750625610352] }, { "array": [-28.383831024169923, 13.103409767150879, 39.94292449951172, -81.08953857421875, -4.134577751159668, -21.590072631835939, -23.217021942138673, -0.3798789978027344, -5.0724334716796879, -5.94474983215332, -26.63843536376953, 16.777332305908204] }, { "array": [-30.048954010009767, 16.332015991210939, 44.49327850341797, -81.91828155517578, -4.171995162963867, -19.618621826171876, -20.534595489501954, -0.9673957824707031, -3.2188777923583986, -6.572293758392334, -26.59181785583496, 18.48187255859375] }, { "array": [-30.987979888916017, 14.425168991088868, 48.951114654541019, -82.33119201660156, -4.302616119384766, -17.642169952392579, -19.921981811523439, -1.7414522171020508, -1.7319145202636719, -6.870545387268066, -22.85688591003418, 19.475980758666993] }] }, { "name": "U", "mfccCalibrationDataList": [{ "array": [35.53300476074219, -11.371288299560547, 16.699298858642579, -39.32943344116211, 2.8931827545166017, -39.35669708251953, -22.81580924987793, -8.255973815917969, -7.3601884841918949, 3.866161346435547, -36.18340301513672, 0.44779300689697268] }, { "array": [34.92810821533203, -12.66100025177002, 19.551185607910158, -38.589942932128909, 1.5609407424926758, -42.781375885009769, -22.405025482177736, -4.380008220672607, -9.27183723449707, 5.952349662780762, -38.181243896484378, 1.4162702560424805] }, { "array": [34.508079528808597, -10.0599946975708, 18.88445281982422, -34.42189407348633, -0.8881950378417969, -42.52379608154297, -20.37147331237793, -0.6865062713623047, -8.89057731628418, 6.381434440612793, -34.816715240478519, 3.530543327331543] }, { "array": [34.508079528808597, -10.0599946975708, 18.88445281982422, -34.42189407348633, -0.8881950378417969, -42.52379608154297, -20.37147331237793, -0.6865062713623047, -8.89057731628418, 6.381434440612793, -34.816715240478519, 3.530543327331543] }, { "array": [37.634185791015628, -14.223724365234375, 21.262420654296876, -39.50825119018555, 0.31142520904541018, -43.201324462890628, -19.136680603027345, -0.7461652755737305, -8.253379821777344, 9.12716293334961, -33.483253479003909, 9.340311050415039] }, { "array": [38.28554916381836, -15.814170837402344, 19.503570556640626, -40.19694900512695, -0.7106151580810547, -45.373783111572269, -20.631587982177736, -2.2332963943481447, -6.609874725341797, 8.43967342376709, -33.560943603515628, 8.549976348876954] }, { "array": [38.28554916381836, -15.814170837402344, 19.503570556640626, -40.19694900512695, -0.7106151580810547, -45.373783111572269, -20.631587982177736, -2.2332963943481447, -6.609874725341797, 8.43967342376709, -33.560943603515628, 8.549976348876954] }, { "array": [36.15192413330078, -19.43944549560547, 19.22710609436035, -40.229862213134769, -1.7119722366333008, -44.58899688720703, -22.39651870727539, -4.6873579025268559, -5.184035301208496, 5.601207733154297, -35.313777923583987, 6.856324195861816] }, { "array": [33.697601318359378, -19.53946304321289, 18.151920318603517, -38.29360580444336, -1.9536991119384766, -39.185218811035159, -21.181903839111329, -8.092260360717774, -6.441320419311523, 3.8472461700439455, -32.4590950012207, 8.79825210571289] }, { "array": [31.768585205078126, -17.137840270996095, 15.726847648620606, -36.897605895996097, -3.8221397399902345, -35.492652893066409, -19.920631408691408, -10.421395301818848, -7.471479415893555, 1.7216854095458985, -25.726573944091798, 12.090950965881348] }, { "array": [31.768585205078126, -17.137840270996095, 15.726847648620606, -36.897605895996097, -3.8221397399902345, -35.492652893066409, -19.920631408691408, -10.421395301818848, -7.471479415893555, 1.7216854095458985, -25.726573944091798, 12.090950965881348] }, { "array": [32.6759033203125, -15.928326606750489, 16.0853271484375, -34.126686096191409, -6.689325332641602, -33.79350662231445, -19.242847442626954, -10.59890079498291, -10.992877006530762, -2.9957642555236818, -23.329811096191408, 14.231583595275879] }] }, { "name": "U", "mfccCalibrationDataList": [{ "array": [50.45172882080078, -7.723395347595215, 32.294891357421878, -11.293773651123047, -22.777332305908204, -36.17817687988281, -17.044910430908204, -3.7672786712646486, -14.233147621154786, -17.250513076782228, -19.240345001220704, -6.971443176269531] }, { "array": [53.59954071044922, -9.874463081359864, 35.237457275390628, -11.929043769836426, -23.915904998779298, -38.07780838012695, -15.71041202545166, -6.737283706665039, -12.06786060333252, -14.436643600463868, -20.401880264282228, -7.594654560089111] }, { "array": [57.36424255371094, -14.57245922088623, 35.10681915283203, -13.044787406921387, -24.196090698242189, -36.5896110534668, -16.161855697631837, -10.228910446166993, -10.43470287322998, -10.997936248779297, -20.92641830444336, -7.631929397583008] }, { "array": [57.65166473388672, -16.92430305480957, 33.022151947021487, -12.667181968688965, -21.69562530517578, -32.90369415283203, -14.275350570678711, -10.332601547241211, -10.544659614562989, -6.801647186279297, -22.47809410095215, -8.94782543182373] }, { "array": [49.60702133178711, -9.94367504119873, 25.04156494140625, -9.98995304107666, -21.814006805419923, -28.99759292602539, -16.792327880859376, -9.5930814743042, -11.523345947265625, -7.568509578704834, -20.688356399536134, -10.5545015335083] }, { "array": [53.433990478515628, -10.947863578796387, 30.426055908203126, -10.506410598754883, -22.997278213500978, -31.270254135131837, -16.90880584716797, -9.606002807617188, -10.785472869873047, -9.04238510131836, -22.82281494140625, -12.42250919342041] }, { "array": [57.11727523803711, -14.77857780456543, 33.16322326660156, -13.437588691711426, -25.450626373291017, -33.84945297241211, -17.6593017578125, -11.152002334594727, -12.203851699829102, -9.72322940826416, -26.653217315673829, -12.098143577575684] }, { "array": [58.65357208251953, -15.040183067321778, 32.92494583129883, -12.598053932189942, -24.03311538696289, -33.4146728515625, -15.565327644348145, -11.081777572631836, -10.47522258758545, -9.327695846557618, -28.43020248413086, -11.224303245544434] }, { "array": [53.888484954833987, -13.592279434204102, 31.6711368560791, -9.440587043762207, -23.316177368164064, -35.3663444519043, -19.26239776611328, -13.472862243652344, -11.312352180480957, -7.2335309982299809, -25.466888427734376, -8.924440383911133] }, { "array": [54.30769348144531, -13.315286636352539, 26.960655212402345, -11.682543754577637, -23.105655670166017, -32.507049560546878, -17.883333206176759, -10.961587905883789, -11.377249717712403, -7.610130786895752, -24.951541900634767, -9.38066577911377] }, { "array": [57.15829849243164, -13.771659851074219, 31.29568099975586, -9.493865966796875, -23.708837509155275, -33.85169982910156, -19.99985122680664, -12.555244445800782, -14.447962760925293, -7.608822822570801, -23.899951934814454, -6.755941867828369] }, { "array": [47.781307220458987, -14.279045104980469, 23.228328704833986, -14.190330505371094, -23.399112701416017, -34.95072555541992, -21.406070709228517, -10.15461254119873, -14.686234474182129, -9.932022094726563, -22.141719818115236, -8.354757308959961] }] }, { "name": "E", "mfccCalibrationDataList": [{ "array": [7.131360054016113, -36.049346923828128, 45.31159210205078, -45.00178527832031, -15.53347110748291, 1.3071203231811524, 2.57974910736084, -1.8762083053588868, -25.02313995361328, -22.275257110595704, -16.383546829223634, -17.868574142456056] }, { "array": [7.131360054016113, -36.049346923828128, 45.31159210205078, -45.00178527832031, -15.53347110748291, 1.3071203231811524, 2.57974910736084, -1.8762083053588868, -25.02313995361328, -22.275257110595704, -16.383546829223634, -17.868574142456056] }, { "array": [11.067873001098633, -36.16781234741211, 45.893943786621097, -43.60368347167969, -15.017866134643555, -1.2796411514282227, 2.090773582458496, -2.00726318359375, -22.139572143554689, -22.111957550048829, -16.975831985473634, -17.187711715698243] }, { "array": [9.000146865844727, -36.58183288574219, 46.39983367919922, -45.5433349609375, -15.245804786682129, -2.223395347595215, 4.547385215759277, -1.7552833557128907, -24.691539764404298, -22.836109161376954, -15.247169494628907, -15.090204238891602] }, { "array": [6.99749755859375, -37.912235260009769, 46.3832893371582, -47.186683654785159, -13.492773056030274, -2.4662675857543947, 5.109303951263428, -1.4368247985839844, -25.965919494628908, -23.327587127685548, -14.712173461914063, -15.369802474975586] }, { "array": [8.53531551361084, -38.04619598388672, 47.71860885620117, -47.188323974609378, -12.696022987365723, 0.9119815826416016, 5.812184810638428, 1.9683570861816407, -28.441104888916017, -21.203857421875, -16.284610748291017, -15.28965950012207] }, { "array": [8.53531551361084, -38.04619598388672, 47.71860885620117, -47.188323974609378, -12.696022987365723, 0.9119815826416016, 5.812184810638428, 1.9683570861816407, -28.441104888916017, -21.203857421875, -16.284610748291017, -15.28965950012207] }, { "array": [10.728363037109375, -37.86867141723633, 48.31739044189453, -47.47754669189453, -12.409613609313965, -0.4410533905029297, 5.663397789001465, 3.449146270751953, -27.80557632446289, -20.967662811279298, -15.19067096710205, -14.877350807189942] }, { "array": [10.065123558044434, -36.660072326660159, 44.76460266113281, -47.56492233276367, -12.322249412536621, -2.3014774322509767, 3.445328712463379, 3.1899805068969728, -25.734312057495118, -21.715232849121095, -13.482653617858887, -17.041053771972658] }, { "array": [8.782588958740235, -36.92534255981445, 44.4034309387207, -46.98899841308594, -8.84085464477539, -3.2711610794067385, 2.8354501724243166, 3.1592531204223635, -24.939672470092775, -21.54747772216797, -15.242938041687012, -17.53165054321289] }, { "array": [6.508709907531738, -36.80110168457031, 42.792266845703128, -46.29795837402344, -6.919498443603516, -3.788252830505371, 2.5329980850219728, 4.23353385925293, -24.539836883544923, -21.791210174560548, -16.75529670715332, -18.159324645996095] }, { "array": [6.508709907531738, -36.80110168457031, 42.792266845703128, -46.29795837402344, -6.919498443603516, -3.788252830505371, 2.5329980850219728, 4.23353385925293, -24.539836883544923, -21.791210174560548, -16.75529670715332, -18.159324645996095] }] }, { "name": "O", "mfccCalibrationDataList": [{ "array": [53.443660736083987, -12.53360652923584, -26.871780395507814, -71.98885345458985, -5.861575126647949, -1.7120800018310547, -32.90825653076172, 23.514209747314454, -10.125606536865235, -11.977684020996094, 9.883563041687012, -5.261895179748535] }, { "array": [55.69048309326172, -10.316678047180176, -26.289718627929689, -74.05561828613281, -4.348355293273926, -1.1102313995361329, -32.498268127441409, 22.016807556152345, -11.227481842041016, -11.753466606140137, 10.583023071289063, -3.583785057067871] }, { "array": [55.69048309326172, -10.316678047180176, -26.289718627929689, -74.05561828613281, -4.348355293273926, -1.1102313995361329, -32.498268127441409, 22.016807556152345, -11.227481842041016, -11.753466606140137, 10.583023071289063, -3.583785057067871] }, { "array": [56.333030700683597, -9.761429786682129, -24.525028228759767, -75.67504119873047, -2.9411144256591799, -1.0509262084960938, -32.58983612060547, 20.397789001464845, -13.52730941772461, -10.221673965454102, 8.806441307067871, -6.1672868728637699] }, { "array": [58.425559997558597, -13.048576354980469, -26.326568603515626, -77.33265686035156, 0.8971290588378906, -0.13758087158203126, -34.79779052734375, 22.524978637695314, -15.138383865356446, -9.027335166931153, 10.94324779510498, -6.867808818817139] }, { "array": [57.16611099243164, -17.380069732666017, -26.70465087890625, -76.53448486328125, 3.203751564025879, 1.6217775344848633, -36.67759323120117, 24.14405059814453, -12.522785186767579, -8.60572338104248, 13.969680786132813, -4.909186840057373] }, { "array": [55.17543029785156, -22.491680145263673, -25.568838119506837, -74.10460662841797, 2.1666202545166017, 4.1397705078125, -37.21670913696289, 21.997055053710939, -11.680967330932618, -8.396781921386719, 13.454421997070313, -3.452665328979492] }, { "array": [55.18680953979492, -23.176528930664064, -23.64011573791504, -69.64533233642578, 1.9893865585327149, 7.129931449890137, -35.871803283691409, 19.410160064697267, -13.193324089050293, -8.643393516540528, 10.173726081848145, -3.2846717834472658] }, { "array": [55.18680953979492, -23.176528930664064, -23.64011573791504, -69.64533233642578, 1.9893865585327149, 7.129931449890137, -35.871803283691409, 19.410160064697267, -13.193324089050293, -8.643393516540528, 10.173726081848145, -3.2846717834472658] }, { "array": [52.92599105834961, -22.969005584716798, -25.62136459350586, -65.79485321044922, 0.01132965087890625, 5.457650184631348, -33.96955871582031, 16.389381408691408, -14.026248931884766, -7.287093162536621, 11.159339904785157, -3.7541093826293947] }, { "array": [51.816688537597659, -22.891870498657228, -29.805912017822267, -68.66901397705078, -1.9484624862670899, 1.6389532089233399, -33.593971252441409, 17.095460891723634, -13.046170234680176, -8.923750877380371, 13.052698135375977, -5.068996429443359] }, { "array": [54.392459869384769, -23.71658706665039, -31.424976348876954, -75.520263671875, -4.738470077514648, -0.12287521362304688, -36.41456604003906, 18.125713348388673, -14.02833366394043, -11.63118839263916, 12.234237670898438, -4.546117782592773] }] }] } diff --git a/packages/stage-ui-mmd/src/components/index.ts b/packages/stage-ui-mmd/src/components/index.ts new file mode 100644 index 000000000..ab6b53306 --- /dev/null +++ b/packages/stage-ui-mmd/src/components/index.ts @@ -0,0 +1 @@ +export * from './scenes' diff --git a/packages/stage-ui-mmd/src/components/scenes/MMD.vue b/packages/stage-ui-mmd/src/components/scenes/MMD.vue new file mode 100644 index 000000000..ff4acb8d5 --- /dev/null +++ b/packages/stage-ui-mmd/src/components/scenes/MMD.vue @@ -0,0 +1,606 @@ + + + diff --git a/packages/stage-ui-mmd/src/components/scenes/index.ts b/packages/stage-ui-mmd/src/components/scenes/index.ts new file mode 100644 index 000000000..e902ac354 --- /dev/null +++ b/packages/stage-ui-mmd/src/components/scenes/index.ts @@ -0,0 +1 @@ +export { default as MMD } from './MMD.vue' diff --git a/packages/stage-ui-mmd/src/composables/mmd/animation-manager.ts b/packages/stage-ui-mmd/src/composables/mmd/animation-manager.ts new file mode 100644 index 000000000..3b0c5221e --- /dev/null +++ b/packages/stage-ui-mmd/src/composables/mmd/animation-manager.ts @@ -0,0 +1,262 @@ +import type { AnimationAction, AnimationClip, AnimationMixer, SkinnedMesh } from 'three' + +import { AnimationClip as AnimationClipCtor, LoopOnce, LoopRepeat, Vector3 } from 'three' +import { MMDAnimationHelper } from 'three-stdlib' + +import { ensureAmmo } from '../../utils/ammo' + +const DEFAULT_CROSSFADE = 0.4 + +export interface MMDAnimationManagerOptions { + /** Initial physics enablement. Ammo always loads so it can be toggled later. */ + physicsEnabled?: boolean +} + +export interface PlayActionOptions { + /** Loop the action instead of reverting to idle when it finishes. */ + loop?: boolean + /** Cross-fade duration in seconds. */ + crossfade?: number +} + +/** + * Owns the per-model {@link MMDAnimationHelper} and the catalog of importable + * VMD motions, exposing a play/crossfade API and physics/IK/grant toggles. + * + * Design: + * - The helper auto-plays whatever clip it is constructed with, so we hand it + * only the idle clip (or an empty placeholder so a mixer always exists for + * physics warmup) and layer every other motion on the same mixer ourselves. + * - Physics is created up front (it cannot be added after the fact) and merely + * toggled via `helper.enable('physics', …)`, so Ammo is required before the + * mesh is added. Ammo is still lazy at the app level: it only loads once an + * MMD model is actually mounted. + * - One-shot actions register a `finished` listener that fades back to idle, + * mirroring how the Spine manager layers emotion clips over the idle track. + * + * `update(delta)` must be called once per frame; it drives animation, IK, + * append-bone (grant) propagation, and the physics simulation in one step. + */ +export function createMMDAnimationManager(mesh: SkinnedMesh, options: MMDAnimationManagerOptions = {}) { + // NOTICE: + // resetPhysicsOnLoop must stay false. When true, MMDAnimationHelper calls + // physics.reset() every time the mixer's clip loops, which snaps every rigid + // body back to its bone pose and re-seeds the sim — a visible periodic + // "fling" of hair/skirt. Continuous simulation looks correct for an idle + // character and avoids the jolt. + const helper = new MMDAnimationHelper({ afterglow: 2.0, resetPhysicsOnLoop: false }) + const registry = new Map() + + let mixer: AnimationMixer | undefined + let idleClip: AnimationClip | undefined + let idleAction: AnimationAction | undefined + let currentAction: AnimationAction | undefined + let initialized = false + + function getMixer(): AnimationMixer | undefined { + if (!mixer) + mixer = helper.objects.get(mesh)?.mixer + return mixer + } + + function revertToIdleOnFinish(action: AnimationAction) { + const m = getMixer() + if (!m) + return + const onFinished = (event: { action: AnimationAction }) => { + if (event.action !== action) + return + m.removeEventListener('finished', onFinished) + playIdle() + } + m.addEventListener('finished', onFinished) + } + + /** + * Builds the helper, physics, IK, and grant solvers for the mesh. + * + * `idle` is the persistent looping motion (optional). Ammo is initialized + * before the mesh is added so the physics world can be constructed. + */ + async function init(idle?: AnimationClip): Promise { + if (initialized) + return + + await ensureAmmo() + + // The helper needs at least one clip to create a mixer (required for action + // playback). When there is no real idle motion we use a long, track-less + // placeholder: a zero-length clip would fire the mixer's "loop" event every + // frame, so the large duration keeps it from ever looping. + idleClip = idle ?? new AnimationClipCtor('__mmd_empty__', Number.MAX_SAFE_INTEGER, []) + + helper.add(mesh, { + animation: idleClip, + physics: true, + }) + + helper.enable('physics', options.physicsEnabled ?? true) + + mixer = helper.objects.get(mesh)?.mixer + if (mixer && idle) + idleAction = mixer.existingAction(idleClip) ?? undefined + currentAction = idleAction + initialized = true + } + + /** Registers a VMD-derived clip under a name for later playback. */ + function registerClip(name: string, clip: AnimationClip): void { + registry.set(name, clip) + } + + /** Names of all registered motions, for settings UIs and action mapping. */ + function availableClips(): string[] { + return Array.from(registry.keys()) + } + + /** Cross-fades back to the persistent idle loop. */ + function playIdle(crossfade = DEFAULT_CROSSFADE): void { + const m = getMixer() + if (!m) + return + + // No idle clip registered (e.g. empty placeholder): just fade the current + // motion out so bones relax to rest instead of clamping on the last frame. + if (!idleAction) { + if (currentAction) + currentAction.fadeOut(crossfade) + currentAction = undefined + return + } + + if (currentAction && currentAction !== idleAction) + currentAction.fadeOut(crossfade) + // Restore LoopRepeat: a one-shot may have reused this same action with + // LoopOnce, which would otherwise leave the idle no longer looping. + idleAction.reset().setLoop(LoopRepeat, Number.POSITIVE_INFINITY).setEffectiveWeight(1).fadeIn(crossfade).play() + currentAction = idleAction + } + + /** + * Plays a registered motion, cross-fading from the current one. One-shots + * revert to idle on completion; looping motions stay until replaced. + * + * Returns `false` when the name is not registered so callers can fall back. + */ + function playAction(name: string, opts: PlayActionOptions = {}): boolean { + const m = getMixer() + const clip = registry.get(name) + if (!m || !clip) { + console.warn(`[mmd] playAction skipped: "${name}" is ${clip ? 'present' : 'not registered'}, mixer ${m ? 'ready' : 'missing'}`) + return false + } + + const loop = opts.loop ?? false + const crossfade = opts.crossfade ?? DEFAULT_CROSSFADE + + const action = m.clipAction(clip) + action.reset() + action.setLoop(loop ? LoopRepeat : LoopOnce, loop ? Number.POSITIVE_INFINITY : 1) + action.clampWhenFinished = !loop + action.setEffectiveWeight(1) + action.fadeIn(crossfade).play() + + if (currentAction && currentAction !== action) + currentAction.fadeOut(crossfade) + currentAction = action + + if (!loop) + revertToIdleOnFinish(action) + + return true + } + + /** + * Makes a registered motion the persistent looping base (the idle the + * character returns to). Cross-fades from whatever is currently playing. + * + * Returns `false` when the name is not registered. + */ + function setIdleMotion(name: string, crossfade = DEFAULT_CROSSFADE): boolean { + const m = getMixer() + const clip = registry.get(name) + if (!m || !clip) { + console.warn(`[mmd] setIdleMotion skipped: "${name}" is ${clip ? 'present' : 'not registered'}, mixer ${m ? 'ready' : 'missing'}`) + return false + } + + const action = m.clipAction(clip) + action.reset() + action.setLoop(LoopRepeat, Number.POSITIVE_INFINITY) + action.clampWhenFinished = false + action.setEffectiveWeight(1) + action.fadeIn(crossfade).play() + + const previous = idleAction + idleClip = clip + idleAction = action + if (currentAction && currentAction !== action) + currentAction.fadeOut(crossfade) + else if (previous && previous !== action) + previous.fadeOut(crossfade) + currentAction = action + return true + } + + function setPhysicsEnabled(enabled: boolean): void { + helper.enable('physics', enabled) + } + + /** Sets the physics world gravity strength, applied as (0, -magnitude, 0). */ + function setGravity(magnitude: number): void { + helper.objects.get(mesh)?.physics?.setGravity(new Vector3(0, -magnitude, 0)) + } + + function setIKEnabled(enabled: boolean): void { + helper.enable('ik', enabled) + } + + function setGrantEnabled(enabled: boolean): void { + helper.enable('grant', enabled) + } + + function update(delta: number): void { + if (!initialized) + return + helper.update(delta) + } + + function dispose(): void { + const m = getMixer() + m?.stopAllAction() + if (initialized) { + try { + helper.remove(mesh) + } + catch {} + } + registry.clear() + mixer = undefined + idleAction = undefined + currentAction = undefined + initialized = false + } + + return { + helper, + init, + registerClip, + availableClips, + playIdle, + playAction, + setIdleMotion, + setPhysicsEnabled, + setGravity, + setIKEnabled, + setGrantEnabled, + update, + dispose, + } +} + +export type MMDAnimationManager = ReturnType diff --git a/packages/stage-ui-mmd/src/composables/mmd/blink.ts b/packages/stage-ui-mmd/src/composables/mmd/blink.ts new file mode 100644 index 000000000..a5926e0dc --- /dev/null +++ b/packages/stage-ui-mmd/src/composables/mmd/blink.ts @@ -0,0 +1,52 @@ +import type { MorphController } from './morph' + +import { ref } from 'vue' + +const BLINK_DURATION = 0.2 // seconds for one open→close→open cycle +const MIN_BLINK_INTERVAL = 1 +const MAX_BLINK_INTERVAL = 6 + +function nextInterval() { + return Math.random() * (MAX_BLINK_INTERVAL - MIN_BLINK_INTERVAL) + MIN_BLINK_INTERVAL +} + +/** + * Procedural eye blinking via the model's blink morph. + * + * Identical timing model to the VRM renderer's `useBlink`: a randomized + * 1–6 s interval between blinks and a 0.2 s sine-shaped close/open so the lid + * motion is smooth rather than a hard on/off. + */ +export function useMMDBlink() { + const isBlinking = ref(false) + const blinkProgress = ref(0) + const timeSinceLastBlink = ref(0) + const nextBlinkTime = ref(nextInterval()) + + function update(morphs: MorphController | undefined, delta: number) { + if (!morphs) + return + + timeSinceLastBlink.value += delta + + if (!isBlinking.value && timeSinceLastBlink.value >= nextBlinkTime.value) { + isBlinking.value = true + blinkProgress.value = 0 + } + + if (!isBlinking.value) + return + + blinkProgress.value += delta / BLINK_DURATION + morphs.set('blink', Math.sin(Math.PI * blinkProgress.value)) + + if (blinkProgress.value >= 1) { + isBlinking.value = false + timeSinceLastBlink.value = 0 + morphs.set('blink', 0) + nextBlinkTime.value = nextInterval() + } + } + + return { update } +} diff --git a/packages/stage-ui-mmd/src/composables/mmd/expression.ts b/packages/stage-ui-mmd/src/composables/mmd/expression.ts new file mode 100644 index 000000000..2a9d999a1 --- /dev/null +++ b/packages/stage-ui-mmd/src/composables/mmd/expression.ts @@ -0,0 +1,120 @@ +import type { MorphSlot } from '../../constants/morphs' +import type { MorphController } from './morph' + +import { ref } from 'vue' + +import { Emotion } from '../../constants/emotions' +import { EMOTION_MORPHS } from '../../constants/morphs' + +function lerp(start: number, end: number, t: number): number { + return start + (end - start) * t +} + +function easeInOutCubic(t: number): number { + return t < 0.5 ? 4 * t * t * t : 1 - (-2 * t + 2) ** 3 / 2 +} + +function clampIntensity(value: number): number { + return Math.min(1, Math.max(0, value)) +} + +/** All emotion-related morph slots, so transitions can zero the untargeted ones. */ +const EXPRESSION_SLOTS: MorphSlot[] = ['smile', 'anger', 'sad', 'surprise', 'troubled', 'serious'] + +/** + * Emotion-driven facial morphs for MMD models. + * + * Mirrors the VRM emote composable: each emotion declares target morph + * weights and a blend duration, and `update(delta)` cross-fades from the + * currently displayed weights to the target with an ease-in-out curve so + * emotion changes never snap. + * + * Lip-sync owns the vowel slots, so emotions only touch expression slots + * (smile/anger/...) plus, for a few emotions, a small fixed vowel accent that + * lip-sync overrides as soon as speech starts. + */ +export function useMMDEmote(morphs: MorphController) { + const currentEmotion = ref(null) + const isTransitioning = ref(false) + const transitionProgress = ref(0) + const startWeights = new Map() + const targetWeights = new Map() + let resetTimeout: ReturnType | undefined + + function clearResetTimeout() { + if (resetTimeout) { + clearTimeout(resetTimeout) + resetTimeout = undefined + } + } + + function setEmotion(emotion: Emotion, intensity = 1) { + clearResetTimeout() + + const state = EMOTION_MORPHS[emotion] + if (!state) { + console.warn(`[mmd] emotion ${emotion} not found`) + return + } + + currentEmotion.value = emotion + isTransitioning.value = true + transitionProgress.value = 0 + startWeights.clear() + targetWeights.clear() + + const normalized = clampIntensity(intensity) + + // Capture where each expression slot currently sits so the cross-fade + // starts from the live value instead of snapping to zero first. + for (const slot of EXPRESSION_SLOTS) { + startWeights.set(slot, morphs.get(slot)) + targetWeights.set(slot, 0) + } + + for (const influence of state.influences) { + startWeights.set(influence.slot, morphs.get(influence.slot)) + targetWeights.set(influence.slot, influence.value * normalized) + } + } + + function setEmotionWithResetAfter(emotion: Emotion, ms: number, intensity = 1) { + clearResetTimeout() + setEmotion(emotion, intensity) + resetTimeout = setTimeout(() => { + setEmotion(Emotion.Neutral) + resetTimeout = undefined + }, ms) + } + + function update(delta: number) { + if (!isTransitioning.value || !currentEmotion.value) + return + + const blendDuration = EMOTION_MORPHS[currentEmotion.value].blendDuration || 0.3 + transitionProgress.value += delta / blendDuration + if (transitionProgress.value >= 1) { + transitionProgress.value = 1 + isTransitioning.value = false + } + + const t = easeInOutCubic(transitionProgress.value) + for (const [slot, target] of targetWeights) { + const start = startWeights.get(slot) ?? 0 + morphs.set(slot, lerp(start, target, t)) + } + } + + function dispose() { + clearResetTimeout() + } + + return { + currentEmotion, + isTransitioning, + setEmotion, + setEmotionWithResetAfter, + update, + dispose, + } +} diff --git a/packages/stage-ui-mmd/src/composables/mmd/gaze.ts b/packages/stage-ui-mmd/src/composables/mmd/gaze.ts new file mode 100644 index 000000000..f28dd8941 --- /dev/null +++ b/packages/stage-ui-mmd/src/composables/mmd/gaze.ts @@ -0,0 +1,139 @@ +import type { Bone, SkinnedMesh } from 'three' + +import { Euler, MathUtils, Quaternion } from 'three' + +/** Normalized screen-space gaze offset, each axis in roughly [-1, 1]. */ +export interface GazeOffset { + x: number + y: number +} + +// Eyes can swing further than the head; values in radians. +export const EYE_YAW_LIMIT = 0.35 +export const EYE_PITCH_LIMIT = 0.25 +const HEAD_YAW_LIMIT = 0.2 +const HEAD_PITCH_LIMIT = 0.12 + +// Idle saccade tuning — small, infrequent darts when nothing is tracked. +const SACCADE_MIN_MS = 400 +const SACCADE_MAX_MS = 2200 +const SACCADE_RANGE = 0.4 + +// Candidate bone names (semi-standard MMD first, then common English exports). +const LEFT_EYE_NAMES = ['左目', 'eye_L', 'EyeLeft', 'LeftEye'] +const RIGHT_EYE_NAMES = ['右目', 'eye_R', 'EyeRight', 'RightEye'] +const BOTH_EYES_NAMES = ['両目', 'Eyes'] +const HEAD_NAMES = ['頭', 'head', 'Head'] + +function findBone(mesh: SkinnedMesh, names: string[]): Bone | undefined { + for (const name of names) { + const bone = mesh.skeleton.bones.find(b => b.name === name) + if (bone) + return bone + } + return undefined +} + +interface GazeBone { + bone: Bone + rest: Quaternion +} + +export interface GazeController { + /** Drives gaze toward a normalized offset; pass `undefined` to idle-saccade. */ + update: (offset: GazeOffset | undefined, delta: number) => void +} + +/** + * Bone-based eye gaze and subtle head aim for MMD models. + * + * VRM exposes a first-class `lookAt`; MMD does not, so we rotate the eye bones + * and add a damped fraction of the same aim to the head bone for a natural + * follow. Rotations are applied relative to each bone's rest pose and must run + * after `MMDAnimationHelper.update()` so they layer on top of the active + * motion. + * + * We rotate the actual `左目`/`右目` eye bones (which the eyeballs are skinned + * to) rather than the `両目` control bone. `両目` drives the eyes through the + * append/grant solver, which runs *inside* `helper.update()`; rotating it + * afterward would be too late and the eyes would not move. `両目` is used only + * as a fallback when a model lacks separate eye bones. + * + * When no tracking target is supplied the controller produces small idle + * saccades, matching the VRM renderer's resting-eye behavior. + */ +export function createGazeController(mesh: SkinnedMesh): GazeController { + const eyeBones: GazeBone[] = [] + const left = findBone(mesh, LEFT_EYE_NAMES) + const right = findBone(mesh, RIGHT_EYE_NAMES) + if (left) + eyeBones.push({ bone: left, rest: left.quaternion.clone() }) + if (right) + eyeBones.push({ bone: right, rest: right.quaternion.clone() }) + + // Fallback: a single both-eyes bone the eyeballs are skinned to directly. + if (eyeBones.length === 0) { + const both = findBone(mesh, BOTH_EYES_NAMES) + if (both) + eyeBones.push({ bone: both, rest: both.quaternion.clone() }) + } + + if (eyeBones.length === 0) { + console.warn( + '[mmd] gaze: no eye bone matched (左目/右目/両目); eye tracking disabled. ' + + 'Bones containing 目/eye:', + mesh.skeleton.bones.filter(b => /目|eye/i.test(b.name)).map(b => b.name), + ) + } + + const headBoneRaw = findBone(mesh, HEAD_NAMES) + const headBone: GazeBone | undefined = headBoneRaw + ? { bone: headBoneRaw, rest: headBoneRaw.quaternion.clone() } + : undefined + + // Smoothed current aim, lerped toward the target each frame. + const current: GazeOffset = { x: 0, y: 0 } + const scratchEuler = new Euler() + const scratchQuat = new Quaternion() + + // Idle saccade state. + let saccade: GazeOffset = { x: 0, y: 0 } + let nextSaccadeIn = 0 + + function applyBone(target: GazeBone, yaw: number, pitch: number) { + // Yaw about Y, pitch about X, relative to the bone's rest pose. + scratchEuler.set(pitch, yaw, 0, 'XYZ') + scratchQuat.setFromEuler(scratchEuler) + target.bone.quaternion.copy(target.rest).multiply(scratchQuat) + } + + return { + update(offset, delta) { + let desired = offset + if (!desired) { + nextSaccadeIn -= delta * 1000 + if (nextSaccadeIn <= 0) { + saccade = { + x: MathUtils.randFloat(-SACCADE_RANGE, SACCADE_RANGE), + y: MathUtils.randFloat(-SACCADE_RANGE, SACCADE_RANGE), + } + nextSaccadeIn = MathUtils.randFloat(SACCADE_MIN_MS, SACCADE_MAX_MS) + } + desired = saccade + } + + // Critically-damped-ish approach; frame-rate independent. + const rate = 1 - Math.exp(-10 * delta) + current.x += (desired.x - current.x) * rate + current.y += (desired.y - current.y) * rate + + // Positive cursor-x (right) turns gaze toward screen-right; positive + // cursor-y (down) looks down. + for (const eye of eyeBones) + applyBone(eye, current.x * EYE_YAW_LIMIT, current.y * EYE_PITCH_LIMIT) + + if (headBone) + applyBone(headBone, current.x * HEAD_YAW_LIMIT, current.y * HEAD_PITCH_LIMIT) + }, + } +} diff --git a/packages/stage-ui-mmd/src/composables/mmd/index.ts b/packages/stage-ui-mmd/src/composables/mmd/index.ts new file mode 100644 index 000000000..deee81c1f --- /dev/null +++ b/packages/stage-ui-mmd/src/composables/mmd/index.ts @@ -0,0 +1,7 @@ +export * from './animation-manager' +export * from './blink' +export * from './expression' +export * from './gaze' +export * from './lip-sync' +export * from './loader' +export * from './morph' diff --git a/packages/stage-ui-mmd/src/composables/mmd/lip-sync.ts b/packages/stage-ui-mmd/src/composables/mmd/lip-sync.ts new file mode 100644 index 000000000..54e4d128f --- /dev/null +++ b/packages/stage-ui-mmd/src/composables/mmd/lip-sync.ts @@ -0,0 +1,148 @@ +import type { Ref } from 'vue' +import type { Profile } from 'wlipsync' + +import type { VowelSlot } from '../../constants/morphs' +import type { MorphController } from './morph' + +import { useAsyncState } from '@vueuse/core' +import { onUnmounted, watch } from 'vue' +import { createWLipSyncNode } from 'wlipsync' + +import profile from '../../assets/lip-sync-profile.json' with { type: 'json' } + +// NOTICE: +// Cross-package source import (no package.json dependency) to reach the shared +// AudioContext. Root cause: the audio context is owned by `@proj-airi/stage-ui`, +// which in turn depends on this package, so adding it as a dependency would +// create a cycle. The VRM renderer reaches the same store the same way. +// Source: packages/stage-ui-three/src/composables/vrm/lip-sync.ts. +// Removal condition: the AudioContext provider moves to a dependency-free +// shared package both renderers can import. +import { useAudioContext } from '../../../../stage-ui/src/stores/audio' + +/** wLipSync emits 6 visemes; we fold the sibilant `S` into `I`. */ +const RAW_KEYS = ['A', 'E', 'I', 'O', 'U', 'S'] as const +type LipKey = 'A' | 'E' | 'I' | 'O' | 'U' +const LIP_KEYS: LipKey[] = ['A', 'E', 'I', 'O', 'U'] + +/** Maps wLipSync visemes onto the MMD vowel morph slots (あいうえお). */ +const LIP_TO_SLOT: Record = { + A: 'vowelA', + E: 'vowelE', + I: 'vowelI', + O: 'vowelO', + U: 'vowelU', +} +const RAW_TO_LIP: Record = { + A: 'A', + E: 'E', + I: 'I', + O: 'O', + U: 'U', + S: 'I', +} + +const ATTACK = 50 // approach speed toward the next mouth shape +const RELEASE = 30 // decay speed when a shape ends +const CAP = 0.7 // max morph weight, mirrors the VRM tuning +const SILENCE_VOL = 0.04 +const SILENCE_GAIN = 0.05 +const IDLE_MS = 160 + +/** + * Audio-driven mouth animation for MMD models. + * + * Reuses the exact wLipSync profile and winner/runner blending strategy as + * the VRM renderer (only the two strongest visemes are mixed, so the wide + * "A" shape does not dominate), but writes the result to MMD vowel morphs + * through a {@link MorphController} instead of VRM expressions. + * + * Returns an `update(delta)` to call once per frame, after the animation + * helper has run, so lip-sync wins over any VMD mouth keyframes. + */ +export function useMMDLipSync(audioNode: Ref) { + const { audioContext } = useAudioContext() + const { state: lipSyncNode, isReady } = useAsyncState(createWLipSyncNode(audioContext, profile as Profile), undefined) + + const smoothState: Record = { A: 0, E: 0, I: 0, O: 0, U: 0 } + let lastActiveAt = 0 + + watch([isReady, audioNode], ([ready, newAudioNode], [, oldAudioNode]) => { + if (oldAudioNode && oldAudioNode !== newAudioNode) { + try { + oldAudioNode.disconnect() + } + catch {} + } + if (!ready || !newAudioNode || !lipSyncNode.value) + return + try { + newAudioNode.connect(lipSyncNode.value) + } + catch {} + }, { immediate: true }) + + onUnmounted(() => audioNode.value?.disconnect()) + + function update(morphs: MorphController | undefined, delta = 0.016) { + const node = lipSyncNode.value + if (!morphs || !node) + return + + const vol = node.volume ?? 0 + const amp = Math.min(vol * 0.9, 1) ** 0.7 + + // Project the 6 raw visemes down to 5 vowels, scaled by amplitude. + const projected: Record = { A: 0, E: 0, I: 0, O: 0, U: 0 } + for (const raw of RAW_KEYS) { + const lip = RAW_TO_LIP[raw] + const rawVal = node.weights[raw] ?? 0 + projected[lip] = Math.max(projected[lip], rawVal * amp) + } + + // Only blend the two strongest vowels. Mixing all five biases toward the + // wide "A" shape because it has the largest deformation. + let winner: LipKey = 'I' + let runner: LipKey = 'E' + let winnerVal = -Infinity + let runnerVal = -Infinity + for (const key of LIP_KEYS) { + const val = projected[key] + if (val > winnerVal) { + runnerVal = winnerVal + runner = winner + winnerVal = val + winner = key + } + else if (val > runnerVal) { + runnerVal = val + runner = key + } + } + + // Treat low energy / brief gaps as silence so the mouth fully closes. + const now = performance.now() + let silent = amp < SILENCE_VOL || winnerVal < SILENCE_GAIN + if (!silent) + lastActiveAt = now + if (now - lastActiveAt > IDLE_MS) + silent = true + + const target: Record = { A: 0, E: 0, I: 0, O: 0, U: 0 } + if (!silent) { + target[winner] = Math.min(CAP, winnerVal) + target[runner] = Math.min(CAP * 0.5, runnerVal * 0.6) + } + + for (const key of LIP_KEYS) { + const from = smoothState[key] + const to = target[key] + const rate = 1 - Math.exp(-(to > from ? ATTACK : RELEASE) * delta) + smoothState[key] = from + (to - from) * rate + const weight = (smoothState[key] <= 0.01 ? 0 : smoothState[key]) * 0.7 + morphs.set(LIP_TO_SLOT[key], weight) + } + } + + return { update } +} diff --git a/packages/stage-ui-mmd/src/composables/mmd/loader.ts b/packages/stage-ui-mmd/src/composables/mmd/loader.ts new file mode 100644 index 000000000..3e1a8def8 --- /dev/null +++ b/packages/stage-ui-mmd/src/composables/mmd/loader.ts @@ -0,0 +1,85 @@ +import type { AnimationClip, SkinnedMesh } from 'three' + +import { LoadingManager } from 'three' +import { MMDLoader } from 'three-stdlib' + +/** Maps in-archive relative asset paths to blob URLs for ZIP-loaded models. */ +export type UrlModifier = (url: string) => string + +export interface MMDLoaderContext { + loader: MMDLoader + manager: LoadingManager +} + +/** + * Builds an {@link MMDLoader} backed by a dedicated {@link LoadingManager}. + * + * MMD models reference their textures (and toon ramps) by relative paths + * baked into the PMX/PMD binary. For ZIP imports those files live behind blob + * URLs, so we install a URL modifier on the manager: the loader asks for + * `tex/face.png`, the modifier rewrites it to the matching `blob:` URL. For + * plain URL/preset models the modifier passes paths through unchanged. + * + * A fresh manager per load keeps URL-rewrite tables isolated between models. + */ +export function createMMDLoaderContext(urlModifier?: UrlModifier): MMDLoaderContext { + const manager = new LoadingManager() + if (urlModifier) { + manager.setURLModifier((url) => { + // NOTICE: + // MMDLoader resolves textures by prepending the model URL's base, so for + // ZIP imports the texture requests arrive blob-prefixed + // (e.g. "blob:http://host/uuid/tex/face.png"). We must still run those + // through the resolver — an earlier `blob:` short-circuit here silently + // broke all ZIP textures. Only data: URIs (embedded toon textures) are + // passed through untouched. The resolver falls back to the original URL + // for the model file and any unmatched path, so this is safe. + if (url.startsWith('data:')) + return url + return urlModifier(url) + }) + } + + return { loader: new MMDLoader(manager), manager } +} + +/** Loads a PMX/PMD model URL into a {@link SkinnedMesh}. */ +export function loadMMDMesh( + loader: MMDLoader, + url: string, + onProgress?: (event: ProgressEvent) => void, +): Promise { + return new Promise((resolve, reject) => { + loader.load(url, resolve, onProgress, reject) + }) +} + +/** + * Loads a VMD motion file and binds it to `mesh`, producing an + * {@link AnimationClip} ready for the mesh's `AnimationMixer`. + * + * `loadAnimation` may hand back either a clip or (for camera motions) a mesh; + * AIRI only consumes model motions, so a non-clip result is rejected. + */ +export function loadMMDAnimationClip( + loader: MMDLoader, + url: string, + mesh: SkinnedMesh, + onProgress?: (event: ProgressEvent) => void, +): Promise { + return new Promise((resolve, reject) => { + loader.loadAnimation( + url, + mesh, + (result) => { + // A bound model motion resolves to an AnimationClip (has `.tracks`). + if (result && 'tracks' in result) + resolve(result as AnimationClip) + else + reject(new Error('Loaded VMD did not produce a model animation clip')) + }, + onProgress, + reject, + ) + }) +} diff --git a/packages/stage-ui-mmd/src/composables/mmd/morph.test.ts b/packages/stage-ui-mmd/src/composables/mmd/morph.test.ts new file mode 100644 index 000000000..481b99516 --- /dev/null +++ b/packages/stage-ui-mmd/src/composables/mmd/morph.test.ts @@ -0,0 +1,90 @@ +import type { SkinnedMesh } from 'three' + +import { describe, expect, it } from 'vitest' + +import { createMorphController } from './morph' + +// Minimal SkinnedMesh double: createMorphController only reads +// morphTargetDictionary and morphTargetInfluences. +function fakeMesh(names: string[]): SkinnedMesh { + const dictionary: Record = {} + names.forEach((name, index) => { + dictionary[name] = index + }) + const mesh = { + morphTargetDictionary: dictionary, + morphTargetInfluences: Array.from({ length: names.length }).fill(0), + } + return mesh as unknown as SkinnedMesh +} + +describe('createMorphController', () => { + it('resolves standard Japanese vowel and blink morphs', () => { + const mesh = fakeMesh(['あ', 'い', 'う', 'え', 'お', 'まばたき']) + const morphs = createMorphController(mesh) + + expect(morphs.resolvedSlots).toContain('vowelA') + expect(morphs.resolvedSlots).toContain('vowelI') + expect(morphs.resolvedSlots).toContain('blink') + }) + + it('falls back to English morph names when Japanese are absent', () => { + const mesh = fakeMesh(['A', 'E', 'I', 'O', 'U', 'Blink']) + const morphs = createMorphController(mesh) + + morphs.set('vowelA', 0.5) + expect(mesh.morphTargetInfluences![0]).toBe(0.5) + expect(morphs.get('vowelA')).toBe(0.5) + }) + + it('clamps weights to the [0, 1] range', () => { + const mesh = fakeMesh(['あ']) + const morphs = createMorphController(mesh) + + morphs.set('vowelA', 5) + expect(morphs.get('vowelA')).toBe(1) + morphs.set('vowelA', -3) + expect(morphs.get('vowelA')).toBe(0) + }) + + it('honors an explicit override over the candidate list', () => { + const mesh = fakeMesh(['あ', 'mouth_open']) + const morphs = createMorphController(mesh, { vowelA: 'mouth_open' }) + + morphs.set('vowelA', 1) + expect(mesh.morphTargetInfluences![1]).toBe(1) + expect(mesh.morphTargetInfluences![0]).toBe(0) + }) + + it('rebinds a slot when override() is called at runtime', () => { + const mesh = fakeMesh(['あ', 'custom_a']) + const morphs = createMorphController(mesh) + + morphs.set('vowelA', 1) + expect(mesh.morphTargetInfluences![0]).toBe(1) + + morphs.override('vowelA', 'custom_a') + morphs.set('vowelA', 0.4) + expect(mesh.morphTargetInfluences![1]).toBe(0.4) + }) + + it('only zeroes managed slots on resetManaged()', () => { + const mesh = fakeMesh(['あ', 'unmanaged']) + const morphs = createMorphController(mesh) + mesh.morphTargetInfluences![1] = 0.9 // not a managed slot + + morphs.set('vowelA', 0.7) + morphs.resetManaged() + + expect(morphs.get('vowelA')).toBe(0) + expect(mesh.morphTargetInfluences![1]).toBe(0.9) + }) + + it('ignores unresolved slots without throwing', () => { + const mesh = fakeMesh(['somethingElse']) + const morphs = createMorphController(mesh) + + expect(() => morphs.set('vowelA', 1)).not.toThrow() + expect(morphs.get('vowelA')).toBe(0) + }) +}) diff --git a/packages/stage-ui-mmd/src/composables/mmd/morph.ts b/packages/stage-ui-mmd/src/composables/mmd/morph.ts new file mode 100644 index 000000000..b7073edc2 --- /dev/null +++ b/packages/stage-ui-mmd/src/composables/mmd/morph.ts @@ -0,0 +1,109 @@ +import type { SkinnedMesh } from 'three' + +import type { MorphSlot } from '../../constants/morphs' + +import { MORPH_CANDIDATES } from '../../constants/morphs' + +/** + * Resolves a logical {@link MorphSlot} to the first matching morph name that + * actually exists on the model. + * + * Before: + * - slot "vowelA" with candidates ["あ", "a", "A"] + * + * After (model that ships English morphs): + * - "A" + */ +function resolveMorphName(slot: MorphSlot, available: Set, overrides: Partial>): string | undefined { + const override = overrides[slot] + if (override && available.has(override)) + return override + + for (const candidate of MORPH_CANDIDATES[slot]) { + if (available.has(candidate)) + return candidate + } + return undefined +} + +export interface MorphController { + /** Logical slots that resolved to a real morph on this model. */ + readonly resolvedSlots: MorphSlot[] + /** Every morph name exposed by the model, for settings UIs. */ + readonly availableMorphs: string[] + /** Sets a logical slot's weight, clamped to [0, 1]. No-op if unresolved. */ + set: (slot: MorphSlot, weight: number) => void + /** Reads back the current weight of a logical slot. */ + get: (slot: MorphSlot) => number + /** Forces a slot to bind to an explicit morph name (settings override). */ + override: (slot: MorphSlot, morphName: string) => void + /** Zeroes every slot this controller manages (leaves VMD morphs intact). */ + resetManaged: () => void +} + +/** + * Builds a controller over an MMD mesh's vertex morphs. + * + * MMD exposes morphs through `SkinnedMesh.morphTargetDictionary` (name → + * index) and `morphTargetInfluences` (index → weight). This controller hides + * the index bookkeeping and the per-model name resolution so the expression, + * blink, and lip-sync composables can speak in logical slots. + * + * Managed weights must be written after `MMDAnimationHelper.update()` each + * frame: a VMD clip can also key morph influences, and we want AIRI's + * lip-sync/expression to win for the slots it owns. + */ +export function createMorphController( + mesh: SkinnedMesh, + overrides: Partial> = {}, +): MorphController { + const dictionary = mesh.morphTargetDictionary ?? {} + const available = new Set(Object.keys(dictionary)) + + const slotIndex = new Map() + const slotOverrides: Partial> = { ...overrides } + + function rebind() { + slotIndex.clear() + for (const slot of Object.keys(MORPH_CANDIDATES) as MorphSlot[]) { + const name = resolveMorphName(slot, available, slotOverrides) + if (name !== undefined && dictionary[name] !== undefined) + slotIndex.set(slot, dictionary[name]) + } + } + rebind() + + return { + get resolvedSlots() { + return Array.from(slotIndex.keys()) + }, + get availableMorphs() { + return Array.from(available) + }, + set(slot, weight) { + const index = slotIndex.get(slot) + // Read morphTargetInfluences live: caching it risks writing to a stale + // or detached array if three (re)assigns it during updateMorphTargets. + const influences = mesh.morphTargetInfluences + if (index === undefined || !influences) + return + influences[index] = weight < 0 ? 0 : weight > 1 ? 1 : weight + }, + get(slot) { + const index = slotIndex.get(slot) + const influences = mesh.morphTargetInfluences + return index === undefined || !influences ? 0 : influences[index] ?? 0 + }, + override(slot, morphName) { + slotOverrides[slot] = morphName + rebind() + }, + resetManaged() { + const influences = mesh.morphTargetInfluences + if (!influences) + return + for (const index of slotIndex.values()) + influences[index] = 0 + }, + } +} diff --git a/packages/stage-ui-mmd/src/constants/actions.ts b/packages/stage-ui-mmd/src/constants/actions.ts new file mode 100644 index 000000000..fdc3aa786 --- /dev/null +++ b/packages/stage-ui-mmd/src/constants/actions.ts @@ -0,0 +1,25 @@ +import { Emotion } from './emotions' + +/** + * Optional emotion → motion-name mapping for gesture playback. + * + * Expressions (morphs) are always applied for an emotion; on top of that, if + * the loaded model has a registered VMD motion under one of these names, the + * scene also plays it as a one-shot gesture. Models rarely ship motions named + * exactly like this, so the animation manager treats a missing name as a + * no-op and the user can remap names from the settings panel. + */ +export const EMOTION_ACTION_NAME: Record = { + [Emotion.Happy]: 'happy', + [Emotion.Sad]: 'sad', + [Emotion.Angry]: 'angry', + [Emotion.Think]: 'think', + [Emotion.Surprise]: 'surprise', + [Emotion.Awkward]: 'awkward', + [Emotion.Question]: 'question', + [Emotion.Curious]: 'curious', + [Emotion.Neutral]: 'idle', +} + +/** Conventional name AIRI uses for the persistent looping idle motion. */ +export const IDLE_ACTION_NAME = 'idle' diff --git a/packages/stage-ui-mmd/src/constants/emotions.ts b/packages/stage-ui-mmd/src/constants/emotions.ts new file mode 100644 index 000000000..9e092e2a1 --- /dev/null +++ b/packages/stage-ui-mmd/src/constants/emotions.ts @@ -0,0 +1,19 @@ +/** + * AIRI emotion vocabulary shared across renderers. + * + * Kept identical to the Live2D/Spine enums so the act-event bus in + * `Stage.vue` can drive any renderer with the same emotion names. + */ +export enum Emotion { + Happy = 'happy', + Sad = 'sad', + Angry = 'angry', + Think = 'think', + Surprise = 'surprised', + Awkward = 'awkward', + Question = 'question', + Curious = 'curious', + Neutral = 'neutral', +} + +export const EMOTION_VALUES = Object.values(Emotion) diff --git a/packages/stage-ui-mmd/src/constants/morphs.test.ts b/packages/stage-ui-mmd/src/constants/morphs.test.ts new file mode 100644 index 000000000..81b7d5a9e --- /dev/null +++ b/packages/stage-ui-mmd/src/constants/morphs.test.ts @@ -0,0 +1,40 @@ +import { describe, expect, it } from 'vitest' + +import { EMOTION_ACTION_NAME } from './actions' +import { EMOTION_VALUES } from './emotions' +import { EMOTION_MORPHS, MORPH_CANDIDATES, VOWEL_SLOTS } from './morphs' + +describe('mmd emotion/morph constants', () => { + it('defines a morph state for every emotion', () => { + for (const emotion of EMOTION_VALUES) + expect(EMOTION_MORPHS[emotion]).toBeDefined() + }) + + it('defines an action name for every emotion', () => { + for (const emotion of EMOTION_VALUES) + expect(typeof EMOTION_ACTION_NAME[emotion]).toBe('string') + }) + + it('keeps emotion morph influence weights within [0, 1]', () => { + for (const emotion of EMOTION_VALUES) { + for (const influence of EMOTION_MORPHS[emotion].influences) { + expect(influence.value).toBeGreaterThanOrEqual(0) + expect(influence.value).toBeLessThanOrEqual(1) + } + } + }) + + it('provides candidate names for every vowel slot', () => { + for (const slot of VOWEL_SLOTS) { + expect(MORPH_CANDIDATES[slot].length).toBeGreaterThan(0) + } + }) + + it('lists the canonical あいうえお as the first vowel candidates', () => { + expect(MORPH_CANDIDATES.vowelA[0]).toBe('あ') + expect(MORPH_CANDIDATES.vowelI[0]).toBe('い') + expect(MORPH_CANDIDATES.vowelU[0]).toBe('う') + expect(MORPH_CANDIDATES.vowelE[0]).toBe('え') + expect(MORPH_CANDIDATES.vowelO[0]).toBe('お') + }) +}) diff --git a/packages/stage-ui-mmd/src/constants/morphs.ts b/packages/stage-ui-mmd/src/constants/morphs.ts new file mode 100644 index 000000000..949e2f1ac --- /dev/null +++ b/packages/stage-ui-mmd/src/constants/morphs.ts @@ -0,0 +1,136 @@ +import { Emotion } from './emotions' + +/** + * Logical morph slots AIRI drives on an MMD model. + * + * MMD (PMX/PMD) models expose vertex morphs by Japanese (or, for some + * Western models, English) names. The exact names differ per model, so we + * never hard-code a single string: each logical slot maps to an ordered list + * of candidate morph names, and the runtime resolver (see + * `resolveMorphName`) picks the first candidate that exists on the loaded + * skeleton. Models can still override the resolved name from the settings + * panel. + */ +export type MorphSlot + = | 'vowelA' + | 'vowelI' + | 'vowelU' + | 'vowelE' + | 'vowelO' + | 'blink' + | 'blinkLeft' + | 'blinkRight' + | 'smile' + | 'anger' + | 'sad' + | 'surprise' + | 'troubled' + | 'serious' + +/** + * Ordered candidate morph names per logical slot. + * + * Order matters: the first existing candidate wins. Japanese names come + * first because the overwhelming majority of MMD models ship them; English + * fallbacks cover the minority of Western/Blender-exported models. + */ +export const MORPH_CANDIDATES: Record = { + // Mouth vowels — the canonical MMD lip set (あいうえお). + vowelA: ['あ', 'a', 'A'], + vowelI: ['い', 'i', 'I'], + vowelU: ['う', 'u', 'U'], + vowelE: ['え', 'e', 'E'], + vowelO: ['お', 'o', 'O'], + + // Eyes. + blink: ['まばたき', 'blink', 'Blink'], + blinkLeft: ['ウィンク', 'ウインク', 'wink', 'Wink'], + blinkRight: ['ウィンク右', 'ウインク右', 'ウィンク2', 'wink_r', 'Wink_R'], + + // Expression morphs. Multiple common spellings per emotion are listed + // because model authors are inconsistent (にこり vs 笑い for "smile", etc.). + smile: ['笑い', 'にこり', 'わらい', 'smile', 'Smile'], + anger: ['怒り', 'いかり', 'anger', 'Anger'], + sad: ['悲しい', '悲しむ', 'sad', 'Sad'], + surprise: ['驚き', 'びっくり', 'surprise', 'Surprise'], + troubled: ['困る', 'こまる', 'troubled', 'Troubled'], + serious: ['真面目', 'じと目', 'serious', 'Serious'], +} + +/** Vowel slots in the order used by the lip-sync mixer. */ +export const VOWEL_SLOTS = ['vowelA', 'vowelI', 'vowelU', 'vowelE', 'vowelO'] as const +export type VowelSlot = typeof VOWEL_SLOTS[number] + +/** A single weighted morph target produced by an emotion definition. */ +export interface MorphInfluence { + slot: MorphSlot + /** Full-weight target in [0, 1]; actual applied weight scales by intensity. */ + value: number +} + +/** Per-emotion morph targets plus the cross-fade time used to reach them. */ +export interface EmotionMorphState { + influences: MorphInfluence[] + /** Cross-fade duration in seconds. */ + blendDuration: number +} + +/** + * Maps AIRI emotions to MMD morph targets. + * + * Primary expression weights stay at 0.7–0.8 (not 1.0) to avoid the + * over-expressive look reported for VRM in #590; the secondary mouth morph + * adds a small amount of shape so the face does not read as flat. + */ +export const EMOTION_MORPHS: Record = { + [Emotion.Happy]: { + influences: [{ slot: 'smile', value: 0.8 }, { slot: 'vowelA', value: 0.2 }], + blendDuration: 0.4, + }, + [Emotion.Sad]: { + influences: [{ slot: 'sad', value: 0.75 }, { slot: 'troubled', value: 0.3 }], + blendDuration: 0.4, + }, + [Emotion.Angry]: { + influences: [{ slot: 'anger', value: 0.8 }], + blendDuration: 0.3, + }, + [Emotion.Surprise]: { + influences: [{ slot: 'surprise', value: 0.8 }, { slot: 'vowelO', value: 0.4 }], + blendDuration: 0.15, + }, + [Emotion.Think]: { + influences: [{ slot: 'troubled', value: 0.5 }, { slot: 'serious', value: 0.4 }], + blendDuration: 0.5, + }, + [Emotion.Awkward]: { + influences: [{ slot: 'troubled', value: 0.6 }, { slot: 'smile', value: 0.2 }], + blendDuration: 0.4, + }, + [Emotion.Question]: { + influences: [{ slot: 'troubled', value: 0.4 }, { slot: 'surprise', value: 0.2 }], + blendDuration: 0.4, + }, + [Emotion.Curious]: { + influences: [{ slot: 'surprise', value: 0.35 }, { slot: 'smile', value: 0.25 }], + blendDuration: 0.4, + }, + [Emotion.Neutral]: { + influences: [], + blendDuration: 0.6, + }, +} + +/** + * Standard MMD humanoid bone names AIRI manipulates for gaze and head aim. + * + * These are the de-facto standard Japanese bone names ("semi-standard bone" + * convention) shared by the vast majority of MMD models. + */ +export const MMD_BONE = { + bothEyes: '両目', + leftEye: '左目', + rightEye: '右目', + head: '頭', + neck: '首', +} as const diff --git a/packages/stage-ui-mmd/src/index.ts b/packages/stage-ui-mmd/src/index.ts new file mode 100644 index 000000000..1de474a17 --- /dev/null +++ b/packages/stage-ui-mmd/src/index.ts @@ -0,0 +1,11 @@ +export { default as MMDScene } from './components/scenes/MMD.vue' +export * from './composables/mmd' +export * from './constants/actions' +export * from './constants/emotions' +export * from './constants/morphs' +export * from './stores' +export * from './utils/ammo' +export * from './utils/mmd-loader' +export * from './utils/mmd-preview' +export * from './utils/mmd-validator' +export * from './utils/mmd-zip-loader' diff --git a/packages/stage-ui-mmd/src/stores/index.ts b/packages/stage-ui-mmd/src/stores/index.ts new file mode 100644 index 000000000..afb20e050 --- /dev/null +++ b/packages/stage-ui-mmd/src/stores/index.ts @@ -0,0 +1,2 @@ +export * from './mmd' +export * from './view-control' diff --git a/packages/stage-ui-mmd/src/stores/mmd.ts b/packages/stage-ui-mmd/src/stores/mmd.ts new file mode 100644 index 000000000..a449288a3 --- /dev/null +++ b/packages/stage-ui-mmd/src/stores/mmd.ts @@ -0,0 +1,328 @@ +import type { Emotion } from '../constants/emotions' +import type { MorphSlot } from '../constants/morphs' + +import localforage from 'localforage' + +import { useLocalStorageManualReset } from '@proj-airi/stage-shared/composables' +import { useBroadcastChannel } from '@vueuse/core' +import { defineStore } from 'pinia' +import { ref, watch } from 'vue' + +import { EMOTION_ACTION_NAME } from '../constants/actions' +import { supportedControl, useMMDViewControl } from './view-control' + +type BroadcastChannelEvents + = | BroadcastChannelEventShouldUpdateView + | BroadcastChannelEventPlayOneShot + +interface BroadcastChannelEventShouldUpdateView { + type: 'mmd-should-update-view' +} + +interface BroadcastChannelEventPlayOneShot { + type: 'mmd-play-one-shot' + request: MMDOneShotAction +} + +/** + * A VMD motion imported for use across windows. + * + * Holds only an id + display name; the VMD file itself lives in IndexedDB + * (keyed by `id`) so the list can sync through localStorage and each window + * can materialize its own object URL. Blob URLs are window-scoped and cannot + * be shared, which is why the URL is never persisted here. + */ +export interface MMDMotionDescriptor { + id: string + name: string +} + +/** IndexedDB record for a persisted VMD file. */ +interface PersistedMMDMotion { + id: string + name: string + file: File +} + +/** A material ("part") of the loaded model, for the materials settings UI. */ +export interface MMDMaterialDescriptor { + /** Raw material name; the key used for opacity overrides. */ + name: string + /** Human-friendly label (falls back to `Material N` for unnamed parts). */ + label: string + /** Order index within the model. */ + index: number +} + +const MOTION_STORAGE_PREFIX = 'mmd-motion-' + +/** Eye/head tracking mode, mirroring the VRM renderer's tracking modes. */ +export type MMDGazeMode = 'camera' | 'mouse' | 'none' + +/** Transient request to play a one-shot motion, bumped per request. */ +export interface MMDOneShotAction { + name: string + loop: boolean + nonce: number +} + +export const useMMD = defineStore('mmd', () => { + const { post, data } = useBroadcastChannel({ + name: 'airi-stores-stage-ui-mmd', + }) + const shouldUpdateViewHooks = ref(new Set<() => void>()) + + /** + * Transient one-shot motion request (fire-and-forget; not persisted). + * + * Declared before the broadcast handler that writes to it. Delivered across + * Electron windows via the BroadcastChannel below — NOT localStorage, whose + * storage events do not fire reliably between BrowserWindows. + */ + const oneShotAction = ref() + + const onShouldUpdateView = (hook: () => void) => { + shouldUpdateViewHooks.value.add(hook) + return () => { + shouldUpdateViewHooks.value.delete(hook) + } + } + + function shouldUpdateView() { + post({ type: 'mmd-should-update-view' }) + shouldUpdateViewHooks.value.forEach(hook => hook()) + } + + watch(data, (event) => { + if (event?.type === 'mmd-should-update-view') + shouldUpdateViewHooks.value.forEach(hook => hook()) + else if (event?.type === 'mmd-play-one-shot') + oneShotAction.value = event.request + }) + + // === Physics & solver toggles === + /** Master switch for the Ammo/Bullet rigid-body simulation (hair, skirt, etc.). */ + const physicsEnabled = useLocalStorageManualReset('settings/mmd/physics-enabled', true) + /** CCD IK solving for limbs/standard rigs. */ + const ikEnabled = useLocalStorageManualReset('settings/mmd/ik-enabled', true) + /** Append-bone ("grant") propagation for derived bones. */ + const grantEnabled = useLocalStorageManualReset('settings/mmd/grant-enabled', true) + /** + * Physics gravity strength (magnitude). Applied as world gravity (0, -g, 0). + * MMD's default is 98 (9.8 × 10 for MMD's scale). Lower = floatier hair and + * cloth, higher = heavier/droopier. + */ + const physicsGravity = useLocalStorageManualReset('settings/mmd/physics-gravity', 98) + + // === Gaze === + /** + * Eye/head tracking mode, mirroring the VRM renderer: + * - `mouse` — follow the cursor + * - `camera` — look toward the camera (forward) + * - `none` — no tracking; idle saccades only + */ + const gazeMode = useLocalStorageManualReset('settings/mmd/gaze-mode', 'mouse') + + // === Scene: camera === + /** Vertical field of view in degrees. */ + const cameraFov = useLocalStorageManualReset('settings/mmd/camera-fov', 30) + + // === Scene: lighting === + const ambientColor = useLocalStorageManualReset('settings/mmd/ambient-color', '#FFFFFF') + const ambientIntensity = useLocalStorageManualReset('settings/mmd/ambient-intensity', 0.6) + const directionalColor = useLocalStorageManualReset('settings/mmd/directional-color', '#FFFBF5') + const directionalIntensity = useLocalStorageManualReset('settings/mmd/directional-intensity', 0.75) + const directionalPosition = useLocalStorageManualReset<{ x: number, y: number, z: number }>( + 'settings/mmd/directional-position', + () => ({ x: 1, y: 2, z: 2 }), + ) + + // === Scene: rendering === + /** + * Albedo self-illumination, 0–1. Drives `emissiveIntensity` on every + * material so the model reads as the flat, luminous MMD/anime look. 0 = lit + * only by scene lights, 1 = nearly self-lit. + */ + const albedoGlow = useLocalStorageManualReset('settings/mmd/albedo-glow', 0.45) + /** Device-pixel-ratio multiplier for render resolution. */ + const renderScale = useLocalStorageManualReset('settings/mmd/render-scale', 1) + + // === Animation === + /** Name of the persistent idle motion; empty means none/static. */ + const idleMotionName = useLocalStorageManualReset('settings/mmd/idle-motion', '') + /** + * Motions imported for use across windows. + * + * localStorage-backed so the list syncs between the settings window and the + * stage window (the desktop app runs them as separate renderer processes + * with independent Pinia stores). Only id + name are stored here; the VMD + * files live in IndexedDB. + */ + const availableMotions = useLocalStorageManualReset('settings/mmd/motions', () => []) + + /** + * Persists a VMD file to IndexedDB and adds it to the synced list. + * + * Re-importing the same name overwrites the stored file (same id) so the + * list stays stable. + */ + async function addMotion(file: File): Promise { + const name = file.name.replace(/\.vmd$/i, '') + const existing = availableMotions.value.find(motion => motion.name === name) + const id = existing?.id ?? `${MOTION_STORAGE_PREFIX}${crypto.randomUUID()}` + + await localforage.setItem(id, { id, name, file }) + if (!existing) + availableMotions.value = [...availableMotions.value, { id, name }] + + return { id, name } + } + + /** Loads a persisted VMD file by id so a window can build its own object URL. */ + async function getMotionFile(id: string): Promise { + const persisted = await localforage.getItem(id) + return persisted?.file + } + + /** Removes every persisted motion file and clears the synced list. */ + async function clearMotions(): Promise { + const motions = availableMotions.value + availableMotions.value = [] + await Promise.all(motions.map(motion => localforage.removeItem(motion.id))) + } + + /** Removes a single imported motion (from the synced list and IndexedDB). */ + async function removeMotion(id: string): Promise { + const motion = availableMotions.value.find(m => m.id === id) + availableMotions.value = availableMotions.value.filter(m => m.id !== id) + // Drop the idle selection if it pointed at the removed motion. + if (motion && idleMotionName.value === motion.name) + idleMotionName.value = '' + await localforage.removeItem(id) + } + /** Per-emotion gesture motion overrides; falls back to EMOTION_ACTION_NAME. */ + const emotionActionMap = useLocalStorageManualReset>( + 'settings/mmd/emotion-action-map', + () => ({ ...EMOTION_ACTION_NAME }), + ) + + // === Morphs === + /** Manual morph-slot → morph-name overrides for non-standard models. */ + const morphOverrides = useLocalStorageManualReset>>( + 'settings/mmd/morph-overrides', + () => ({}), + ) + /** All morph names exposed by the active model (runtime). */ + const availableMorphs = useLocalStorageManualReset('settings/mmd/available-morphs', () => []) + + /** + * Materials (parts) of the active model, published by the scene so the + * settings window can render a control per part. + */ + const availableMaterials = useLocalStorageManualReset( + 'settings/mmd/available-materials', + () => [], + ) + /** + * Per-material opacity overrides, keyed by material name. Missing entries + * render at full opacity. Lets the user fade or hide individual parts. + */ + const materialOpacity = useLocalStorageManualReset>( + 'settings/mmd/material-opacity', + () => ({}), + ) + + /** + * Whether an MMD mesh is currently mounted. Runtime-only: the persisted + * descriptor lists survive reloads and cannot indicate live mount state. + */ + const isModelLoaded = ref(false) + + /** + * Queues a one-shot motion. Applies it in this window and broadcasts it to + * the others. The bumped `nonce` makes repeat requests for the same motion + * re-trigger the scene's watcher. + * + * Uses the BroadcastChannel rather than localStorage because this is a + * transient trigger that must reach the stage window's animation manager, + * and localStorage storage events do not fire reliably across Electron + * BrowserWindows (the live stage and settings run as separate processes). + */ + function playOneShotAction(name: string, loop = false) { + const request: MMDOneShotAction = { name, loop, nonce: (oneShotAction.value?.nonce ?? 0) + 1 } + oneShotAction.value = request + post({ type: 'mmd-play-one-shot', request }) + } + + const { position, scale, rotationY, reset: resetViewControl } = useMMDViewControl() + + function resetState() { + supportedControl.forEach(c => resetViewControl(c)) + physicsEnabled.reset() + ikEnabled.reset() + grantEnabled.reset() + physicsGravity.reset() + gazeMode.reset() + cameraFov.reset() + ambientColor.reset() + ambientIntensity.reset() + directionalColor.reset() + directionalIntensity.reset() + directionalPosition.reset() + albedoGlow.reset() + renderScale.reset() + idleMotionName.reset() + void clearMotions() + oneShotAction.value = undefined + emotionActionMap.reset() + morphOverrides.reset() + availableMorphs.reset() + availableMaterials.reset() + materialOpacity.reset() + shouldUpdateView() + } + + return { + position, + scale, + rotationY, + + physicsEnabled, + ikEnabled, + grantEnabled, + physicsGravity, + gazeMode, + + cameraFov, + ambientColor, + ambientIntensity, + directionalColor, + directionalIntensity, + directionalPosition, + albedoGlow, + renderScale, + + idleMotionName, + availableMotions, + addMotion, + getMotionFile, + clearMotions, + removeMotion, + emotionActionMap, + + morphOverrides, + availableMorphs, + availableMaterials, + materialOpacity, + + isModelLoaded, + oneShotAction, + playOneShotAction, + + onShouldUpdateView, + shouldUpdateView, + resetState, + } +}) + +export { useMMDViewControl } diff --git a/packages/stage-ui-mmd/src/stores/view-control.ts b/packages/stage-ui-mmd/src/stores/view-control.ts new file mode 100644 index 000000000..2aff677d9 --- /dev/null +++ b/packages/stage-ui-mmd/src/stores/view-control.ts @@ -0,0 +1,55 @@ +import { useLocalStorage } from '@vueuse/core' +import { ref } from 'vue' + +export const supportedControl = ['x', 'y', 'scale', 'rotationY'] as const +type SupportedControl = typeof supportedControl[number] +interface ControlConfig { min: number, max: number, step: number, default: number, format: (val: number) => string } + +const viewControlsEnabled = ref(false) +const viewControlMode = ref('scale') + +/** Model world offset from origin, in scene units. */ +const position = useLocalStorage<{ x: number, y: number }>('settings/mmd/position', { x: 0, y: 0 }) +/** Uniform model scale. MMD models are authored large, so default is small. */ +const scale = useLocalStorage('settings/mmd/scale', 0.1) +/** Yaw applied to the model group, in radians. */ +const rotationY = useLocalStorage('settings/mmd/rotationY', 0) + +const formatUnits = (val: number) => val.toFixed(2) +const formatToPercent = (val: number) => `${(val * 100).toFixed(0)}%` +const formatDegrees = (val: number) => `${(val * 180 / Math.PI).toFixed(0)}°` + +export const controlConfig: Record = { + x: { min: -20, max: 20, step: 0.1, default: 0, format: formatUnits }, + y: { min: -20, max: 20, step: 0.1, default: 0, format: formatUnits }, + scale: { min: 0.01, max: 1, step: 0.01, default: 0.1, format: formatToPercent }, + rotationY: { min: -Math.PI, max: Math.PI, step: 0.01, default: 0, format: formatDegrees }, +} + +export function useMMDViewControl() { + function reset(key: SupportedControl) { + switch (key) { + case 'x': + position.value.x = controlConfig.x.default + break + case 'y': + position.value.y = controlConfig.y.default + break + case 'scale': + scale.value = controlConfig.scale.default + break + case 'rotationY': + rotationY.value = controlConfig.rotationY.default + break + } + } + + return { + position, + scale, + rotationY, + reset, + viewControlsEnabled, + viewControlMode, + } +} diff --git a/packages/stage-ui-mmd/src/utils/ammo.ts b/packages/stage-ui-mmd/src/utils/ammo.ts new file mode 100644 index 000000000..d38b32960 --- /dev/null +++ b/packages/stage-ui-mmd/src/utils/ammo.ts @@ -0,0 +1,49 @@ +import type Ammo from 'ammojs-typed' + +/** + * Lazily initializes the Ammo.js (Bullet) physics runtime and exposes it as + * the global `Ammo` that three-stdlib's `MMDPhysics` expects. + * + * Why a global: `MMDPhysics` (a straight port of three's example) reads + * `Ammo.btVector3`, `Ammo.btRigidBody`, etc. off the global scope rather + * than taking the runtime as a constructor argument. We therefore have to + * publish the resolved module on `globalThis` before constructing any + * physics world. + * + * Why lazy: the Ammo WASM binary plus its JS glue is large (~1 MB+). It is + * pulled in via dynamic `import()` so neither the glue nor the WASM lands in + * the main bundle until the user actually mounts an MMD model. + * + * The promise is memoized: concurrent callers and re-mounts share a single + * WASM instantiation. + */ +let ammoReady: Promise | undefined + +interface AmmoGlobal { + Ammo?: typeof Ammo +} + +export async function ensureAmmo(): Promise { + if (ammoReady) + return ammoReady + + ammoReady = import('ammojs-typed') + .then(module => module.default()) + .then((lib) => { + // NOTICE: + // MMDPhysics resolves Bullet classes from the ambient global `Ammo`. + // Root cause: three-stdlib/animation/MMDPhysics.js does `typeof Ammo` + // and `new Ammo.btVector3(...)` against the global scope. + // Source: node_modules/three-stdlib/animation/MMDPhysics.js (lines 14, 98+). + // Removal condition: three-stdlib accepts an injected Ammo instance. + ;(globalThis as AmmoGlobal).Ammo = lib + return lib + }) + + return ammoReady +} + +/** Whether the Ammo runtime has already been published on the global scope. */ +export function isAmmoReady(): boolean { + return Boolean((globalThis as AmmoGlobal).Ammo) +} diff --git a/packages/stage-ui-mmd/src/utils/mmd-loader.test.ts b/packages/stage-ui-mmd/src/utils/mmd-loader.test.ts new file mode 100644 index 000000000..d4d9ef238 --- /dev/null +++ b/packages/stage-ui-mmd/src/utils/mmd-loader.test.ts @@ -0,0 +1,28 @@ +import { describe, expect, it } from 'vitest' + +import { withModelExtension } from './mmd-loader' + +describe('withModelExtension', () => { + // ROOT CAUSE: + // + // MMDLoader.load() chooses the PMX/PMD parser from the URL file extension + // (_extractExtension -> lastIndexOf('.')). Object/blob URLs produced by + // URL.createObjectURL have no extension, so importing an MMD .zip failed with + // "THREE.MMDLoader: Unknown model file extension .". + // + // We append the known format as a URL fragment so the extension sniff + // succeeds; the blob URL store ignores the fragment when fetching. + it('tags an extensionless blob URL with the known format (Issue: blob import)', () => { + expect(withModelExtension('blob:http://host/9f1c-abc', 'pmx')).toBe('blob:http://host/9f1c-abc#airi-model.pmx') + expect(withModelExtension('blob:http://host/9f1c-abc', 'pmd')).toBe('blob:http://host/9f1c-abc#airi-model.pmd') + }) + + it('leaves URLs that already carry a real extension unchanged', () => { + expect(withModelExtension('https://cdn/models/miku.pmx', 'pmx')).toBe('https://cdn/models/miku.pmx') + expect(withModelExtension('https://cdn/models/model.PMD', 'pmd')).toBe('https://cdn/models/model.PMD') + }) + + it('ignores query/fragment when checking for an existing extension', () => { + expect(withModelExtension('https://cdn/miku.pmx?v=2', 'pmx')).toBe('https://cdn/miku.pmx?v=2') + }) +}) diff --git a/packages/stage-ui-mmd/src/utils/mmd-loader.ts b/packages/stage-ui-mmd/src/utils/mmd-loader.ts new file mode 100644 index 000000000..3a2894442 --- /dev/null +++ b/packages/stage-ui-mmd/src/utils/mmd-loader.ts @@ -0,0 +1,206 @@ +import type { Color, LoadingManager, Material, SkinnedMesh, Texture } from 'three' + +import type { MMDLoadedAssets, MMDModelFormat } from './mmd-zip-loader' + +import { Mesh, SRGBColorSpace } from 'three' + +import { createMMDLoaderContext, loadMMDMesh } from '../composables/mmd/loader' +import { loadMMDZip } from './mmd-zip-loader' + +export interface ResolvedMMDModel { + mesh: SkinnedMesh + format: MMDModelFormat + /** Present only when the source was a ZIP archive. */ + assets?: MMDLoadedAssets + /** Revokes any blob URLs created while resolving the model. */ + dispose: () => void +} + +export interface LoadMMDOptions { + /** + * Wait for the model's textures to finish loading before resolving. + * + * MMDLoader resolves the mesh as soon as it is parsed; textures continue + * loading through the LoadingManager. The live scene renders continuously so + * textures appear within a frame or two, but a one-shot offscreen render + * (the preview) would capture an untextured/transparent frame. Enable this + * for previews. Defaults to `false`. + */ + waitForTextures?: boolean +} + +// ZIP local-file-header magic: "PK\x03\x04". +function isZip(buffer: ArrayBuffer): boolean { + const head = new Uint8Array(buffer, 0, Math.min(4, buffer.byteLength)) + return head[0] === 0x50 && head[1] === 0x4B && head[2] === 0x03 && head[3] === 0x04 +} + +/** + * Resolves once the manager has no more pending loads. + * + * `LoadingManager.onLoad` fires when the last queued item finishes. A timeout + * guards the case where everything is already loaded (so `onLoad` never fires) + * or a texture stalls, so preview generation can never hang. + */ +function waitForManagerIdle(manager: LoadingManager, timeoutMs = 4000): Promise { + return new Promise((resolve) => { + let settled = false + let timer: ReturnType + const finish = () => { + if (settled) + return + settled = true + clearTimeout(timer) + resolve() + } + timer = setTimeout(finish, timeoutMs) + manager.onLoad = finish + }) +} + +/** Material with the slots we adjust for correct MMD shading under r184. */ +type ColorMappedMaterial = Material & { + map?: Texture | null + emissiveMap?: Texture | null + emissive?: Color + emissiveIntensity?: number + color?: Color +} + +/** Fraction of the albedo fed back as self-illumination for the anime glow. */ +const MMD_ALBEDO_GLOW = 0.45 + +/** + * Corrects MMD materials for three r184 and gives them the flat, luminous + * anime look, after load. + * + * Fixes: + * + * 1. Color space — three-stdlib's MMDLoader predates the + * `encoding` → `colorSpace` migration and assigns color textures without a + * color space, so under r184 they decode in linear space and read too + * bright/desaturated. We retag color maps as sRGB; data maps + * (normal/gradient/sphere) stay linear. + * + * 2. Baked ambient → albedo glow — MMDLoader maps each PMX material's ambient + * color (環境色, a strong grey) onto `material.emissive`, which washes the + * model out as a flat grey. MMD's actual look is a bright, slightly-shaded + * albedo with a soft self-glow. We replace the grey emissive with the + * material's own diffuse map (or color) at {@link MMD_ALBEDO_GLOW} + * intensity, so each surface self-illuminates in its own color — skin glows + * skin-colored — instead of grey. + */ +function fixupMMDMaterials(mesh: SkinnedMesh): void { + mesh.traverse((object) => { + if (!(object instanceof Mesh)) + return + // Skinned MMD meshes report a bind-pose bounding sphere that does not + // cover the posed/animated mesh, so they get frustum-culled when the + // camera pulls back (e.g. the offscreen preview renders blank). Disable + // culling, as the VRM loader does. + object.frustumCulled = false + const materials = Array.isArray(object.material) ? object.material : [object.material] + for (const material of materials) { + const mapped = material as ColorMappedMaterial + + if (mapped.map) + mapped.map.colorSpace = SRGBColorSpace + + if (mapped.emissive) { + if (mapped.map) { + // Self-illuminate from the albedo: emissive = white × diffuse map. + mapped.emissiveMap = mapped.map + mapped.emissive.setScalar(1) + } + else if (mapped.color) { + // No texture: glow in the flat diffuse color instead. + mapped.emissive.copy(mapped.color) + } + else { + mapped.emissive.setScalar(0) + } + if (typeof mapped.emissiveIntensity === 'number') + mapped.emissiveIntensity = MMD_ALBEDO_GLOW + } + + if (mapped.emissiveMap) + mapped.emissiveMap.colorSpace = SRGBColorSpace + + material.needsUpdate = true + } + }) +} + +function formatFromUrl(url: string): MMDModelFormat { + return url.split(/[?#]/)[0].toLowerCase().endsWith('.pmd') ? 'pmd' : 'pmx' +} + +/** + * Ensures a model URL ends with a `.pmx`/`.pmd` extension that + * `MMDLoader` can sniff. + * + * MMDLoader chooses the PMX vs PMD parser purely from the URL's file + * extension (`_extractExtension` → `lastIndexOf('.')`). Object/blob URLs from + * `URL.createObjectURL` have no extension, so the loader throws "Unknown model + * file extension". We append the known format as a URL fragment: the blob URL + * store ignores the fragment when fetching the blob, but the extension sniff + * reads it. URLs that already carry a real extension are returned unchanged. + * + * Before: + * - "blob:http://host/9f1c-…" (format known to be pmx) + * + * After: + * - "blob:http://host/9f1c-…#airi-model.pmx" + */ +export function withModelExtension(url: string, format: MMDModelFormat): string { + const path = url.split(/[?#]/)[0].toLowerCase() + if (path.endsWith('.pmx') || path.endsWith('.pmd')) + return url + return `${url}#airi-model.${format}` +} + +/** + * Loads an MMD model from an arbitrary source URL into a {@link SkinnedMesh}. + * + * Accepts either a packaged ZIP (the usual distribution form: model plus + * textures) or a bare `.pmx`/`.pmd` URL. ZIP archives are unpacked to blob + * URLs and a basename-based texture resolver is installed on the loader; raw + * URLs are loaded directly and rely on the server's relative paths. + * + * The returned `dispose()` revokes any blob URLs created during the load. It + * does not dispose the mesh's GPU resources — the scene owns that lifecycle. + */ +export async function loadMMDModelFromSource(src: string, options: LoadMMDOptions = {}): Promise { + const response = await fetch(src) + if (!response.ok) + throw new Error(`Failed to fetch MMD model: ${response.status} ${response.statusText}`) + + const buffer = await response.arrayBuffer() + + if (isZip(buffer)) { + const assets = await loadMMDZip(buffer) + const { loader, manager } = createMMDLoaderContext(assets.urlModifier) + const mesh = await loadMMDMesh(loader, withModelExtension(assets.modelBlobUrl, assets.variant.format)) + fixupMMDMaterials(mesh) + if (options.waitForTextures) + await waitForManagerIdle(manager) + return { + mesh, + format: assets.variant.format, + assets, + dispose: () => assets.dispose(), + } + } + + // Raw model URL: load directly, textures resolve against the server path. + const { loader, manager } = createMMDLoaderContext() + const mesh = await loadMMDMesh(loader, withModelExtension(src, formatFromUrl(src))) + fixupMMDMaterials(mesh) + if (options.waitForTextures) + await waitForManagerIdle(manager) + return { + mesh, + format: formatFromUrl(src), + dispose: () => {}, + } +} diff --git a/packages/stage-ui-mmd/src/utils/mmd-preview.ts b/packages/stage-ui-mmd/src/utils/mmd-preview.ts new file mode 100644 index 000000000..7d99df9f9 --- /dev/null +++ b/packages/stage-ui-mmd/src/utils/mmd-preview.ts @@ -0,0 +1,98 @@ +import type { Object3D } from 'three' + +import { + AmbientLight, + Box3, + DirectionalLight, + Group, + Mesh, + PerspectiveCamera, + Scene, + SRGBColorSpace, + Vector3, + WebGLRenderer, +} from 'three' + +import { loadMMDModelFromSource } from './mmd-loader' + +function disposeObject(root: Object3D) { + root.traverse((obj) => { + if (obj instanceof Mesh) { + obj.geometry?.dispose?.() + const material = obj.material + if (Array.isArray(material)) + material.forEach(m => m.dispose()) + else + material?.dispose?.() + } + }) +} + +/** + * Renders an MMD model file to an offscreen canvas and returns a preview data + * URL for the model-selector card. + * + * Deliberately physics-free: the thumbnail shows the model in its rest pose, + * so there is no need to initialize Ammo or step the simulation. This keeps + * preview generation cheap and avoids loading the WASM physics binary just to + * import a model. + */ +export async function loadMMDModelPreview(file: File): Promise { + const canvas = document.createElement('canvas') + canvas.width = 1440 + canvas.height = 2560 + + const renderer = new WebGLRenderer({ canvas, alpha: true, antialias: true, preserveDrawingBuffer: true }) + renderer.outputColorSpace = SRGBColorSpace + renderer.setSize(canvas.width, canvas.height, false) + renderer.setPixelRatio(1) + + const scene = new Scene() + const camera = new PerspectiveCamera(30, canvas.width / canvas.height, 0.1, 1000) + scene.add(new AmbientLight(0xFFFFFF, 0.9)) + const directional = new DirectionalLight(0xFFFFFF, 1.0) + directional.position.set(1, 2, 2) + scene.add(directional) + + const objectUrl = URL.createObjectURL(file) + let resolved: Awaited> | undefined + let group: Group | undefined + + try { + resolved = await loadMMDModelFromSource(objectUrl, { waitForTextures: true }) + group = new Group() + group.add(resolved.mesh) + scene.add(group) + group.updateMatrixWorld(true) + + const box = new Box3().setFromObject(group) + const size = box.getSize(new Vector3()) + const center = box.getCenter(new Vector3()) + // Fit to both axes for the portrait preview canvas so the model is not + // cropped or shrunk into the distance. + const vFov = (camera.fov * Math.PI) / 180 + const fitHeightDistance = (size.y / 2) / Math.tan(vFov / 2) + const fitWidthDistance = (size.x / 2) / (Math.tan(vFov / 2) * camera.aspect) + const distance = 1.15 * Math.max(fitHeightDistance, fitWidthDistance) + camera.position.set(center.x, center.y, center.z + distance) + camera.near = Math.max(distance / 100, 0.01) + camera.far = distance * 100 + camera.lookAt(center) + camera.updateProjectionMatrix() + + renderer.render(scene, camera) + return canvas.toDataURL() + } + finally { + if (group) + disposeObject(group) + resolved?.dispose() + scene.clear() + renderer.renderLists.dispose() + renderer.dispose() + renderer.forceContextLoss() + URL.revokeObjectURL(objectUrl) + canvas.width = 0 + canvas.height = 0 + } +} diff --git a/packages/stage-ui-mmd/src/utils/mmd-validator.ts b/packages/stage-ui-mmd/src/utils/mmd-validator.ts new file mode 100644 index 000000000..5ed05bf78 --- /dev/null +++ b/packages/stage-ui-mmd/src/utils/mmd-validator.ts @@ -0,0 +1,69 @@ +import type { MMDModelFormat } from './mmd-zip-loader' + +import JSZip from 'jszip' + +import { errorMessageFrom } from '@moeru/std' + +export type MMDValidationStatus = 'VALID' | 'INVALID' + +export interface MMDValidationReport { + status: MMDValidationStatus + errors: string[] + warnings: string[] + detected: { + modelPath?: string + format?: MMDModelFormat + textureCount: number + } +} + +const TEXTURE_RE = /\.(?:png|jpe?g|bmp|tga|gif|dds|spa|sph|webp)$/i + +/** + * Inspects an MMD ZIP without decoding textures into GPU memory. + * + * Mirrors the Live2D/Spine validator return shape so the model-selector + * dialog can present consistent error/warning UX across formats. + */ +export async function validateMMDZip(file: File): Promise { + const errors: string[] = [] + const warnings: string[] = [] + const detected: MMDValidationReport['detected'] = { textureCount: 0 } + + try { + const zip = new JSZip() + const archive = await zip.loadAsync(file) + const files = Object.keys(archive.files).filter(name => !archive.files[name].dir) + + const pmx = files.filter(name => name.toLowerCase().endsWith('.pmx')) + const pmd = files.filter(name => name.toLowerCase().endsWith('.pmd')) + + if (pmx.length === 0 && pmd.length === 0) { + errors.push('No model (`.pmx` or `.pmd`) found in the ZIP.') + return { status: 'INVALID', errors, warnings, detected } + } + + if (pmx.length + pmd.length > 1) + warnings.push(`Multiple model files detected (${pmx.length + pmd.length}). The import will use the first one.`) + + if (pmx.length > 0) { + detected.modelPath = pmx[0] + detected.format = 'pmx' + } + else { + detected.modelPath = pmd[0] + detected.format = 'pmd' + } + + const textures = files.filter(name => TEXTURE_RE.test(name)) + detected.textureCount = textures.length + if (textures.length === 0) + warnings.push('No texture files detected. The model may render untextured.') + } + catch (err) { + errors.push(`Failed to read ZIP: ${errorMessageFrom(err) ?? 'Unknown error'}`) + return { status: 'INVALID', errors, warnings, detected } + } + + return { status: 'VALID', errors, warnings, detected } +} diff --git a/packages/stage-ui-mmd/src/utils/mmd-zip-loader.test.ts b/packages/stage-ui-mmd/src/utils/mmd-zip-loader.test.ts new file mode 100644 index 000000000..f07a419fa --- /dev/null +++ b/packages/stage-ui-mmd/src/utils/mmd-zip-loader.test.ts @@ -0,0 +1,35 @@ +import { describe, expect, it } from 'vitest' + +import { detectMMDVariants } from './mmd-zip-loader' + +describe('detectMMDVariants', () => { + it('detects a PMX model and derives its display name', () => { + const variants = detectMMDVariants(['Miku/Miku.pmx', 'Miku/tex/face.png']) + expect(variants).toHaveLength(1) + expect(variants[0].modelPath).toBe('Miku/Miku.pmx') + expect(variants[0].format).toBe('pmx') + expect(variants[0].name).toBe('Miku') + }) + + it('detects a PMD model', () => { + const variants = detectMMDVariants(['model.pmd', 'toon01.bmp']) + expect(variants).toHaveLength(1) + expect(variants[0].format).toBe('pmd') + }) + + it('returns every model file when an archive bundles several', () => { + const variants = detectMMDVariants(['a/a.pmx', 'b/b.pmx', 'shared/tex.png']) + expect(variants.map(v => v.modelPath)).toEqual(['a/a.pmx', 'b/b.pmx']) + }) + + it('returns an empty list when no model file is present', () => { + const variants = detectMMDVariants(['readme.txt', 'tex/face.png']) + expect(variants).toHaveLength(0) + }) + + it('is case-insensitive on the model extension', () => { + const variants = detectMMDVariants(['Model.PMX']) + expect(variants).toHaveLength(1) + expect(variants[0].format).toBe('pmx') + }) +}) diff --git a/packages/stage-ui-mmd/src/utils/mmd-zip-loader.ts b/packages/stage-ui-mmd/src/utils/mmd-zip-loader.ts new file mode 100644 index 000000000..1e9348e1a --- /dev/null +++ b/packages/stage-ui-mmd/src/utils/mmd-zip-loader.ts @@ -0,0 +1,151 @@ +import type { UrlModifier } from '../composables/mmd/loader' + +import JSZip from 'jszip' + +export type MMDModelFormat = 'pmx' | 'pmd' + +export interface MMDModelVariant { + /** Display name derived from the model file's basename. */ + name: string + /** In-archive path of the `.pmx`/`.pmd` file. */ + modelPath: string + format: MMDModelFormat +} + +export interface MMDLoadedAssets { + /** Primary model variant (first discovered). */ + variant: MMDModelVariant + /** All model files found in the archive. */ + variants: MMDModelVariant[] + /** Blob URL for the primary model file, passed to `MMDLoader.load`. */ + modelBlobUrl: string + /** Blob URLs for every archive entry, keyed by in-archive path. */ + blobUrls: Record + /** + * Resolves an in-PMX texture reference to a blob URL. + * + * Install on the loader's `LoadingManager.setURLModifier`. PMX files store + * textures as relative paths, but the model is loaded from a `blob:` URL + * whose base cannot resolve those relatives, so we match by basename (and + * fall back to a normalized full path). Texture basenames are effectively + * unique within a single model, which makes this reliable in practice. + */ + urlModifier: UrlModifier + /** Revokes every blob URL allocated for this load. */ + dispose: () => void +} + +const MODEL_EXTS: Record = { '.pmx': 'pmx', '.pmd': 'pmd' } + +function basename(path: string): string { + const norm = path.replace(/\\/g, '/') + const slash = norm.lastIndexOf('/') + return slash === -1 ? norm : norm.slice(slash + 1) +} + +function stripExt(name: string): string { + const dot = name.lastIndexOf('.') + return dot === -1 ? name : name.slice(0, dot) +} + +function modelFormatOf(path: string): MMDModelFormat | undefined { + const lower = path.toLowerCase() + for (const [ext, format] of Object.entries(MODEL_EXTS)) { + if (lower.endsWith(ext)) + return format + } + return undefined +} + +/** Detects every PMX/PMD model file inside a zip's entry list. */ +export function detectMMDVariants(paths: string[]): MMDModelVariant[] { + const variants: MMDModelVariant[] = [] + for (const path of paths) { + const format = modelFormatOf(path) + if (!format) + continue + variants.push({ name: stripExt(basename(path)), modelPath: path, format }) + } + return variants +} + +/** + * Builds a basename-keyed resolver from the archive's blob URLs. + * + * PMX texture references can include subdirectories and backslashes; we + * normalize the requested URL down to its basename and look that up, with a + * normalized full-path fallback for the rare case of duplicate basenames. + */ +function createUrlModifier(blobUrls: Record): UrlModifier { + const byBasename = new Map() + const byPath = new Map() + for (const [path, url] of Object.entries(blobUrls)) { + byPath.set(path.replace(/\\/g, '/').toLowerCase(), url) + byBasename.set(basename(path).toLowerCase(), url) + } + + return (requested: string) => { + const clean = requested.split(/[?#]/)[0] + let decoded = clean + try { + decoded = decodeURIComponent(clean) + } + catch {} + + const normalizedPath = decoded.replace(/\\/g, '/').toLowerCase() + for (const [path, url] of byPath) { + if (normalizedPath.endsWith(path)) + return url + } + + const base = basename(decoded).toLowerCase() + return byBasename.get(base) ?? requested + } +} + +/** + * Loads an MMD model packaged as a ZIP (the common distribution format: a + * `.pmx`/`.pmd` plus its texture/toon/sphere-map files) into blob URLs ready + * for {@link createMMDLoaderContext}. + * + * Call `dispose()` on unmount or reload to revoke the blob URLs. + */ +export async function loadMMDZip(file: File | Blob | ArrayBuffer): Promise { + const zip = new JSZip() + const archive = await zip.loadAsync(file) + + const paths = Object.keys(archive.files).filter(name => !archive.files[name].dir) + const variants = detectMMDVariants(paths) + if (variants.length === 0) + throw new Error('MMD ZIP must contain a .pmx or .pmd model file') + + const blobUrls: Record = {} + await Promise.all(paths.map(async (path) => { + const entry = archive.files[path] + if (!entry) + return + const blob = await entry.async('blob') + blobUrls[path] = URL.createObjectURL(blob) + })) + + const variant = variants[0] + const modelBlobUrl = blobUrls[variant.modelPath] + + return { + variant, + variants, + modelBlobUrl, + blobUrls, + urlModifier: createUrlModifier(blobUrls), + dispose: () => { + for (const url of Object.values(blobUrls)) { + if (url.startsWith('blob:')) { + try { + URL.revokeObjectURL(url) + } + catch {} + } + } + }, + } +} diff --git a/packages/stage-ui-mmd/tsconfig.json b/packages/stage-ui-mmd/tsconfig.json new file mode 100644 index 000000000..2365daacf --- /dev/null +++ b/packages/stage-ui-mmd/tsconfig.json @@ -0,0 +1,36 @@ +{ + "compilerOptions": { + "target": "ESNext", + "jsx": "preserve", + "lib": [ + "DOM", + "DOM.AsyncIterable", + "ESNext" + ], + "module": "ESNext", + "moduleResolution": "Bundler", + "resolveJsonModule": true, + "types": [ + "vite/client" + ], + "allowJs": true, + "strict": true, + "strictNullChecks": true, + "noUnusedLocals": true, + "noEmit": true, + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true, + "isolatedModules": true, + "verbatimModuleSyntax": true, + "skipLibCheck": true + }, + "include": [ + "src/**/*.ts", + "src/**/*.d.ts", + "src/**/*.vue" + ], + "exclude": [ + "dist", + "node_modules" + ] +} diff --git a/packages/stage-ui-mmd/vitest.config.ts b/packages/stage-ui-mmd/vitest.config.ts new file mode 100644 index 000000000..647f39364 --- /dev/null +++ b/packages/stage-ui-mmd/vitest.config.ts @@ -0,0 +1,7 @@ +import { defineConfig } from 'vitest/config' + +export default defineConfig({ + test: { + include: ['src/**/*.test.ts'], + }, +}) diff --git a/packages/stage-ui/package.json b/packages/stage-ui/package.json index acd3af64c..b7531d232 100644 --- a/packages/stage-ui/package.json +++ b/packages/stage-ui/package.json @@ -88,6 +88,7 @@ "@proj-airi/server-sdk-shared": "workspace:^", "@proj-airi/stage-shared": "workspace:^", "@proj-airi/stage-ui-live2d": "workspace:^", + "@proj-airi/stage-ui-mmd": "workspace:^", "@proj-airi/stage-ui-spine": "workspace:^", "@proj-airi/stage-ui-three": "workspace:^", "@proj-airi/ui": "workspace:^", diff --git a/packages/stage-ui/src/components/scenarios/dialogs/model-selector/model-selector.vue b/packages/stage-ui/src/components/scenarios/dialogs/model-selector/model-selector.vue index 30d2ab1a0..f16277611 100644 --- a/packages/stage-ui/src/components/scenarios/dialogs/model-selector/model-selector.vue +++ b/packages/stage-ui/src/components/scenarios/dialogs/model-selector/model-selector.vue @@ -28,7 +28,15 @@ const { displayModelsFromIndexedDBLoading, displayModels } = storeToRefs(display const { t } = useI18n() function handleRemoveModel(model: DisplayModel) { + const wasActive = props.selectedModel?.id === model.id displayModelStore.removeDisplayModel(model.id) + // Removing the model that is currently on stage must also take it off the + // stage; otherwise the scene keeps rendering the already-loaded mesh (its + // blob URL stays valid). Switch to the first remaining model, or none. + if (wasActive) { + const fallback = displayModels.value.find(m => m.id !== model.id) + emits('pick', fallback) + } } const highlightDisplayModelCard = ref(props.selectedModel?.id) @@ -118,6 +126,37 @@ async function handleAddSpineModel(file: FileList | null) { highlightDisplayModelCard.value = displayModel.id } +async function handleAddMMDModel(file: FileList | null) { + if (file === null || file.length === 0) + return + + const picked = file[0] + const lower = picked.name.toLowerCase() + // MMD distributes models as a zip (model + textures) or, less commonly, as a + // bare .pmx/.pmd. The renderer detects zip vs raw by magic bytes at load + // time, so the format here is mainly a label; .pmd keeps its own enum value. + let format: DisplayModelFormat + if (lower.endsWith('.pmd')) + format = DisplayModelFormat.PMD + else if (lower.endsWith('.pmx') || lower.endsWith('.zip')) + format = DisplayModelFormat.PMXZip + else + return + + // NOTICE: + // Keep this await for the same import-then-pick race as the other formats. + // Source/context: model selector import flow -> settings model pick -> settings-stage-model.getDisplayModel(). + // Removal condition: addDisplayModel becomes a synchronous transaction or pick is blocked by explicit import state. + try { + const displayModel = await displayModelStore.addDisplayModel(format, picked) + highlightDisplayModelCard.value = displayModel.id + } + catch (err) { + // Surface the failure instead of leaving the dialog looking inert. + console.error('[model-selector] failed to import MMD model:', err) + } +} + const mapFormatRenderer: Record = { [DisplayModelFormat.Live2dZip]: 'Live2D', [DisplayModelFormat.Live2dDirectory]: 'Live2D', @@ -131,10 +170,12 @@ const mapFormatRenderer: Record = { const live2dDialog = useFileDialog({ accept: '.zip', multiple: false, reset: true }) const vrmDialog = useFileDialog({ accept: '.vrm', multiple: false, reset: true }) const spineDialog = useFileDialog({ accept: '.zip', multiple: false, reset: true }) +const mmdDialog = useFileDialog({ accept: '.zip,.pmx,.pmd', multiple: false, reset: true }) live2dDialog.onChange(handleAddLive2DModel) vrmDialog.onChange(handleAddVRMModel) spineDialog.onChange(handleAddSpineModel) +mmdDialog.onChange(handleAddMMDModel) diff --git a/packages/stage-ui/src/components/scenarios/settings/model-settings/mmd.vue b/packages/stage-ui/src/components/scenarios/settings/model-settings/mmd.vue new file mode 100644 index 000000000..3756dae3a --- /dev/null +++ b/packages/stage-ui/src/components/scenarios/settings/model-settings/mmd.vue @@ -0,0 +1,373 @@ + + + diff --git a/packages/stage-ui/src/components/scenarios/settings/model-settings/panel.vue b/packages/stage-ui/src/components/scenarios/settings/model-settings/panel.vue index c6a41d0d4..65f72b957 100644 --- a/packages/stage-ui/src/components/scenarios/settings/model-settings/panel.vue +++ b/packages/stage-ui/src/components/scenarios/settings/model-settings/panel.vue @@ -15,6 +15,7 @@ import { useI18n } from 'vue-i18n' import Godot from './godot.vue' import Live2D from './live2d.vue' +import MMD from './mmd.vue' import Spine from './spine.vue' import VRM from './vrm.vue' @@ -83,6 +84,9 @@ async function handleModelPick(selectedModel: DisplayModel | undefined) { +

{{ t('settings.model-select.panel-callout.model-type-example') }} @@ -117,6 +121,13 @@ async function handleModelPick(selectedModel: DisplayModel | undefined) { :runtime-snapshot="runtimeSnapshot" @extract-colors-from-model="$emit('extractColorsFromModel')" /> + () const emit = defineEmits<{ @@ -29,8 +31,10 @@ const modelStore = useModelStore() const live2dSceneRef = ref<{ canvasElement: () => HTMLCanvasElement | undefined }>() const vrmSceneRef = ref<{ canvasElement: () => HTMLCanvasElement | undefined }>() const spineSceneRef = ref<{ canvasElement: () => HTMLCanvasElement | undefined }>() +const mmdSceneRef = ref<{ canvasElement: () => HTMLCanvasElement | undefined }>() const live2dComponentState = ref<'pending' | 'loading' | 'mounted'>('pending') const spineComponentState = ref<'pending' | 'loading' | 'mounted'>('pending') +const mmdComponentState = ref<'pending' | 'loading' | 'mounted'>('pending') const vrmPreviewStageInstanceId = `model-settings-preview-stage:${Math.random().toString(36).slice(2, 10)}` const { @@ -53,6 +57,7 @@ const { sceneMutationLocked, scenePhase } = storeToRefs(modelStore) const live2dSceneClassList = computed(() => normalizeClassList(props.live2dSceneClass)) const vrmSceneClassList = computed(() => normalizeClassList(props.vrmSceneClass)) const spineSceneClassList = computed(() => normalizeClassList(props.spineSceneClass)) +const mmdSceneClassList = computed(() => normalizeClassList(props.mmdSceneClass)) function normalizeClassList(value?: string | string[]) { if (!value) @@ -80,6 +85,9 @@ async function capturePreviewFrame() { if (stageModelRenderer.value === 'spine') return captureCanvasFrame(spineSceneRef.value?.canvasElement()) + if (stageModelRenderer.value === 'mmd') + return captureCanvasFrame(mmdSceneRef.value?.canvasElement()) + return undefined } @@ -126,6 +134,20 @@ const runtimeSnapshot = computed(() => { }) } + if (stageModelRenderer.value === 'mmd') { + const phase = resolveComponentStateToRuntimePhase(mmdComponentState.value, { hasModel }) + + return createEmptyModelSettingsRuntimeSnapshot({ + ownerInstanceId: vrmPreviewStageInstanceId, + renderer: 'mmd', + phase, + controlsLocked: hasModel ? phase !== 'mounted' : false, + previewAvailable: hasModel, + canCapturePreview: !!mmdSceneRef.value?.canvasElement(), + updatedAt: Date.now(), + }) + } + if (stageModelRenderer.value === 'godot') { return createEmptyModelSettingsRuntimeSnapshot({ ownerInstanceId: vrmPreviewStageInstanceId, @@ -191,4 +213,16 @@ const cursorPosition = computed(() => ({ /> + diff --git a/packages/stage-ui/src/components/scenarios/settings/model-settings/runtime.ts b/packages/stage-ui/src/components/scenarios/settings/model-settings/runtime.ts index 459790708..441cfb548 100644 --- a/packages/stage-ui/src/components/scenarios/settings/model-settings/runtime.ts +++ b/packages/stage-ui/src/components/scenarios/settings/model-settings/runtime.ts @@ -2,7 +2,7 @@ import type { StageAvatarBoundsPayload, StageViewState } from '@proj-airi/stage- import type { StageModelRenderer } from '../../../../stores/settings/stage-model' -export type ModelSettingsRuntimeRenderer = 'disabled' | 'live2d' | 'vrm' | 'spine' | 'godot' +export type ModelSettingsRuntimeRenderer = 'disabled' | 'live2d' | 'vrm' | 'spine' | 'mmd' | 'godot' export type ModelSettingsRuntimePhase = 'pending' | 'loading' | 'binding' | 'mounted' | 'no-model' | 'error' export interface ModelSettingsRuntimeSnapshot { diff --git a/packages/stage-ui/src/components/scenes/Stage.vue b/packages/stage-ui/src/components/scenes/Stage.vue index f4103cecb..7eec8d4b8 100644 --- a/packages/stage-ui/src/components/scenes/Stage.vue +++ b/packages/stage-ui/src/components/scenes/Stage.vue @@ -13,6 +13,7 @@ import { createLive2DLipSync } from '@proj-airi/model-driver-lipsync' import { wlipsyncProfile } from '@proj-airi/model-driver-lipsync/shared/wlipsync' import { createPlaybackManager, createSpeechPipeline, normalizeActPayload } from '@proj-airi/pipelines-audio' import { Live2DScene, useLive2dParams } from '@proj-airi/stage-ui-live2d' +import { MMDScene } from '@proj-airi/stage-ui-mmd' import { SpineScene } from '@proj-airi/stage-ui-spine' import { ThreeScene } from '@proj-airi/stage-ui-three' import { animations } from '@proj-airi/stage-ui-three/assets/vrm' @@ -65,6 +66,7 @@ const { getDb } = useDuckDb() const vrmViewerRef = ref>() const live2dSceneRef = ref>() const spineSceneRef = ref>() +const mmdSceneRef = ref>() const settingsStore = useSettings() const { @@ -198,6 +200,9 @@ const emotionsQueue = createQueue({ else if (stageModelRenderer.value === 'spine') { spineSceneRef.value?.setEmotion(ctx.data.name, ctx.data.intensity) } + else if (stageModelRenderer.value === 'mmd') { + mmdSceneRef.value?.setEmotion(ctx.data.name, ctx.data.intensity) + } }, ], }) @@ -901,6 +906,9 @@ function canvasElement() { else if (stageModelRenderer.value === 'spine') return spineSceneRef.value?.canvasElement() + + else if (stageModelRenderer.value === 'mmd') + return mmdSceneRef.value?.canvasElement() } function readRenderTargetRegionAtClientPoint(clientX: number, clientY: number, radius: number) { @@ -915,7 +923,9 @@ async function captureFrame() { ? live2dSceneRef.value?.captureFrame() : stageModelRenderer.value === 'vrm' ? vrmViewerRef.value?.captureFrame() - : spineSceneRef.value?.captureFrame()) + : stageModelRenderer.value === 'mmd' + ? mmdSceneRef.value?.captureFrame() + : spineSceneRef.value?.captureFrame()) if (!activeBackgroundUrl.value || !charBlob) return charBlob @@ -1049,6 +1059,20 @@ defineExpose({ :max-fps="spineMaxFps" :render-scale="spineRenderScale" /> +