nanakura/moeka-project
1
0
Fork 0
Code Issues Pull Requests 1 Actions 1 Packages Projects Releases Wiki Activity

refactor(stage-ui): motion mapping editor (#45)

Browse Source 
* refactor(stage-ui): move motion to pinia store
* wip: patch model file to change motion mapping
* feat(stage-ui): save and export remapped motions
This commit is contained in:
LemonNeko
2025-03-06 11:29:23 +08:00
committed by GitHub
parent d63855ad8f
commit c4a3c21529
6 changed files with 245 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/stage-web/src/components/Widgets/Live2DSettings.vue
+152 -6
View File
@@ -1,8 +1,11 @@
<script setup lang="ts">
import { Collapsable } from '@proj-airi/stage-ui/components'
import { Emotion, EmotionNeutralMotionName } from '@proj-airi/stage-ui/constants'
import { useSettings } from '@proj-airi/stage-ui/stores'
import { useFileDialog } from '@vueuse/core'
import { ref } from 'vue'
import { useFileDialog, useObjectUrl } from '@vueuse/core'
import JSZip from 'jszip'
import localforage from 'localforage'
import { ref, watch } from 'vue'
import { useI18n } from 'vue-i18n'
const { t } = useI18n()
@@ -12,13 +15,76 @@ const modelFile = useFileDialog({
})
const settings = useSettings()
const modelUrl = ref(settings.live2dModel)
const modelUrl = ref(settings.live2dModelUrl)
modelFile.onChange((files) => {
if (files && files.length > 0) {
settings.live2dModel = files[0]
settings.live2dMotionMap = {}
settings.live2dModelFile = files[0]
settings.live2dLoadSource = 'file'
settings.loadingLive2dModel = true
}
})
watch(() => settings.loadingLive2dModel, (value) => {
if (value) {
return
}
settings.availableLive2dMotions.forEach((motion) => {
if (!settings.live2dMotionMap[motion.fileName]) {
settings.live2dMotionMap[motion.fileName] = EmotionNeutralMotionName
}
})
})
async function patchMotionMap(source: File, motionMap: Record<string, string>): Promise<File> {
if (!Object.keys(motionMap).length)
return source
const jsZip = new JSZip()
const zip = await jsZip.loadAsync(source)
const fileName = Object.keys(zip.files).find(key => key.endsWith('model3.json'))
if (!fileName) {
throw new Error('model3.json not found')
}
const model3Json = await zip.file(fileName)!.async('string')
const model3JsonObject = JSON.parse(model3Json)
const motions: Record<string, { File: string }[]> = {}
Object.entries(motionMap).forEach(([key, value]) => {
if (motions[value]) {
motions[value].push({ File: key })
return
}
motions[value] = [{ File: key }]
})
model3JsonObject.FileReferences.Motions = motions
zip.file(fileName, JSON.stringify(model3JsonObject, null, 2))
const zipBlob = await zip.generateAsync({ type: 'blob' })
return new File([zipBlob], source.name, {
type: source.type,
lastModified: source.lastModified,
})
}
async function saveMotionMap() {
const fileFromIndexedDB = await localforage.getItem<File>('live2dModel')
if (!fileFromIndexedDB) {
return
}
const patchedFile = await patchMotionMap(fileFromIndexedDB, settings.live2dMotionMap)
settings.live2dModelFile = patchedFile
settings.live2dLoadSource = 'file'
settings.loadingLive2dModel = true
}
const exportObjectUrl = useObjectUrl(settings.live2dModelFile)
</script>
<template>
@@ -67,12 +133,11 @@ modelFile.onChange((files) => {
>
<button
:disabled="settings.loadingLive2dModel"
bg="zinc-100 dark:zinc-800"
hover="bg-zinc-200 dark:bg-zinc-700"
transition="all ease-in-out duration-250"
ml-2 rounded px-2 py-1 text-sm outline-none
@click="settings.live2dModel = modelUrl"
@click="settings.live2dModelUrl = modelUrl"
>
{{ t('settings.live2d.change-model.from-url-confirm') }}
</button>
@@ -99,4 +164,85 @@ modelFile.onChange((files) => {
</div>
</div>
</Collapsable>
<Collapsable mt-4 w-full>
<template #trigger="slotProps">
<button
bg="zinc-100 dark:zinc-800"
hover="bg-zinc-200 dark:bg-zinc-700"
transition="all ease-in-out duration-250"
w-full flex items-center gap-1.5 rounded-lg px-4 py-3 outline-none
class="[&_.provider-icon]:grayscale-100 [&_.provider-icon]:hover:grayscale-0"
@click="slotProps.setVisible(!slotProps.visible)"
>
<div flex="~ row 1" items-center gap-1.5>
<div
i-solar:face-scan-circle-bold-duotone class="provider-icon size-6"
transition="filter duration-250 ease-in-out"
/>
<div>
Edit motion map
</div>
</div>
<div transform transition="transform duration-250" :class="{ 'rotate-180': slotProps.visible }">
<div i-solar:alt-arrow-down-bold-duotone />
</div>
</button>
</template>
<div p-4>
<div v-if="settings.live2dLoadSource === 'file'" class="space-y-4">
<div v-for="motion in settings.availableLive2dMotions" :key="motion.fileName" class="flex items-center justify-between">
<div class="flex items-center gap-1 text-sm font-medium">
{{ motion.fileName }}
</div>
<div flex gap-2>
<select v-model="settings.live2dMotionMap[motion.fileName]">
<option v-for="emotion in Object.keys(Emotion)" :key="emotion">
{{ emotion }}
</option>
</select>
<button
:disabled="settings.loadingLive2dModel"
rounded
bg="zinc-100 dark:zinc-800"
hover="bg-zinc-200 dark:bg-zinc-700"
transition="all ease-in-out duration-250"
px-2 py-1 text-sm outline-none
@click="settings.live2dCurrentMotion = { group: motion.motionName, index: motion.motionIndex }"
>
Play
</button>
</div>
</div>
<button
:disabled="settings.loadingLive2dModel"
w-full rounded
bg="zinc-100 dark:zinc-800"
hover="bg-zinc-200 dark:bg-zinc-700"
transition="all ease-in-out duration-250"
@click="saveMotionMap"
>
Save and patch
</button>
<a
mt-2 block :href="exportObjectUrl"
:download="`${settings.live2dModelFile?.name}-motion-edited.zip`"
>
<button
:disabled="settings.loadingLive2dModel"
w-full rounded
bg="zinc-100 dark:zinc-800"
hover="bg-zinc-200 dark:bg-zinc-700"
transition="all ease-in-out duration-250"
>
Export
</button>
</a>
</div>
<div v-else>
Not available for URL model
</div>
</div>
</Collapsable>
</template>
packages/stage-ui/src/components/Live2D/Model.vue
+50 -21
View File
@@ -21,11 +21,9 @@ const props = withDefaults(defineProps<{
mouthOpenSize?: number
width: number
height: number
motion?: string
paused: boolean
}>(), {
mouthOpenSize: 0,
motion: '',
})
const pixiApp = toRef(() => props.app)
@@ -60,10 +58,17 @@ function setScale(model: Ref<Live2DModel<InternalModel> | undefined>) {
model.value.scale.set(scale, scale)
}
const { live2dModel, loadingLive2dModel } = storeToRefs(useSettings())
const {
live2dModelFile,
loadingLive2dModel,
live2dCurrentMotion,
availableLive2dMotions,
live2dLoadSource,
live2dModelUrl,
} = storeToRefs(useSettings())
const currentMotion = ref<{ group: string, index: number }>({ group: 'Idle', index: 0 })
// FIXME: it cannot blink if loading other model
async function loadModel(source: string | Blob) {
async function loadModel() {
if (!pixiApp.value)
return
@@ -73,15 +78,13 @@ async function loadModel(source: string | Blob) {
model.value = undefined
}
loadingLive2dModel.value = true
const modelInstance = new Live2DModel()
if (source instanceof Blob) {
await Live2DFactory.setupLive2DModel(modelInstance, [source])
if (live2dLoadSource.value === 'file') {
await Live2DFactory.setupLive2DModel(modelInstance, [live2dModelFile.value])
}
else {
await Live2DFactory.setupLive2DModel(modelInstance, source)
else if (live2dLoadSource.value === 'url') {
await Live2DFactory.setupLive2DModel(modelInstance, live2dModelUrl.value)
}
model.value = modelInstance
@@ -106,8 +109,20 @@ async function loadModel(source: string | Blob) {
const motionManager = internalModel.motionManager
coreModel.setParameterValueById('ParamMouthOpenY', mouthOpenSize.value)
availableLive2dMotions.value = Object.entries(motionManager.definitions).flatMap(([motionName, definition]) => {
if (!definition)
return []
return definition.map((motion: any, index: number) => ({
motionName,
motionIndex: index,
fileName: motion.File,
}))
}).filter(Boolean)
// Remove eye ball movements from idle motion group to prevent conflicts
// This is too hacky
// FIXME: it cannot blink if loading a model only have idle motion
if (motionManager.groups.idle) {
motionManager.motionGroups[motionManager.groups.idle]?.forEach((motion) => {
motion._motionData.curves.forEach((curve: any) => {
@@ -130,8 +145,14 @@ async function loadModel(source: string | Blob) {
return true
}
motionManager.on('motionStart', (group, index) => {
currentMotion.value = { group, index }
})
// save to indexdb
await localforage.setItem('live2dModel', source)
if (live2dModelFile.value) {
await localforage.setItem('live2dModel', live2dModelFile.value)
}
loadingLive2dModel.value = false
}
@@ -146,17 +167,25 @@ async function initLive2DPixiStage() {
extensions.add(InteractionManager)
// load indexdb model first
const live2dModelBlob = await localforage.getItem<Blob>('live2dModel')
if (live2dModelBlob) {
await loadModel(live2dModelBlob)
const live2dModelFromIndexedDB = await localforage.getItem<File>('live2dModel')
if (live2dModelFromIndexedDB) {
live2dModelFile.value = live2dModelFromIndexedDB
live2dLoadSource.value = 'file'
loadingLive2dModel.value = true
return
}
await loadModel(live2dModel.value)
if (live2dModelUrl.value) {
live2dLoadSource.value = 'url'
loadingLive2dModel.value = true
return
}
loadingLive2dModel.value = false
}
async function setMotion(motionName: string) {
await model.value!.motion(motionName, undefined, MotionPriority.FORCE)
async function setMotion(motionName: string, index: number) {
await model.value!.motion(motionName, index, MotionPriority.FORCE)
}
const handleResize = useDebounceFn(() => {
@@ -184,16 +213,16 @@ watch(dark, updateDropShadowFilter, { immediate: true })
watch(model, updateDropShadowFilter)
watch(mouthOpenSize, value => getCoreModel().setParameterValueById('ParamMouthOpenY', value))
watch(pixiApp, initLive2DPixiStage)
watch(() => props.motion, () => props.motion && setMotion(props.motion))
watch(live2dCurrentMotion, value => setMotion(value.group, value.index))
watch(paused, (value) => {
value ? pixiApp.value?.stop() : pixiApp.value?.start()
})
watchDebounced(live2dModel, (value) => {
watchDebounced(loadingLive2dModel, (value) => {
if (!value)
return
loadModel(value)
loadModel()
}, { debounce: 1000 })
onMounted(updateDropShadowFilter)
packages/stage-ui/src/components/Scenes/Live2D.vue
+11 -19
View File
@@ -1,19 +1,11 @@
<script setup lang="ts">
import { storeToRefs } from 'pinia'
import { ref } from 'vue'
import {
EmotionAngryMotionName,
EmotionAwkwardMotionName,
EmotionHappyMotionName,
EmotionQuestionMotionName,
EmotionSadMotionName,
EmotionSurpriseMotionName,
EmotionThinkMotionName,
} from '../../constants/emotions'
import { useSettings } from '../../stores'
import Live2DCanvas from '../Live2D/Canvas.vue'
import Live2DModel from '../Live2D/Model.vue'
import Screen from '../Screen.vue'
import TransitionVertical from '../TransitionVertical.vue'
withDefaults(defineProps<{
paused: boolean
@@ -22,14 +14,14 @@ withDefaults(defineProps<{
mouthOpenSize: 0,
})
const motion = defineModel<string>('motion')
const show = ref(false)
const { live2dCurrentMotion } = storeToRefs(useSettings())
</script>
<template>
<Screen v-slot="{ width, height }" relative>
<Live2DCanvas v-slot="{ app }" :width="width" :height="height">
<Live2DModel :app="app" :mouth-open-size="mouthOpenSize" :width="width" :height="height" :motion="motion" :paused="paused" />
<Live2DModel :app="app" :mouth-open-size="mouthOpenSize" :width="width" :height="height" :paused="paused" />
</Live2DCanvas>
<div absolute bottom="3" right="3">
<div flex="~ row" cursor-pointer>
@@ -61,43 +53,43 @@ const show = ref(false)
<div flex="~ row" flex-wrap gap-2>
<button
rounded-lg bg="zinc-100/70 dark:zinc-800/50" px-2 py-1 backdrop-blur-sm
@click="motion = EmotionSurpriseMotionName"
@click="live2dCurrentMotion = { group: 'Surprise', index: 0 }"
>
{{ $t('stage.viewers.debug-menu.emotions-btn.surprised') }}
</button>
<button
rounded-lg bg="zinc-100/70 dark:zinc-800/50" px-2 py-1 backdrop-blur-sm
@click="motion = EmotionSadMotionName"
@click="live2dCurrentMotion = { group: 'Sad', index: 0 }"
>
{{ $t('stage.viewers.debug-menu.emotions-btn.sad') }}
</button>
<button
rounded-lg bg="zinc-100/70 dark:zinc-800/50" px-2 py-1 backdrop-blur-sm
@click="motion = EmotionAngryMotionName"
@click="live2dCurrentMotion = { group: 'Angry', index: 0 }"
>
{{ $t('stage.viewers.debug-menu.emotions-btn.angry') }}
</button>
<button
rounded-lg bg="zinc-100/70 dark:zinc-800/50" px-2 py-1 backdrop-blur-sm
@click="motion = EmotionHappyMotionName"
@click="live2dCurrentMotion = { group: 'Happy', index: 0 }"
>
{{ $t('stage.viewers.debug-menu.emotions-btn.happy') }}
</button>
<button
rounded-lg bg="zinc-100/70 dark:zinc-800/50" px-2 py-1 backdrop-blur-sm
@click="motion = EmotionAwkwardMotionName"
@click="live2dCurrentMotion = { group: 'Awkward', index: 0 }"
>
{{ $t('stage.viewers.debug-menu.emotions-btn.awkward') }}
</button>
<button
rounded-lg bg="zinc-100/70 dark:zinc-800/50" px-2 py-1 backdrop-blur-sm
@click="motion = EmotionQuestionMotionName"
@click="live2dCurrentMotion = { group: 'Question', index: 0 }"
>
{{ $t('stage.viewers.debug-menu.emotions-btn.question') }}
</button>
<button
rounded-lg bg="zinc-100/70 dark:zinc-800/50" px-2 py-1 backdrop-blur-sm
@click="motion = EmotionThinkMotionName"
@click="live2dCurrentMotion = { group: 'Think', index: 0 }"
>
{{ $t('stage.viewers.debug-menu.emotions-btn.think') }}
</button>
packages/stage-ui/src/components/Widgets/Stage.vue
+7 -7
View File
@@ -34,7 +34,6 @@ const db = ref<DuckDBWasmDrizzleDatabase>()
// const transformersProvider = createTransformers({ embedWorkerURL })
const vrmViewerRef = ref<{ setExpression: (expression: string) => void }>()
const motion = ref<string>('')
const { stageView, elevenLabsApiKey, elevenlabsVoiceEnglish, elevenlabsVoiceJapanese } = storeToRefs(useSettings())
const { mouthOpenSize } = storeToRefs(useSpeakingStore())
@@ -116,6 +115,8 @@ ttsQueue.on('add', (content) => {
const messageContentQueue = useMessageContentQueue(ttsQueue)
const { live2dCurrentMotion } = storeToRefs(useSettings())
const emotionsQueue = useQueue<Emotion>({
handlers: [
async (ctx) => {
@@ -127,7 +128,7 @@ const emotionsQueue = useQueue<Emotion>({
await vrmViewerRef.value!.setExpression(value)
}
else if (stageView.value === '2d') {
motion.value = EMOTION_EmotionMotionName_value[ctx.data]
live2dCurrentMotion.value = EMOTION_EmotionMotionName_value[ctx.data]
}
},
],
@@ -172,7 +173,7 @@ onBeforeMessageComposed(async () => {
})
onBeforeSend(async () => {
motion.value = EmotionThinkMotionName
live2dCurrentMotion.value = EmotionThinkMotionName
})
onTokenLiteral(async (literal) => {
@@ -211,13 +212,13 @@ onMounted(() => {
const extendedWindow = window as Window & typeof globalThis & { electron?: ElectronAPI }
extendedWindow.electron?.ipcRenderer.on('before-hide', () => {
motion.value = EmotionAngryMotionName
live2dCurrentMotion.value = EmotionAngryMotionName
})
extendedWindow.electron?.ipcRenderer.on('after-show', () => {
motion.value = EmotionHappyMotionName
live2dCurrentMotion.value = EmotionHappyMotionName
})
extendedWindow.electron?.ipcRenderer.on('before-quit', () => {
motion.value = EmotionThinkMotionName
live2dCurrentMotion.value = EmotionThinkMotionName
})
})
@@ -232,7 +233,6 @@ onMounted(async () => {
<div h-full w-full>
<Live2DScene
v-if="stageView === '2d'"
:motion="motion"
:mouth-open-size="mouthOpenSize"
min-w="50% <lg:full" min-h="100 sm:100" h-full w-full flex-1
:paused="paused"
packages/stage-ui/src/constants/emotions.ts
+11 -8
View File
@@ -7,6 +7,7 @@ export const EMOTION_AWKWARD = '<|EMOTE_AWKWARD|>'
export const EMOTION_QUESTION = '<|EMOTE_QUESTION|>'
export enum Emotion {
Idle = '<|EMOTE_NEUTRAL|>',
Happy = '<|EMOTE_HAPPY|>',
Sad = '<|EMOTE_SAD|>',
Angry = '<|EMOTE_ANGRY|>',
@@ -18,14 +19,14 @@ export enum Emotion {
export const EMOTION_VALUES = Object.values(Emotion)
// FIXME: need a editor to remap the motion
export const EmotionHappyMotionName = 'Tap'
export const EmotionSadMotionName = 'EmotionSad'
export const EmotionAngryMotionName = 'Tap@Body'
export const EmotionAwkwardMotionName = 'FlickDown'
export const EmotionThinkMotionName = 'Flick'
export const EmotionSurpriseMotionName = 'Flick'
export const EmotionQuestionMotionName = 'Flick@Body'
export const EmotionHappyMotionName = 'Happy'
export const EmotionSadMotionName = 'Sad'
export const EmotionAngryMotionName = 'Angry'
export const EmotionAwkwardMotionName = 'Awkward'
export const EmotionThinkMotionName = 'Think'
export const EmotionSurpriseMotionName = 'Surprise'
export const EmotionQuestionMotionName = 'Question'
export const EmotionNeutralMotionName = 'Idle'
export const EMOTION_EmotionMotionName_value = {
[Emotion.Happy]: EmotionHappyMotionName,
@@ -35,6 +36,7 @@ export const EMOTION_EmotionMotionName_value = {
[Emotion.Surprise]: EmotionSurpriseMotionName,
[Emotion.Awkward]: EmotionAwkwardMotionName,
[Emotion.Question]: EmotionQuestionMotionName,
[Emotion.Idle]: EmotionNeutralMotionName,
}
export const EMOTION_VRMExpressionName_value = {
@@ -45,4 +47,5 @@ export const EMOTION_VRMExpressionName_value = {
[Emotion.Surprise]: 'surprised',
[Emotion.Awkward]: undefined,
[Emotion.Question]: undefined,
[Emotion.Idle]: undefined,
} satisfies Record<Emotion, string | undefined>
packages/stage-ui/src/stores/settings.ts
+14 -4
View File
@@ -23,10 +23,15 @@ export const useSettings = defineStore('settings', () => {
const elevenlabsVoiceEnglish = useLocalStorage<Voice>('settings/llm/elevenlabs/voice/en', Voice.Myriam)
const elevenlabsVoiceJapanese = useLocalStorage<Voice>('settings/llm/elevenlabs/voice/ja', Voice.Morioki)
// TODO: extract to a separate store
const live2dModel = ref<File | string>('./assets/live2d/models/hiyori_pro_zh.zip')
// TODO: extract to a separate store, use a single page to do this
const live2dModelFile = ref<File>()
const live2dModelUrl = ref<string>('./assets/live2d/models/hiyori_pro_zh.zip')
const live2dLoadSource = ref<'file' | 'url'>('url')
const loadingLive2dModel = ref(false) // if set to true, the model will be loaded
const live2dPosition = useLocalStorage('settings/live2d/position', { x: 0, y: 0 }) // position is relative to the center of the screen
const loadingLive2dModel = ref(false)
const live2dCurrentMotion = ref<{ group: string, index: number }>({ group: 'Idle', index: 0 })
const availableLive2dMotions = ref<{ motionName: string, motionIndex: number, fileName: string }[]>([])
const live2dMotionMap = useLocalStorage<Record<string, string>>('settings/live2d/motion-map', {})
watch(isAudioInputOn, (value) => {
if (value === 'false') {
@@ -54,8 +59,13 @@ export const useSettings = defineStore('settings', () => {
openAiApiBaseURL,
openAiModel,
elevenLabsApiKey,
live2dModel,
live2dModelFile,
live2dModelUrl,
live2dLoadSource,
live2dCurrentMotion,
live2dPosition,
availableLive2dMotions,
live2dMotionMap,
loadingLive2dModel,
language,
stageView,