From d43d4dbc5ea3d7155e67ed301897e054f42c171a Mon Sep 17 00:00:00 2001 From: nagikazu <122757243+youetube@users.noreply.github.com> Date: Tue, 31 Mar 2026 15:22:16 +0900 Subject: [PATCH] feat(stage-ui-live2d): exp3 expression system + auto-blink rework (#1033) --------- Co-authored-by-agent: Claude Opus 4.6 --- .gitignore | 1 + packages/stage-ui-live2d/package.json | 6 +- .../src/components/scenes/Live2D.vue | 3 + .../src/components/scenes/live2d/Model.vue | 112 ++++- .../live2d/expression-controller.ts | 282 ++++++++++++ .../src/composables/live2d/index.ts | 1 + .../src/composables/live2d/motion-manager.ts | 184 +++++--- .../src/stores/expression-store.ts | 422 ++++++++++++++++++ packages/stage-ui-live2d/src/stores/index.ts | 1 + .../src/tools/expression-tools.ts | 135 ++++++ .../settings/model-settings/live2d.vue | 105 ++++- .../stage-ui/src/components/scenes/Stage.vue | 2 + .../stage-ui/src/stores/settings/index.ts | 1 + .../stage-ui/src/stores/settings/live2d.ts | 3 + .../components/form/select-tab/select-tab.vue | 6 +- pnpm-lock.yaml | 6 + 16 files changed, 1211 insertions(+), 59 deletions(-) create mode 100644 packages/stage-ui-live2d/src/composables/live2d/expression-controller.ts create mode 100644 packages/stage-ui-live2d/src/stores/expression-store.ts create mode 100644 packages/stage-ui-live2d/src/tools/expression-tools.ts diff --git a/.gitignore b/.gitignore index ff1ce700a..faf4046f4 100644 --- a/.gitignore +++ b/.gitignore @@ -49,6 +49,7 @@ bundle/ *.flatpak # WXT specified this too *.output +.vfox.toml # Vite dist diff --git a/packages/stage-ui-live2d/package.json b/packages/stage-ui-live2d/package.json index 18c675576..26fb7f2ec 100644 --- a/packages/stage-ui-live2d/package.json +++ b/packages/stage-ui-live2d/package.json @@ -24,6 +24,8 @@ "./constants/emotions": "./src/constants/emotions.ts", "./stores": "./src/stores/index.ts", "./stores/live2d": "./src/stores/live2d.ts", + "./stores/expression-store": "./src/stores/expression-store.ts", + "./tools/expression-tools": "./src/tools/expression-tools.ts", "./utils/eye-motions": "./src/utils/eye-motions.ts", "./utils/live2d-preview": "./src/utils/live2d-preview.ts", "./utils/live2d-zip-loader": "./src/utils/live2d-zip-loader.ts", @@ -53,6 +55,7 @@ "@proj-airi/ui": "workspace:^", "@vueuse/core": "^14.2.1", "animejs": "^4.3.6", + "@xsai/tool": "catalog:", "culori": "^4.0.2", "es-toolkit": "catalog:", "jszip": "^3.10.1", @@ -60,7 +63,8 @@ "pixi-filters": "4", "pixi-live2d-display": "^0.4.0", "three": "^0.183.2", - "vue": "catalog:" + "vue": "catalog:", + "zod": "catalog:" }, "devDependencies": { "@types/culori": "^4.0.1", diff --git a/packages/stage-ui-live2d/src/components/scenes/Live2D.vue b/packages/stage-ui-live2d/src/components/scenes/Live2D.vue index 4d7d9c6a6..c5a75a494 100644 --- a/packages/stage-ui-live2d/src/components/scenes/Live2D.vue +++ b/packages/stage-ui-live2d/src/components/scenes/Live2D.vue @@ -25,6 +25,7 @@ withDefaults(defineProps<{ live2dIdleAnimationEnabled?: boolean live2dAutoBlinkEnabled?: boolean live2dForceAutoBlinkEnabled?: boolean + live2dExpressionEnabled?: boolean live2dShadowEnabled?: boolean live2dMaxFps?: number }>(), { @@ -37,6 +38,7 @@ withDefaults(defineProps<{ live2dIdleAnimationEnabled: true, live2dAutoBlinkEnabled: true, live2dForceAutoBlinkEnabled: false, + live2dExpressionEnabled: true, live2dShadowEnabled: true, live2dMaxFps: 0, }) @@ -94,6 +96,7 @@ defineExpose({ :live2d-idle-animation-enabled="live2dIdleAnimationEnabled" :live2d-auto-blink-enabled="live2dAutoBlinkEnabled" :live2d-force-auto-blink-enabled="live2dForceAutoBlinkEnabled" + :live2d-expression-enabled="live2dExpressionEnabled" :live2d-shadow-enabled="live2dShadowEnabled" /> diff --git a/packages/stage-ui-live2d/src/components/scenes/live2d/Model.vue b/packages/stage-ui-live2d/src/components/scenes/live2d/Model.vue index 49bc5ceec..69c576da1 100644 --- a/packages/stage-ui-live2d/src/components/scenes/live2d/Model.vue +++ b/packages/stage-ui-live2d/src/components/scenes/live2d/Model.vue @@ -16,10 +16,12 @@ import { computed, onMounted, onUnmounted, ref, shallowRef, toRef, watch } from import { createBeatSyncController, + useExpressionController, useLive2DMotionManagerUpdate, useMotionUpdatePluginAutoEyeBlink, useMotionUpdatePluginBeatSync, + useMotionUpdatePluginExpression, useMotionUpdatePluginIdleDisable, useMotionUpdatePluginIdleFocus, } from '../../../composables/live2d' @@ -45,6 +47,7 @@ const props = withDefaults(defineProps<{ live2dIdleAnimationEnabled?: boolean live2dAutoBlinkEnabled?: boolean live2dForceAutoBlinkEnabled?: boolean + live2dExpressionEnabled?: boolean live2dShadowEnabled?: boolean }>(), { mouthOpenSize: 0, @@ -57,6 +60,7 @@ const props = withDefaults(defineProps<{ live2dIdleAnimationEnabled: true, live2dAutoBlinkEnabled: true, live2dForceAutoBlinkEnabled: false, + live2dExpressionEnabled: true, live2dShadowEnabled: true, }) @@ -190,8 +194,19 @@ const themeColorsHueDynamic = toRef(() => props.themeColorsHueDynamic) const live2dIdleAnimationEnabled = toRef(() => props.live2dIdleAnimationEnabled) const live2dAutoBlinkEnabled = toRef(() => props.live2dAutoBlinkEnabled) const live2dForceAutoBlinkEnabled = toRef(() => props.live2dForceAutoBlinkEnabled) +const live2dExpressionEnabled = toRef(() => props.live2dExpressionEnabled) const live2dShadowEnabled = toRef(() => props.live2dShadowEnabled) +// --- Expression controller +const internalModelRef = ref() +const expressionController = useExpressionController({ + internalModel: internalModelRef, + modelId: props.modelId, +}) +// Saved SDK manager references for runtime expression toggle (restore on disable) +const savedEyeBlink = shallowRef(null) +const savedExpressionManager = shallowRef(null) + const localCurrentMotion = ref<{ group: string, index: number }>({ group: 'Idle', index: 0 }) const beatSync = createBeatSyncController({ baseAngles: () => ({ @@ -230,6 +245,10 @@ async function loadModel() { // REVIEW: here as await until(...) guarded the pixiApp and stage to be valid. if (model.value && pixiApp.value?.stage) { + // Dispose expression controller before destroying the old model + expressionController.dispose() + internalModelRef.value = undefined + try { pixiApp.value.stage.removeChild(model.value) model.value.destroy() @@ -353,7 +372,12 @@ async function loadModel() { motionManagerUpdate.register(useMotionUpdatePluginBeatSync(beatSync), 'pre') motionManagerUpdate.register(useMotionUpdatePluginIdleDisable(), 'pre') motionManagerUpdate.register(useMotionUpdatePluginIdleFocus(), 'post') - motionManagerUpdate.register(useMotionUpdatePluginAutoEyeBlink(), 'post') + // Both run in 'final' stage (ignores handled state). + // Expression first: sets desired parameter values (e.g. closed eyes = 0). + // Blink second: reads post-expression eye values, Multiply-modulates on top. + // This ensures blink respects expression state (0 × blinkFactor = 0). + motionManagerUpdate.register(useMotionUpdatePluginExpression(expressionController), 'final') + motionManagerUpdate.register(useMotionUpdatePluginAutoEyeBlink(live2dExpressionEnabled), 'final') const hookedUpdate = motionManager.update as (model: PixiLive2DInternalModel['coreModel'], now: number) => boolean motionManager.update = function (model: PixiLive2DInternalModel['coreModel'], now: number) { @@ -405,6 +429,33 @@ async function loadModel() { coreModel.setParameterValueById('ParamBodyAngleZ', modelParameters.value.bodyAngleZ) coreModel.setParameterValueById('ParamBreath', modelParameters.value.breath) + // Save SDK manager references so they can be restored if expression is + // toggled off at runtime. + savedEyeBlink.value = internalModel.eyeBlink + savedExpressionManager.value = motionManager.expressionManager + + // --- Expression controller initialisation (conditional) + if (live2dExpressionEnabled.value) { + // Disable built-in Cubism expression manager — our expression-controller + // replaces it. The SDK's manager runs after motionManager.update() and + // would overwrite our final-plugin values every frame. + if (motionManager.expressionManager) { + ;(motionManager as any).expressionManager = null + } + // Disable SDK eyeBlink — it runs on frames where motionUpdated=false and + // would conflict with expression eye parameter overrides. Our auto-blink + // plugin (Force Auto Blink setting) provides the replacement for models + // without idle-motion blink curves. + if (internalModel.eyeBlink) { + ;(internalModel as any).eyeBlink = null + } + + internalModelRef.value = internalModel + initExpressionController(internalModel).catch((err) => { + console.warn('[Model.vue] Expression controller initialisation failed:', err) + }) + } + emits('modelLoaded') } catch (error) { @@ -418,6 +469,40 @@ async function loadModel() { } } +/** + * Initialise the expression controller by reading expression definitions from + * the model settings (model3.json) and parsing each referenced exp3.json file. + * + * This is intentionally fire-and-forget from loadModel so that a failure in + * expression loading does not prevent the model itself from rendering. + */ +async function initExpressionController(internalModel: PixiLive2DInternalModel) { + // Dispose any previous state (handles model reloads) + expressionController.dispose() + + const settings = (internalModel as any).settings + if (!settings) + return + + // model3.json stores expressions as { Name, File }[] under settings.expressions + const expressionRefs: { Name: string, File: string }[] = settings.expressions ?? [] + if (expressionRefs.length === 0) + return + + // Build a function that can read exp3 files relative to the model root. + // For URL-loaded models, resolveURL gives us the full URL. For ZIP-loaded + // models the resolved URL points to an in-memory blob/object URL. + const readExpFile = async (filePath: string): Promise => { + const resolvedUrl: string = settings.resolveURL?.(filePath) ?? filePath + const response = await fetch(resolvedUrl) + if (!response.ok) + throw new Error(`Failed to fetch exp3 file: ${filePath} (${response.status})`) + return response.text() + } + + await expressionController.initialise(expressionRefs, readExpFile) +} + async function setMotion(motionName: string, index?: number) { // TODO: motion? Not every Live2D model has motion, we do need to help users to set motion if (!model.value) { @@ -644,6 +729,30 @@ watch(live2dIdleAnimationEnabled, (enabled) => { } }) +// Watch for expression system toggle — nullify/restore SDK managers at runtime +watch(live2dExpressionEnabled, (enabled) => { + if (!model.value) + return + const im = model.value.internalModel + const mm = im.motionManager + if (enabled) { + if (mm.expressionManager) + ;(mm as any).expressionManager = null + if (im.eyeBlink) + ;(im as any).eyeBlink = null + internalModelRef.value = im + initExpressionController(im).catch((err) => { + console.warn('[Model.vue] Expression controller initialisation failed:', err) + }) + } + else { + ;(mm as any).expressionManager = savedExpressionManager.value + ;(im as any).eyeBlink = savedEyeBlink.value + expressionController.dispose() + internalModelRef.value = undefined + } +}) + watch(focusAt, (value) => { if (!model.value) return @@ -666,6 +775,7 @@ onUnmounted(() => { isUnmounted = true resizeAnimation?.pause() disposeShouldUpdateView?.() + expressionController.dispose() }) function listMotionGroups() { diff --git a/packages/stage-ui-live2d/src/composables/live2d/expression-controller.ts b/packages/stage-ui-live2d/src/composables/live2d/expression-controller.ts new file mode 100644 index 000000000..3c9ed7339 --- /dev/null +++ b/packages/stage-ui-live2d/src/composables/live2d/expression-controller.ts @@ -0,0 +1,282 @@ +import type { Ref } from 'vue' + +import type { ExpressionBlendMode, ExpressionEntry, ExpressionGroupDefinition } from '../../stores/expression-store' +import type { PixiLive2DInternalModel } from './motion-manager' + +import { useExpressionStore } from '../../stores/expression-store' + +// --------------------------------------------------------------------------- +// Types for model3.json / exp3.json data +// --------------------------------------------------------------------------- + +/** A single expression reference inside model3.json FileReferences.Expressions[]. */ +interface Model3ExpressionRef { + Name: string + File: string +} + +/** Parameter entry inside an exp3.json file. */ +interface Exp3Parameter { + Id: string + Value: number + Blend: 'Add' | 'Multiply' | 'Overwrite' +} + +/** Root structure of an exp3.json file. */ +interface Exp3Json { + Type: string + Parameters: Exp3Parameter[] + // FadeInTime / FadeOutTime are intentionally ignored (we do direct application). +} + +// --------------------------------------------------------------------------- +// Controller +// --------------------------------------------------------------------------- + +export interface ExpressionControllerOptions { + /** + * The loaded Live2D internal model reference (reactive so it can be null + * while a model is being swapped). + */ + internalModel: Ref + + /** + * An optional model identifier used for persistence scoping. + * Falls back to `'unknown'` if not provided. + */ + modelId?: string +} + +/** + * Create an expression controller that: + * 1. Parses exp3 data from the model settings + * 2. Registers entries into the Pinia expression store + * 3. Provides an `applyExpressions()` function to be called every frame + */ +export function useExpressionController(options: ExpressionControllerOptions) { + const store = useExpressionStore() + + // Track which parameter IDs were written in the previous frame so we can + // detect active→inactive transitions and explicitly reset them. + const activeLastFrame = new Set() + + // ---- Initialisation (called once after model load) ---------------------- + + /** + * Parse model3.json expression references and the corresponding exp3 data, + * then register everything in the store. + * + * @param expressionRefs - `FileReferences.Expressions` from model3.json + * @param readExpFile - An async function that reads the content of an + * exp3.json file given its path (relative to the + * model root inside the ZIP / OPFS). + */ + async function initialise( + expressionRefs: Model3ExpressionRef[], + readExpFile: (path: string) => Promise, + ) { + const groups: ExpressionGroupDefinition[] = [] + const entryMap = new Map() + + for (const expRef of expressionRefs) { + try { + const raw = await readExpFile(expRef.File) + const exp3: Exp3Json = JSON.parse(raw) + + const groupParams: ExpressionGroupDefinition['parameters'] = [] + + for (const param of exp3.Parameters) { + const blend = normaliseBlend(param.Blend) + + groupParams.push({ + parameterId: param.Id, + blend, + value: param.Value, + }) + + // Only create the entry once per parameterId (first-come basis for + // modelDefault; the store handles last-write-wins at runtime). + if (!entryMap.has(param.Id)) { + const modelDefault = getModelParameterDefault(param.Id) + entryMap.set(param.Id, { + name: param.Id, + parameterId: param.Id, + blend, + currentValue: modelDefault, + defaultValue: modelDefault, + modelDefault, + targetValue: param.Value, + }) + } + else if (param.Value !== 0) { + // Update targetValue if this group has a non-zero value + // (prefer non-zero over zero as the "intended activation value") + const existing = entryMap.get(param.Id)! + if (existing.targetValue === 0) { + existing.targetValue = param.Value + } + } + } + + groups.push({ name: expRef.Name, parameters: groupParams }) + } + catch (err) { + console.warn(`[expression-controller] Failed to parse exp3 for "${expRef.Name}" (${expRef.File}):`, err) + } + } + + store.registerExpressions( + options.modelId ?? 'unknown', + groups, + Array.from(entryMap.values()), + ) + } + + // ---- Per-frame application ----------------------------------------------- + + /** + * Apply all expression entries from the store onto the Live2D model. + * + * Two key mechanisms: + * + * 1. **Noop detection**: entries whose currentValue is the blend-mode + * identity (Add:0, Multiply:1, Overwrite:modelDefault) are skipped. + * + * 2. **Transition reset**: when an entry was active last frame but is + * noop this frame, we explicitly write modelDefault to clear the + * stale value (expression-only params are not reset by motion). + * + * Multiply blend reads the current frame parameter value (post-blink) + * so auto-blink modulation is naturally preserved. + * + * @param coreModel - The Cubism core model (coreModel from internalModel). + */ + function applyExpressions(coreModel: PixiLive2DInternalModel['coreModel']) { + const activeThisFrame = new Set() + + for (const entry of store.expressions.values()) { + if (isNoopValue(entry)) + continue + + const blendedValue = computeTargetValue(entry, coreModel) + + coreModel.setParameterValueById(entry.parameterId, blendedValue) + activeThisFrame.add(entry.parameterId) + } + + // Reset parameters that were active last frame but not this frame. + // This handles the active→inactive transition (e.g. toggle OFF). + for (const paramId of activeLastFrame) { + if (!activeThisFrame.has(paramId)) { + const entry = findEntryByParameterId(paramId) + if (entry) + coreModel.setParameterValueById(paramId, entry.modelDefault) + } + } + + activeLastFrame.clear() + for (const id of activeThisFrame) + activeLastFrame.add(id) + } + + /** + * Does this entry's currentValue produce no visual effect when blended? + * - Add: adding 0 changes nothing. + * - Multiply: multiplying by 1 changes nothing. + * - Overwrite: writing modelDefault changes nothing. + */ + function isNoopValue(entry: ExpressionEntry): boolean { + switch (entry.blend) { + case 'Add': + return entry.currentValue === 0 + case 'Multiply': + return entry.currentValue === 1 + default: + return entry.currentValue === entry.modelDefault + } + } + + /** + * Compute the final blended value for a parameter. + * + * - **Add**: modelDefault + currentValue (stable base prevents accumulation + * on expression-only parameters not reset by motion each frame). + * - **Multiply**: reads the CURRENT frame parameter value (post-blink/motion) + * and scales it. This preserves auto-blink modulation because blink/motion + * always write a fresh value before the expression plugin runs. + * - **Overwrite**: direct replacement. + */ + function computeTargetValue(entry: ExpressionEntry, coreModel: PixiLive2DInternalModel['coreModel']): number { + switch (entry.blend) { + case 'Add': + return entry.modelDefault + entry.currentValue + case 'Multiply': { + const currentFrameValue = coreModel.getParameterValueById(entry.parameterId) as number + return currentFrameValue * entry.currentValue + } + default: + return entry.currentValue + } + } + + /** Look up an entry by its Live2D parameter ID. */ + function findEntryByParameterId(paramId: string): ExpressionEntry | undefined { + for (const entry of store.expressions.values()) { + if (entry.parameterId === paramId) + return entry + } + return undefined + } + + // ---- Cleanup ------------------------------------------------------------- + + function dispose() { + store.dispose() + } + + // ---- Private helpers ----------------------------------------------------- + + function normaliseBlend(raw: string): ExpressionBlendMode { + switch (raw) { + case 'Add': + return 'Add' + case 'Multiply': + return 'Multiply' + default: + return 'Overwrite' + } + } + + /** + * Read the model default for a parameter from the currently loaded model. + * Falls back to 0 if the model is not available yet (the store will sync + * later via `restoreDefaults`). + */ + function getModelParameterDefault(parameterId: string): number { + const im = options.internalModel.value + if (!im) + return 0 + + try { + // Prefer the dedicated default-value API when available (Cubism 4+). + const defaultApi = (im.coreModel as any).getParameterDefaultValueById + if (typeof defaultApi === 'function') { + const val = defaultApi.call(im.coreModel, parameterId) + if (val != null) + return val as number + } + // Fall back to the current value which, right after model load, IS + // the default. + return (im.coreModel.getParameterValueById(parameterId) as number) ?? 0 + } + catch { + return 0 + } + } + + return { + initialise, + applyExpressions, + dispose, + } +} diff --git a/packages/stage-ui-live2d/src/composables/live2d/index.ts b/packages/stage-ui-live2d/src/composables/live2d/index.ts index 32408db9a..d929e54e7 100644 --- a/packages/stage-ui-live2d/src/composables/live2d/index.ts +++ b/packages/stage-ui-live2d/src/composables/live2d/index.ts @@ -1,3 +1,4 @@ export * from './animation' export * from './beat-sync' +export * from './expression-controller' export * from './motion-manager' diff --git a/packages/stage-ui-live2d/src/composables/live2d/motion-manager.ts b/packages/stage-ui-live2d/src/composables/live2d/motion-manager.ts index 01bb276f0..6fa2c2232 100644 --- a/packages/stage-ui-live2d/src/composables/live2d/motion-manager.ts +++ b/packages/stage-ui-live2d/src/composables/live2d/motion-manager.ts @@ -2,6 +2,7 @@ import type { Cubism4InternalModel, InternalModel } from 'pixi-live2d-display/cu import type { Ref } from 'vue' import type { BeatSyncController } from './beat-sync' +import type { useExpressionController } from './expression-controller' import { useLive2DIdleEyeFocus } from './animation' @@ -57,10 +58,13 @@ export function useLive2DMotionManagerUpdate(options: UseLive2DMotionManagerUpda const prePlugins: MotionManagerPlugin[] = [] const postPlugins: MotionManagerPlugin[] = [] + const finalPlugins: MotionManagerPlugin[] = [] - function register(plugin: MotionManagerPlugin, stage: 'pre' | 'post' = 'pre') { + function register(plugin: MotionManagerPlugin, stage: 'pre' | 'post' | 'final' = 'pre') { if (stage === 'pre') prePlugins.push(plugin) + else if (stage === 'final') + finalPlugins.push(plugin) else postPlugins.push(plugin) } @@ -108,6 +112,11 @@ export function useLive2DMotionManagerUpdate(options: UseLive2DMotionManagerUpda runPlugins(postPlugins, ctx) + // Final plugins always run regardless of handled state (e.g. expression overrides) + for (const plugin of finalPlugins) { + plugin(ctx) + } + lastUpdateTime.value = now return ctx.handled } @@ -219,7 +228,9 @@ export function useMotionUpdatePluginIdleFocus(idleEyeFocus = useLive2DIdleEyeFo } } -export function useMotionUpdatePluginAutoEyeBlink(): MotionManagerPlugin { +export function useMotionUpdatePluginAutoEyeBlink( + live2dExpressionEnabled?: Ref, +): MotionManagerPlugin { const blinkState = { phase: 'idle' as 'idle' | 'closing' | 'opening', progress: 0, @@ -227,8 +238,14 @@ export function useMotionUpdatePluginAutoEyeBlink(): MotionManagerPlugin { startRight: 1, delayMs: 0, } - const blinkCloseDuration = 200 // ms - const blinkOpenDuration = 200 // ms + + // Eye values captured at blink start. Used as the base during + // closing/opening so that models without eye motion curves don't + // get stuck at 0 (since 0 × factor = 0 forever). + let preBlinkLeft = 1.0 + let preBlinkRight = 1.0 + const blinkCloseDuration = 75 // ms + const blinkOpenDuration = 75 // ms const minDelay = 3000 const maxDelay = 8000 @@ -291,71 +308,132 @@ export function useMotionUpdatePluginAutoEyeBlink(): MotionManagerPlugin { } return (ctx) => { - // Possibility 1: Only update eye focus when the model is idle - // Possibility 2: For models having no motion groups, currentGroup will be undefined while groups can be { idle: ... } - if (!ctx.isIdleMotion || ctx.handled) + // ===== EXPRESSION OFF: MAIN-IDENTICAL BEHAVIOR ===== + // When the expression system is disabled, replicate the exact auto-blink + // logic from main so that hookUpdate returns the same handled state and + // the SDK eyeBlink/motion pipeline is not disrupted. + if (!live2dExpressionEnabled?.value) { + if (!ctx.isIdleMotion || ctx.handled) + return + + const baseLeft = clamp01(ctx.modelParameters.value.leftEyeOpen) + const baseRight = clamp01(ctx.modelParameters.value.rightEyeOpen) + + // Auto-blink OFF: absolute write + markHandled (same as main). + if (!ctx.live2dAutoBlinkEnabled.value) { + resetBlinkState() + ctx.model.setParameterValueById('ParamEyeLOpen', baseLeft) + ctx.model.setParameterValueById('ParamEyeROpen', baseRight) + ctx.markHandled() + return + } + + // Force ON or eyeBlink null: timer blink + markHandled. + if (ctx.live2dForceAutoBlinkEnabled.value || !ctx.internalModel.eyeBlink) { + const rawDelta = Math.max(ctx.timeDelta ?? 0, 0) + const dt = rawDelta < 5 ? rawDelta * 1000 : rawDelta + const safeDt = dt || 16 + const { eyeLOpen, eyeROpen } = updateForcedBlink(safeDt, baseLeft, baseRight) + ctx.model.setParameterValueById('ParamEyeLOpen', eyeLOpen) + ctx.model.setParameterValueById('ParamEyeROpen', eyeROpen) + ctx.markHandled() + return + } + + // SDK eyeBlink path: explicit call → read back → multiply by base → markHandled. + ctx.internalModel.eyeBlink!.updateParameters(ctx.model, ctx.timeDelta / 1000) + const blinkLeft = ctx.model.getParameterValueById('ParamEyeLOpen') as number + const blinkRight = ctx.model.getParameterValueById('ParamEyeROpen') as number + ctx.model.setParameterValueById('ParamEyeLOpen', clamp01(blinkLeft * baseLeft)) + ctx.model.setParameterValueById('ParamEyeROpen', clamp01(blinkRight * baseRight)) + ctx.markHandled() + return + } + + // ===== EXPRESSION ON: MULTIPLY-MODULATE BEHAVIOR ===== + // Run during idle motion only (non-idle motions control eyes via curves). + if (!ctx.isIdleMotion) return const baseLeft = clamp01(ctx.modelParameters.value.leftEyeOpen) const baseRight = clamp01(ctx.modelParameters.value.rightEyeOpen) - // If the user disabled auto blink entirely, keep manual values and bail. Reset state so re-enabling starts fresh. + // Auto-blink OFF: apply manual base values only (multiply with current). if (!ctx.live2dAutoBlinkEnabled.value) { resetBlinkState() - ctx.model.setParameterValueById('ParamEyeLOpen', baseLeft) - ctx.model.setParameterValueById('ParamEyeROpen', baseRight) - ctx.markHandled() + const currentLeft = ctx.model.getParameterValueById('ParamEyeLOpen') as number + const currentRight = ctx.model.getParameterValueById('ParamEyeROpen') as number + ctx.model.setParameterValueById('ParamEyeLOpen', clamp01(currentLeft * baseLeft)) + ctx.model.setParameterValueById('ParamEyeROpen', clamp01(currentRight * baseRight)) return } - // Option 1: Force auto blink via our own timer (for models without eyeBlink or when forced in settings). - if (ctx.live2dForceAutoBlinkEnabled.value || !ctx.internalModel.eyeBlink) { - // timeDelta can be seconds or milliseconds depending on source; normalize to ms. - const rawDelta = Math.max(ctx.timeDelta ?? 0, 0) - const dt = rawDelta < 5 ? rawDelta * 1000 : rawDelta // If less than 5, treat as seconds (e.g., 0.016s -> 16ms). - const safeDt = dt || 16 // Fallback to ~1 frame to avoid getting stuck when timeDelta is 0 on first tick. - - const { eyeLOpen, eyeROpen } = updateForcedBlink(safeDt, baseLeft, baseRight) - - ctx.model.setParameterValueById('ParamEyeLOpen', eyeLOpen) - ctx.model.setParameterValueById('ParamEyeROpen', eyeROpen) - ctx.markHandled() + // Force OFF and SDK eyeBlink alive: should not happen when expression ON + // (eyeBlink is nullified), but guard defensively — just apply multiplier. + if (!ctx.live2dForceAutoBlinkEnabled.value && ctx.internalModel.eyeBlink != null) { + resetBlinkState() + const currentLeft = ctx.model.getParameterValueById('ParamEyeLOpen') as number + const currentRight = ctx.model.getParameterValueById('ParamEyeROpen') as number + ctx.model.setParameterValueById('ParamEyeLOpen', clamp01(currentLeft * baseLeft)) + ctx.model.setParameterValueById('ParamEyeROpen', clamp01(currentRight * baseRight)) return } - // Option 2: Let Cubism drive the blink, but scale it with the user-provided base. - // If the model has eye blink parameters - if (ctx.internalModel.eyeBlink != null) { - // For the part of the auto eye blink implementation in pixi-live2d-display - // - // this.emit("beforeMotionUpdate"); - // const motionUpdated = this.motionManager.update(this.coreModel, now); - // this.emit("afterMotionUpdate"); - // model.saveParameters(); - // this.motionManager.expressionManager?.update(model, now); - // if (!motionUpdated) { - // this.eyeBlink?.updateParameters(model, dt); - // } - // - // https://github.com/guansss/pixi-live2d-display/blob/31317b37d5e22955a44d5b11f37f421e94a11269/src/cubism4/Cubism4InternalModel.ts#L202-L214 - // - // If the this.motionManager.update returns true, as motion updated flag on, - // the eye blink parameters will not be updated, in another hand, the auto eye blink is disabled - // - // Since we are hooking the motionManager.update method currently, - // and previously a always `true` was returned, eye blink parameters were never updated. - // - // Thous we are here to manually update the eye blink parameters within this hooked method - ctx.internalModel.eyeBlink.updateParameters(ctx.model, ctx.timeDelta / 1000) + // --- Force Auto Blink: stateful blink for models without idle blink curves --- + + const currentLeft = ctx.model.getParameterValueById('ParamEyeLOpen') as number + const currentRight = ctx.model.getParameterValueById('ParamEyeROpen') as number + + // Skip blink when eyes are already nearly/fully closed (e.g. by expression). + const BLINK_THRESHOLD = 0.15 + if (blinkState.phase === 'idle' && currentLeft <= BLINK_THRESHOLD && currentRight <= BLINK_THRESHOLD) { + resetBlinkState() + return } - // Apply manual eye parameters after auto eye blink - const blinkLeft = ctx.model.getParameterValueById('ParamEyeLOpen') as number - const blinkRight = ctx.model.getParameterValueById('ParamEyeROpen') as number + // Track post-expression eye values during idle as the blink baseline. + if (blinkState.phase === 'idle') { + preBlinkLeft = currentLeft + preBlinkRight = currentRight + } - ctx.model.setParameterValueById('ParamEyeLOpen', clamp01(blinkLeft * baseLeft)) - ctx.model.setParameterValueById('ParamEyeROpen', clamp01(blinkRight * baseRight)) + // Advance blink timer. + const wasActive = blinkState.phase !== 'idle' + const rawDelta = Math.max(ctx.timeDelta ?? 0, 0) + const dt = rawDelta < 5 ? rawDelta * 1000 : rawDelta + const safeDt = dt || 16 + const { eyeLOpen: blinkFactorL, eyeROpen: blinkFactorR } = updateForcedBlink(safeDt, 1.0, 1.0) - ctx.markHandled() + // Blink cycle complete: restore exact pre-blink values. + if (wasActive && blinkState.phase === 'idle') { + ctx.model.setParameterValueById('ParamEyeLOpen', clamp01(preBlinkLeft * baseLeft)) + ctx.model.setParameterValueById('ParamEyeROpen', clamp01(preBlinkRight * baseRight)) + return + } + + // Idle: don't write (avoids feedback-loop decay). + if (blinkState.phase === 'idle') + return + + // Active blink: saved pre-blink values × blinkFactor. + ctx.model.setParameterValueById('ParamEyeLOpen', clamp01(preBlinkLeft * blinkFactorL * baseLeft)) + ctx.model.setParameterValueById('ParamEyeROpen', clamp01(preBlinkRight * blinkFactorR * baseRight)) + } +} + +/** + * Post-plugin that applies expression parameter overrides from the expression + * store onto the Live2D model every frame. + * + * This plugin intentionally ignores `ctx.handled` so that expression values + * are always applied on top of whatever the motion / blink plugins produced. + * It also does NOT call `ctx.markHandled()` so it never blocks other plugins. + */ +export function useMotionUpdatePluginExpression( + controller: ReturnType, +): MotionManagerPlugin { + return (ctx) => { + // Always apply regardless of handled state – expressions layer on top. + controller.applyExpressions(ctx.model) } } diff --git a/packages/stage-ui-live2d/src/stores/expression-store.ts b/packages/stage-ui-live2d/src/stores/expression-store.ts new file mode 100644 index 000000000..e50f38747 --- /dev/null +++ b/packages/stage-ui-live2d/src/stores/expression-store.ts @@ -0,0 +1,422 @@ +import { defineStore } from 'pinia' +import { ref } from 'vue' + +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- + +export type ExpressionBlendMode = 'Add' | 'Multiply' | 'Overwrite' + +/** + * A single expression parameter entry tracked by the store. + * + * Each entry maps to a Live2D parameter that is controlled through the + * expression system (either via exp3 files or direct parameter access). + */ +export interface ExpressionEntry { + /** Human-readable name (Expression name or raw parameter ID). */ + name: string + /** Live2D parameter ID (e.g. "ParamWatermarkOFF"). */ + parameterId: string + /** How this value is applied on top of the base value. */ + blend: ExpressionBlendMode + /** Runtime value that will be applied every frame. */ + currentValue: number + /** Application-level default (may be overridden by the user via saveDefaults). */ + defaultValue: number + /** Original default baked into the moc3 / exp3 file. */ + modelDefault: number + /** + * The exp3-specified target value for this parameter (e.g. -1, 1, 10). + * Used by toggle to know what value to set when activating. + * For parameters referenced by multiple groups, this stores the first + * non-zero value encountered. + */ + targetValue: number + /** Active auto-reset timer handle, if any. */ + resetTimer?: ReturnType +} + +/** + * Describes a named expression group loaded from model3.json / exp3.json. + * + * One expression group can contain multiple parameter entries (e.g. "Cry" + * may set both "ParamTear" and "ParamEyeWet"). + */ +export interface ExpressionGroupDefinition { + /** Expression name as declared in model3.json Expressions[].Name. */ + name: string + /** Parameter entries that belong to this expression group. */ + parameters: { + parameterId: string + blend: ExpressionBlendMode + value: number + }[] +} + +/** Serialisable snapshot returned to the LLM. */ +export interface ExpressionState { + name: string + value: number + default: number + active: boolean + autoResetAt?: number +} + +/** Unified tool result envelope. */ +export interface ExpressionToolResult { + success: boolean + error?: string + state?: ExpressionState | ExpressionState[] + available?: string[] +} + +// --------------------------------------------------------------------------- +// Persistence helpers (localStorage – no extra dependency needed) +// --------------------------------------------------------------------------- + +function persistenceKey(modelId: string): string { + return `expression-defaults:${modelId}` +} + +function loadPersistedDefaults(modelId: string): Record | null { + try { + const raw = localStorage.getItem(persistenceKey(modelId)) + if (!raw) + return null + return JSON.parse(raw) as Record + } + catch { + return null + } +} + +function savePersistedDefaults(modelId: string, defaults: Record): void { + try { + localStorage.setItem(persistenceKey(modelId), JSON.stringify(defaults)) + } + catch (err) { + console.warn('[expression-store] Failed to persist defaults:', err) + } +} + +// --------------------------------------------------------------------------- +// Store +// --------------------------------------------------------------------------- + +export const useExpressionStore = defineStore('live2d-expressions', () => { + // ---- state --------------------------------------------------------------- + + /** Map keyed by expression/parameter name -> entry. */ + const expressions = ref>(new Map()) + + /** Currently loaded model ID (used for persistence scoping). */ + const modelId = ref('') + + /** + * Named expression groups parsed from model3.json + exp3.json. + * Keyed by expression name. + */ + const expressionGroups = ref>(new Map()) + + /** LLM exposure mode: 'all' exposes everything, 'none' exposes nothing, 'custom' uses per-group map. */ + const llmMode = ref<'all' | 'none' | 'custom'>('none') + + /** Per-group LLM exposure flags (only used when llmMode === 'custom'). */ + const llmExposed = ref>(new Map()) + + // ---- internal helpers ---------------------------------------------------- + + function clearAllTimers() { + for (const entry of expressions.value.values()) { + if (entry.resetTimer != null) { + clearTimeout(entry.resetTimer) + entry.resetTimer = undefined + } + } + } + + function toState(entry: ExpressionEntry): ExpressionState { + return { + name: entry.name, + value: entry.currentValue, + default: entry.defaultValue, + active: entry.currentValue !== entry.defaultValue, + autoResetAt: entry.resetTimer != null ? Date.now() : undefined, + } + } + + function allNames(): string[] { + return Array.from(expressions.value.keys()) + } + + // ---- public API ---------------------------------------------------------- + + /** + * Register all expression entries parsed from the model. + * Called by the expression-controller after parsing exp3 data. + */ + function registerExpressions( + id: string, + groups: ExpressionGroupDefinition[], + parameterEntries: ExpressionEntry[], + ) { + clearAllTimers() + expressions.value = new Map() + expressionGroups.value = new Map() + modelId.value = id + + // Register expression groups + for (const group of groups) { + expressionGroups.value.set(group.name, group) + } + + // Register individual parameter entries + for (const entry of parameterEntries) { + expressions.value.set(entry.name, { ...entry }) + } + + // Restore persisted defaults + const persisted = loadPersistedDefaults(id) + if (persisted) { + for (const [name, defaultVal] of Object.entries(persisted)) { + const entry = expressions.value.get(name) + if (entry) { + entry.defaultValue = defaultVal + entry.currentValue = defaultVal + } + } + } + } + + /** + * Resolve a name to either an expression group or a direct parameter entry. + * Returns `'group'`, `'param'`, or `null`. + */ + function resolve(name: string): { kind: 'group', group: ExpressionGroupDefinition } | { kind: 'param', entry: ExpressionEntry } | null { + const group = expressionGroups.value.get(name) + if (group) + return { kind: 'group', group } + + const entry = expressions.value.get(name) + if (entry) + return { kind: 'param', entry } + + return null + } + + /** + * Set an expression or parameter value. + */ + function set(name: string, value: boolean | number, duration?: number): ExpressionToolResult { + const resolved = resolve(name) + + if (!resolved) { + return { + success: false, + error: `Expression or parameter "${name}" not found.`, + available: allNames(), + } + } + + const numericValue = typeof value === 'boolean' ? (value ? 1 : 0) : value + + if (resolved.kind === 'group') { + const states: ExpressionState[] = [] + for (const param of resolved.group.parameters) { + const entry = expressions.value.get(param.parameterId) + if (entry) { + applyValue(entry, numericValue, duration) + states.push(toState(entry)) + } + } + return { success: true, state: states } + } + + // Direct parameter + applyValue(resolved.entry, numericValue, duration) + return { success: true, state: toState(resolved.entry) } + } + + /** + * Get expression state. + */ + function get(name?: string): ExpressionToolResult { + if (!name) { + // Return all + const states: ExpressionState[] = [] + for (const entry of expressions.value.values()) { + states.push(toState(entry)) + } + return { success: true, state: states } + } + + const resolved = resolve(name) + if (!resolved) { + return { + success: false, + error: `Expression or parameter "${name}" not found.`, + available: allNames(), + } + } + + if (resolved.kind === 'group') { + const states: ExpressionState[] = [] + for (const param of resolved.group.parameters) { + const entry = expressions.value.get(param.parameterId) + if (entry) + states.push(toState(entry)) + } + return { success: true, state: states } + } + + return { success: true, state: toState(resolved.entry) } + } + + /** + * Toggle an expression (flip between default and non-default). + */ + function toggle(name: string, duration?: number): ExpressionToolResult { + const resolved = resolve(name) + if (!resolved) { + return { + success: false, + error: `Expression or parameter "${name}" not found.`, + available: allNames(), + } + } + + if (resolved.kind === 'group') { + // A group is "active" when at least one of its non-zero (activation) + // params is currently set to the exp3 value. Zero-valued params are + // "reset" instructions and are excluded from the active check. + const isActive = resolved.group.parameters.some((p) => { + if (p.value === 0) + return false + const entry = expressions.value.get(p.parameterId) + return entry && entry.currentValue === p.value + }) + const states: ExpressionState[] = [] + for (const param of resolved.group.parameters) { + const entry = expressions.value.get(param.parameterId) + if (entry) { + const newValue = isActive ? entry.modelDefault : param.value + applyValue(entry, newValue, duration) + states.push(toState(entry)) + } + } + return { success: true, state: states } + } + + // Direct parameter toggle: flip between modelDefault and exp3 target value + const entry = resolved.entry + const newValue = entry.currentValue !== entry.modelDefault ? entry.modelDefault : entry.targetValue + applyValue(entry, newValue, duration) + return { success: true, state: toState(entry) } + } + + /** + * Save current values as defaults (persisted across restarts). + */ + function saveDefaults(): ExpressionToolResult { + if (!modelId.value) { + return { success: false, error: 'No model loaded.' } + } + + const defaults: Record = {} + for (const [name, entry] of expressions.value) { + entry.defaultValue = entry.currentValue + defaults[name] = entry.currentValue + } + + savePersistedDefaults(modelId.value, defaults) + return { success: true } + } + + /** + * Reset all expressions to their default values. + */ + function resetAll(): ExpressionToolResult { + clearAllTimers() + const states: ExpressionState[] = [] + for (const entry of expressions.value.values()) { + entry.currentValue = entry.modelDefault + states.push(toState(entry)) + } + return { success: true, state: states } + } + + /** + * Full cleanup when a model is unloaded. + */ + function dispose() { + clearAllTimers() + expressions.value = new Map() + expressionGroups.value = new Map() + llmMode.value = 'none' + llmExposed.value = new Map() + modelId.value = '' + } + + // ---- LLM exposure -------------------------------------------------------- + + function setLlmMode(mode: 'all' | 'none' | 'custom') { + llmMode.value = mode + } + + function setLlmExposed(name: string, value: boolean) { + llmExposed.value.set(name, value) + } + + /** Check if a specific expression group is exposed to LLM tools. */ + function isExposedToLlm(name: string): boolean { + if (llmMode.value === 'all') + return true + if (llmMode.value === 'none') + return false + return llmExposed.value.get(name) ?? false + } + + // ---- private ------------------------------------------------------------- + + function applyValue(entry: ExpressionEntry, value: number, duration?: number) { + // Cancel existing timer + if (entry.resetTimer != null) { + clearTimeout(entry.resetTimer) + entry.resetTimer = undefined + } + + entry.currentValue = value + + // Schedule auto-reset if duration > 0 + if (duration && duration > 0) { + const resetTo = entry.defaultValue + entry.resetTimer = setTimeout(() => { + entry.currentValue = resetTo + entry.resetTimer = undefined + }, duration * 1000) + } + } + + return { + // State (read-only externally, but reactive) + expressions, + modelId, + expressionGroups, + llmMode, + llmExposed, + + // Actions + registerExpressions, + resolve, + set, + get, + toggle, + saveDefaults, + resetAll, + dispose, + setLlmMode, + setLlmExposed, + isExposedToLlm, + } +}) diff --git a/packages/stage-ui-live2d/src/stores/index.ts b/packages/stage-ui-live2d/src/stores/index.ts index 66e76b8f6..8e99680a1 100644 --- a/packages/stage-ui-live2d/src/stores/index.ts +++ b/packages/stage-ui-live2d/src/stores/index.ts @@ -1 +1,2 @@ +export * from './expression-store' export * from './live2d' diff --git a/packages/stage-ui-live2d/src/tools/expression-tools.ts b/packages/stage-ui-live2d/src/tools/expression-tools.ts new file mode 100644 index 000000000..2b75df21b --- /dev/null +++ b/packages/stage-ui-live2d/src/tools/expression-tools.ts @@ -0,0 +1,135 @@ +import type { ExpressionToolResult } from '../stores/expression-store' + +import { tool } from '@xsai/tool' +import { z } from 'zod' + +import { useExpressionStore } from '../stores/expression-store' + +// --------------------------------------------------------------------------- +// Helper +// --------------------------------------------------------------------------- + +function ensureModelLoaded(): ExpressionToolResult | null { + const store = useExpressionStore() + if (!store.modelId || store.expressions.size === 0) { + return { success: false, error: 'No Live2D model is currently loaded.' } + } + return null +} + +function serialize(result: ExpressionToolResult): string { + return JSON.stringify(result) +} + +// --------------------------------------------------------------------------- +// Tool definitions +// --------------------------------------------------------------------------- + +const tools = [ + // ----- expression.set ---------------------------------------------------- + tool({ + name: 'expression_set', + description: [ + 'Set a Live2D expression or parameter value.', + 'Use a boolean (true/false) to toggle an expression, or a number (0.0-1.0) for fine control.', + 'Optionally provide a duration in seconds for auto-reset.', + 'Examples: expression_set("Cry", true), expression_set("Blush", 0.7, 3)', + ].join(' '), + execute: async ({ name, value, duration }) => { + const err = ensureModelLoaded() + if (err) + return serialize(err) + + const store = useExpressionStore() + const numericValue = typeof value === 'boolean' ? (value ? 1 : 0) : value + const result = store.set(name, numericValue, duration ?? undefined) + return serialize(result) + }, + parameters: z.object({ + name: z.string().describe('Expression name or Live2D parameter ID (e.g. "Cry", "ParamWatermarkOFF")'), + value: z.union([z.boolean(), z.number()]).describe('true/false for toggle, or 0.0-1.0 for numeric control'), + duration: z.number().optional().describe('Seconds until auto-reset to default. Omit for permanent change.'), + }), + }), + + // ----- expression.get ---------------------------------------------------- + tool({ + name: 'expression_get', + description: [ + 'Get the current state of a Live2D expression or parameter.', + 'Omit the name to list all available expressions with their current values.', + ].join(' '), + execute: async ({ name }) => { + const err = ensureModelLoaded() + if (err) + return serialize(err) + + const store = useExpressionStore() + const result = store.get(name ?? undefined) + return serialize(result) + }, + parameters: z.object({ + name: z.string().optional().describe('Expression name or parameter ID. Omit to list all.'), + }), + }), + + // ----- expression.toggle ------------------------------------------------- + tool({ + name: 'expression_toggle', + description: [ + 'Toggle a Live2D expression (flip between default and active state).', + 'Optionally provide a duration in seconds for auto-reset.', + ].join(' '), + execute: async ({ name, duration }) => { + const err = ensureModelLoaded() + if (err) + return serialize(err) + + const store = useExpressionStore() + const result = store.toggle(name, duration ?? undefined) + return serialize(result) + }, + parameters: z.object({ + name: z.string().describe('Expression name or parameter ID to toggle'), + duration: z.number().optional().describe('Seconds until auto-reset. Omit for permanent toggle.'), + }), + }), + + // ----- expression.saveDefaults ------------------------------------------- + tool({ + name: 'expression_save_defaults', + description: 'Save the current expression state as the new defaults. Persists across app restarts.', + execute: async () => { + const err = ensureModelLoaded() + if (err) + return serialize(err) + + const store = useExpressionStore() + const result = store.saveDefaults() + return serialize(result) + }, + parameters: z.object({}), + }), + + // ----- expression.resetAll ----------------------------------------------- + tool({ + name: 'expression_reset_all', + description: 'Reset all expressions to their default values.', + execute: async () => { + const err = ensureModelLoaded() + if (err) + return serialize(err) + + const store = useExpressionStore() + const result = store.resetAll() + return serialize(result) + }, + parameters: z.object({}), + }), +] + +/** + * Export all expression tools as a resolved promise array, matching the + * pattern used by other tool modules in the AIRI codebase. + */ +export const expressionTools = async () => Promise.all(tools) diff --git a/packages/stage-ui/src/components/scenarios/settings/model-settings/live2d.vue b/packages/stage-ui/src/components/scenarios/settings/model-settings/live2d.vue index 053009311..34930e838 100644 --- a/packages/stage-ui/src/components/scenarios/settings/model-settings/live2d.vue +++ b/packages/stage-ui/src/components/scenarios/settings/model-settings/live2d.vue @@ -1,7 +1,7 @@