From 7b7fb0f23806094a60b2237a7e36587342e7f191 Mon Sep 17 00:00:00 2001 From: felix tremblay Date: Wed, 13 May 2026 00:10:03 -0400 Subject: [PATCH] fix(stage-ui-live2d): prevent motions from overriding lip sync mouth values (#1783) --------- Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com> Co-authored-by-agent: Unknown --- packages/i18n/src/locales/es/server/auth.yaml | 2 +- packages/i18n/src/locales/ru/settings.yaml | 4 +- .../i18n/src/locales/zh-Hans/settings.yaml | 2 +- .../src/components/scenes/Live2D.vue | 3 ++ .../src/components/scenes/live2d/Model.vue | 10 ++-- .../src/composables/live2d/motion-manager.ts | 52 ++++++++++++++++--- .../stage-ui/src/components/scenes/Stage.vue | 4 +- 7 files changed, 60 insertions(+), 17 deletions(-) diff --git a/packages/i18n/src/locales/es/server/auth.yaml b/packages/i18n/src/locales/es/server/auth.yaml index f3014c2e8..5c1ddfe5a 100644 --- a/packages/i18n/src/locales/es/server/auth.yaml +++ b/packages/i18n/src/locales/es/server/auth.yaml @@ -66,7 +66,7 @@ signIn: footer: prefix: Al continuar, aceptas nuestros terms: Términos - and: "y" + and: 'y' privacy: Política de Privacidad verifyEmail: title: diff --git a/packages/i18n/src/locales/ru/settings.yaml b/packages/i18n/src/locales/ru/settings.yaml index 5912f849b..02f9d3a43 100644 --- a/packages/i18n/src/locales/ru/settings.yaml +++ b/packages/i18n/src/locales/ru/settings.yaml @@ -685,7 +685,7 @@ pages: empty: Здесь пока ничего нет. Добавьте одно ниже! add: title: Новый - description: "Заполните новый сервер, затем нажмите кнопку «Сохранить и перезапустить» — он будет перемещен в «Конфигурация» выше." + description: 'Заполните новый сервер, затем нажмите кнопку «Сохранить и перезапустить» — он будет перемещен в «Конфигурация» выше.' pending-badge: Не сохранено status: unknown: Не загружен @@ -902,7 +902,7 @@ pages: description: >- Провайдеры транскрипции (speech-to-text): Whisper.cpp, OpenAI, Azure Speech artistry: - title: Artistry + title: Artistry description: Поставщики моделей генерации и создания изображений, например ComfyUI, Replicate. items: comfyui: diff --git a/packages/i18n/src/locales/zh-Hans/settings.yaml b/packages/i18n/src/locales/zh-Hans/settings.yaml index aa5166fd9..423a2a011 100644 --- a/packages/i18n/src/locales/zh-Hans/settings.yaml +++ b/packages/i18n/src/locales/zh-Hans/settings.yaml @@ -685,7 +685,7 @@ pages: empty: 这里什么都还没有哦，在下面添加一个！ add: title: 新建 - description: "填写新的服务器配置，然后点击「保存并重启」，完成后将会更新至上方的「已配置」" + description: '填写新的服务器配置，然后点击「保存并重启」，完成后将会更新至上方的「已配置」' pending-badge: 未保存 status: unknown: 未加载 diff --git a/packages/stage-ui-live2d/src/components/scenes/Live2D.vue b/packages/stage-ui-live2d/src/components/scenes/Live2D.vue index 10d8b0383..bb65bb7de 100644 --- a/packages/stage-ui-live2d/src/components/scenes/Live2D.vue +++ b/packages/stage-ui-live2d/src/components/scenes/Live2D.vue @@ -15,6 +15,7 @@ withDefaults(defineProps<{ paused?: boolean mouthOpenSize?: number + nowSpeaking?: boolean focusAt?: { x: number, y: number } disableFocusAt?: boolean themeColorsHue?: number @@ -30,6 +31,7 @@ withDefaults(defineProps<{ paused: false, focusAt: () => ({ x: 0, y: 0 }), mouthOpenSize: 0, + nowSpeaking: false, themeColorsHue: 220.44, themeColorsHueDynamic: false, live2dIdleAnimationEnabled: true, @@ -84,6 +86,7 @@ defineExpose({ :model-id="modelId" :app="app" :mouth-open-size="mouthOpenSize" + :now-speaking="nowSpeaking" :width="width" :height="height" :paused="paused" diff --git a/packages/stage-ui-live2d/src/components/scenes/live2d/Model.vue b/packages/stage-ui-live2d/src/components/scenes/live2d/Model.vue index a01822959..5af76b09b 100644 --- a/packages/stage-ui-live2d/src/components/scenes/live2d/Model.vue +++ b/packages/stage-ui-live2d/src/components/scenes/live2d/Model.vue @@ -24,6 +24,7 @@ import { useMotionUpdatePluginExpression, useMotionUpdatePluginIdleDisable, useMotionUpdatePluginIdleFocus, + useMotionUpdatePluginLipSync, } from '../../../composables/live2d' import { Emotion, EmotionNeutralMotionName } from '../../../constants/emotions' import { useL2dViewControl, useLive2d } from '../../../stores/live2d' @@ -34,6 +35,7 @@ const props = withDefaults(defineProps<{ app?: Application mouthOpenSize?: number + nowSpeaking?: boolean width: number height: number paused?: boolean @@ -48,6 +50,7 @@ const props = withDefaults(defineProps<{ live2dShadowEnabled?: boolean }>(), { mouthOpenSize: 0, + nowSpeaking: false, paused: false, focusAt: () => ({ x: 0, y: 0 }), disableFocusAt: false, @@ -96,6 +99,7 @@ const model = ref>() const initialModelWidth = ref(0) const initialModelHeight = ref(0) const mouthOpenSize = computed(() => Math.max(0, Math.min(100, props.mouthOpenSize))) +const nowSpeaking = toRef(() => props.nowSpeaking) const lastUpdateTime = ref(0) const { isDark: dark } = useTheme() @@ -108,10 +112,6 @@ const dropShadowFilter = shallowRef(new DropShadowFilter({ rotation: 45, })) -function getCoreModel() { - return model.value!.internalModel.coreModel as any -} - let resizeAnimation: ReturnType | undefined function computeScaleAndPosition() { @@ -369,6 +369,7 @@ async function loadModel() { // This ensures blink respects expression state (0 × blinkFactor = 0). motionManagerUpdate.register(useMotionUpdatePluginExpression(expressionController), 'final') motionManagerUpdate.register(useMotionUpdatePluginAutoEyeBlink(live2dExpressionEnabled), 'final') + motionManagerUpdate.register(useMotionUpdatePluginLipSync(mouthOpenSize, nowSpeaking), 'final') const hookedUpdate = motionManager.update as (model: PixiLive2DInternalModel['coreModel'], now: number) => boolean motionManager.update = function (model: PixiLive2DInternalModel['coreModel'], now: number) { @@ -563,7 +564,6 @@ watch([themeColorsHueDynamic, live2dShadowEnabled], ([dynamic, shadowEnabled]) = } }, { immediate: true }) -watch(mouthOpenSize, value => getCoreModel().setParameterValueById('ParamMouthOpenY', value)) watch(currentMotion, value => setMotion(value.group, value.index)) watch(paused, value => value ? pixiApp.value?.stop() : pixiApp.value?.start()) diff --git a/packages/stage-ui-live2d/src/composables/live2d/motion-manager.ts b/packages/stage-ui-live2d/src/composables/live2d/motion-manager.ts index 6fa2c2232..0e7ea0492 100644 --- a/packages/stage-ui-live2d/src/composables/live2d/motion-manager.ts +++ b/packages/stage-ui-live2d/src/composables/live2d/motion-manager.ts @@ -16,7 +16,9 @@ export type PixiLive2DInternalModel = InternalModel & { export interface MotionManagerUpdateContext { model: CubismModel + // in seconds now: number + // in seconds timeDelta: number hookedUpdate?: (model: CubismModel, now: number) => boolean } @@ -330,9 +332,7 @@ export function useMotionUpdatePluginAutoEyeBlink( // Force ON or eyeBlink null: timer blink + markHandled. if (ctx.live2dForceAutoBlinkEnabled.value || !ctx.internalModel.eyeBlink) { - const rawDelta = Math.max(ctx.timeDelta ?? 0, 0) - const dt = rawDelta < 5 ? rawDelta * 1000 : rawDelta - const safeDt = dt || 16 + const safeDt = ctx.timeDelta * 1000 || 16 const { eyeLOpen, eyeROpen } = updateForcedBlink(safeDt, baseLeft, baseRight) ctx.model.setParameterValueById('ParamEyeLOpen', eyeLOpen) ctx.model.setParameterValueById('ParamEyeROpen', eyeROpen) @@ -399,9 +399,7 @@ export function useMotionUpdatePluginAutoEyeBlink( // Advance blink timer. const wasActive = blinkState.phase !== 'idle' - const rawDelta = Math.max(ctx.timeDelta ?? 0, 0) - const dt = rawDelta < 5 ? rawDelta * 1000 : rawDelta - const safeDt = dt || 16 + const safeDt = ctx.timeDelta * 1000 || 16 const { eyeLOpen: blinkFactorL, eyeROpen: blinkFactorR } = updateForcedBlink(safeDt, 1.0, 1.0) // Blink cycle complete: restore exact pre-blink values. @@ -437,3 +435,45 @@ export function useMotionUpdatePluginExpression( controller.applyExpressions(ctx.model) } } + +/** + * Final-phase plugin that owns ParamMouthOpenY while speech is active and + * smoothly cross-fades back to the motion-driven value when speech ends. + * + * `nowSpeaking` (not `mouthOpenSize > 0`) is the speech boundary, so silent + * gaps between phonemes write 0 directly instead of triggering the release. + */ +export function useMotionUpdatePluginLipSync( + mouthOpenSize: Ref, + nowSpeaking: Ref, +): MotionManagerPlugin { + // 200 ms covers a typical phoneme tail without lagging behind the next utterance. + const RELEASE_DURATION_MS = 200 + + let releaseRemainingMs = 0 + let lastForcedValue = 0 + + // Smoothstep: 3t^2 - 2t^3, eases in/out with zero slope at endpoints. + const smoothstep = (t: number) => t * t * (3 - 2 * t) + + return (ctx) => { + if (nowSpeaking.value) { + lastForcedValue = mouthOpenSize.value + releaseRemainingMs = RELEASE_DURATION_MS + ctx.model.setParameterValueById('ParamMouthOpenY', mouthOpenSize.value) + return + } + + if (releaseRemainingMs <= 0) + return + + releaseRemainingMs = Math.max(0, releaseRemainingMs - ctx.timeDelta * 1000) + const blend = smoothstep(1 - releaseRemainingMs / RELEASE_DURATION_MS) + + // ParamMouthOpenY was already written by motion + expression plugins this frame. + const motionValue = ctx.model.getParameterValueById('ParamMouthOpenY') as number + const blended = lastForcedValue * (1 - blend) + motionValue * blend + + ctx.model.setParameterValueById('ParamMouthOpenY', blended) + } +} diff --git a/packages/stage-ui/src/components/scenes/Stage.vue b/packages/stage-ui/src/components/scenes/Stage.vue index fa9a91cda..0942db1e9 100644 --- a/packages/stage-ui/src/components/scenes/Stage.vue +++ b/packages/stage-ui/src/components/scenes/Stage.vue @@ -69,7 +69,7 @@ const { live2dMaxFps, live2dRenderScale, } = storeToRefs(settingsStore) -const { mouthOpenSize } = storeToRefs(useSpeakingStore()) +const { mouthOpenSize, nowSpeaking } = storeToRefs(useSpeakingStore()) const { audioContext } = useAudioContext() const currentAudioSource = ref() @@ -106,7 +106,6 @@ viewUpdateCleanups.push(live2dStore.onShouldUpdateView(async () => { })) const audioAnalyser = ref() -const nowSpeaking = ref(false) const lipSyncStarted = ref(false) const lipSyncLoopId = ref() const live2dLipSync = ref() @@ -648,6 +647,7 @@ defineExpose({ :model-id="stageModelSelected" :focus-at="focusAt" :mouth-open-size="mouthOpenSize" + :now-speaking="nowSpeaking" :paused="paused" :disable-focus-at="live2dDisableFocus" :theme-colors-hue="themeColorsHue"