diff --git a/packages/stage-ui-three/src/components/Model/VRMModel.vue b/packages/stage-ui-three/src/components/Model/VRMModel.vue index b721c43de..57fc6d0ed 100644 --- a/packages/stage-ui-three/src/components/Model/VRMModel.vue +++ b/packages/stage-ui-three/src/components/Model/VRMModel.vue @@ -422,7 +422,7 @@ async function loadModel() { blink.update(vrm.value, delta) idleEyeSaccades.update(vrm.value, lookAtTarget, delta) vrmEmote.value?.update(delta) - vrmLipSync.update(vrm.value) + vrmLipSync.update(vrm.value, delta) vrm.value?.springBoneManager?.update(delta) }).off diff --git a/packages/stage-ui-three/src/composables/vrm/lip-sync.ts b/packages/stage-ui-three/src/composables/vrm/lip-sync.ts index cac786d83..ae8a3f33f 100644 --- a/packages/stage-ui-three/src/composables/vrm/lip-sync.ts +++ b/packages/stage-ui-three/src/composables/vrm/lip-sync.ts @@ -15,30 +15,111 @@ export function useVRMLipSync(audioNode: Ref = { A: 'aa', E: 'ee', I: 'ih', O: 'oh', U: 'ou', } + const RAW_TO_LIP: Record = { + A: 'A', + E: 'E', + I: 'I', + O: 'O', + U: 'U', + S: 'I', + } + + const smoothState: Record = { A: 0, E: 0, I: 0, O: 0, U: 0 } + const ATTACK = 50 // the speed moving to the next mouth shape animation + const RELEASE = 30 // the speed ending the current mouth shape animation + const CAP = 0.7 + const SILENCE_VOL = 0.04 + const SILENCE_GAIN = 0.05 + const IDLE_MS = 160 + let lastActiveAt = 0 watch([isReady, audioNode], ([ready, newAudioNode], [, oldAudioNode]) => { - if (oldAudioNode) - oldAudioNode.disconnect() - - if (ready && newAudioNode) - newAudioNode.connect(lipSyncNode.value!) + if (oldAudioNode && oldAudioNode !== newAudioNode) { + try { + oldAudioNode.disconnect() + } + catch {} + } + if (!ready || !newAudioNode || !lipSyncNode.value) + return + try { + newAudioNode.connect(lipSyncNode.value) + } + catch {} }, { immediate: true }) onUnmounted(() => audioNode.value?.disconnect()) - function update(vrm?: VRMCore) { - if (!vrm?.expressionManager || !lipSyncNode.value) + function update(vrm?: VRMCore, delta = 0.016) { + const node = lipSyncNode.value + if (!vrm?.expressionManager || !node) return - for (const key of Object.keys(lipSyncNode.value.weights)) { - const weight = lipSyncNode.value.weights[key] * lipSyncNode.value.volume - vrm.expressionManager?.setValue(lipSyncMap[key as keyof typeof lipSyncMap], weight) + const vol = node.volume ?? 0 + const amp = Math.min(vol * 0.9, 1) ** 0.7 + + // Remapping wLipSync output AEIOUS to AEIOU + const projected: Record = { A: 0, E: 0, I: 0, O: 0, U: 0 } + for (const raw of RAW_KEYS) { + const lip = RAW_TO_LIP[raw] + const rawVal = node.weights[raw] ?? 0 + projected[lip] = Math.max(projected[lip], rawVal * amp) + } + + // winner + runner + // Original code: all AEIOU mouth shape blended together. Because the A mouth shape has the largest deformation, mixing A-E-I-O-U based on their raw weights causes the combined result to be biased heavily toward A in most cases. + // Improved code: Only the 2 mouth shapes with the largest weights will be blended. + let winner: LipKey = 'I' + let runner: LipKey = 'E' + let winnerVal = -Infinity + let runnerVal = -Infinity + for (const key of LIP_KEYS) { + const val = projected[key] + if (val > winnerVal) { + runnerVal = winnerVal + runner = winner + winnerVal = val + winner = key + } + else if (val > runnerVal) { + runnerVal = val + runner = key + } + } + + // Detect pause or keep silence + const now = performance.now() + let silent = amp < SILENCE_VOL || winnerVal < SILENCE_GAIN + if (!silent) + lastActiveAt = now + if (now - lastActiveAt > IDLE_MS) + silent = true + + // winner + runner weights + const target: Record = { A: 0, E: 0, I: 0, O: 0, U: 0 } + if (!silent) { + target[winner] = Math.min(CAP, winnerVal) + target[runner] = Math.min(CAP * 0.5, runnerVal * 0.6) + } + + // smoothness and expression generation + for (const key of LIP_KEYS) { + const from = smoothState[key] + const to = target[key] + // lerp + const rate = 1 - Math.exp(-(to > from ? ATTACK : RELEASE) * delta) + smoothState[key] = from + (to - from) * rate + const weight = (smoothState[key] <= 0.01 ? 0 : smoothState[key]) * 0.7 + vrm.expressionManager.setValue(BLENDSHAPE_MAP[key], weight) } } diff --git a/packages/stage-ui/src/stores/providers/openai-compatible-builder.ts b/packages/stage-ui/src/stores/providers/openai-compatible-builder.ts index 2959e864f..faaf743fe 100644 --- a/packages/stage-ui/src/stores/providers/openai-compatible-builder.ts +++ b/packages/stage-ui/src/stores/providers/openai-compatible-builder.ts @@ -289,12 +289,12 @@ export function buildOpenAICompatibleProvider( validators: finalValidators, ...(resolvedCategory === 'transcription' ? { - transcriptionFeatures: transcriptionFeatures ?? { - supportsGenerate: true, - supportsStreamOutput: false, - supportsStreamInput: false, - }, - } + transcriptionFeatures: transcriptionFeatures ?? { + supportsGenerate: true, + supportsStreamOutput: false, + supportsStreamInput: false, + }, + } : {}), ...rest, } as ProviderMetadata