From 1d486655edd49d4507fb0468a990f56f83df7736 Mon Sep 17 00:00:00 2001 From: Lilia_Chen Date: Fri, 25 Jul 2025 12:03:50 +0100 Subject: [PATCH] feat(stage-ui): Now the VRM model can always look to the camera! (#322) --- .../stage-ui/src/components/Scenes/VRM.vue | 58 ++++++++++++------- .../src/components/Scenes/VRM/Model.vue | 18 +++--- .../stage-ui/src/composables/vrm/animation.ts | 41 ++++++++----- packages/stage-ui/src/stores/vrm.ts | 3 + 4 files changed, 80 insertions(+), 40 deletions(-) diff --git a/packages/stage-ui/src/components/Scenes/VRM.vue b/packages/stage-ui/src/components/Scenes/VRM.vue index 86eb90b2f..5e2c521e5 100644 --- a/packages/stage-ui/src/components/Scenes/VRM.vue +++ b/packages/stage-ui/src/components/Scenes/VRM.vue @@ -2,7 +2,7 @@ import { TresCanvas } from '@tresjs/core' import { useElementBounding } from '@vueuse/core' import { storeToRefs } from 'pinia' -import { ref, shallowRef, watch } from 'vue' +import { onUnmounted, ref, shallowRef, watch } from 'vue' import * as THREE from 'three' @@ -49,8 +49,38 @@ watch(cameraFOV, (newFov) => { // }) // If controls are ready watch(() => controlsRef.value?.controls, (ctrl) => { - if (ctrl) + if (ctrl && camera.value) { controlsReady.value = true + + const updateCameraFromControls = () => { + if (isUpdatingCamera) + return + isUpdatingCamera = true + + const newPos = camera.value!.position + const newDist = controlsRef.value!.controls!.getDistance() + + const posChanged + = Math.abs(cameraPosition.value.x - newPos.x) > 1e-6 + || Math.abs(cameraPosition.value.y - newPos.y) > 1e-6 + || Math.abs(cameraPosition.value.z - newPos.z) > 1e-6 + + const distChanged = Math.abs(cameraDistance.value - newDist) > 1e-6 + + if (posChanged || distChanged) { + cameraPosition.value = { x: newPos.x, y: newPos.y, z: newPos.z } + cameraDistance.value = newDist + } + + isUpdatingCamera = false + } + + ctrl.addEventListener('change', updateCameraFromControls) + + onUnmounted(() => { + ctrl.removeEventListener('change', updateCameraFromControls) + }) + } }) // If model is ready function handleLoadModelProgress() { @@ -75,6 +105,7 @@ watch( ) camera.value.updateProjectionMatrix() controlsRef.value.controls.update() + cameraDistance.value = controlsRef.value!.controls!.getDistance() } finally { isUpdatingCamera = false @@ -83,24 +114,7 @@ watch( } }, ) - // Bidirectional watch between slider and OrbitControls -watch(() => controlsRef.value?.getDistance(), (newDistance) => { - if (!isUpdatingCamera && newDistance !== undefined && camera.value) { - // To avoid floating point inaccuracies causing a feedback loop with the other watcher, - // we can check if the distance has changed significantly. - isUpdatingCamera = true - if (Math.abs(cameraDistance.value - newDistance) > 1e-6) { - cameraDistance.value = newDistance - cameraPosition.value = { - x: camera.value.position.x, - y: camera.value.position.y, - z: camera.value.position.z, - } - } - isUpdatingCamera = false - } -}) watch(cameraDistance, (newDistance) => { if (!isUpdatingCamera && camera.value && controlsRef.value && controlsRef.value.controls) { isUpdatingCamera = true @@ -120,9 +134,13 @@ watch(cameraDistance, (newDistance) => { z: newPosition.z, } } - isUpdatingCamera = false }) +watch(cameraPosition, (newPosition) => { + if (modelRef.value) { + modelRef.value.lookAtUpdate(newPosition) + } +}, { deep: true }) defineExpose({ setExpression: (expression: string) => { diff --git a/packages/stage-ui/src/components/Scenes/VRM/Model.vue b/packages/stage-ui/src/components/Scenes/VRM/Model.vue index 541763ead..530d588fe 100644 --- a/packages/stage-ui/src/components/Scenes/VRM/Model.vue +++ b/packages/stage-ui/src/components/Scenes/VRM/Model.vue @@ -33,7 +33,6 @@ const vrmAnimationMixer = ref() const { scene } = useTresContext() const { onBeforeRender } = useLoop() const blink = useBlink() -const idleEyeSaccades = useIdleEyeSaccades() const vrmEmote = ref>() const vrmStore = useVRM() @@ -46,6 +45,7 @@ const { modelRotationY, } = storeToRefs(vrmStore) const vrmGroup = ref() +const idleEyeSaccades = useIdleEyeSaccades() onMounted(async () => { if (!scene.value) { @@ -95,10 +95,10 @@ onMounted(async () => { // Set model facing direction const targetDirection = new Vector3(0, 0, -1) // Default facing direction - const lookAtTarget = _vrm.lookAt - if (lookAtTarget) { - const facingDirection = lookAtTarget.faceFront - const quaternion = new Quaternion() + const lookAt = _vrm.lookAt + const quaternion = new Quaternion() + if (lookAt) { + const facingDirection = lookAt.faceFront quaternion.setFromUnitVectors(facingDirection.normalize(), targetDirection.normalize()) _vrmGroup.quaternion.premultiply(quaternion) _vrmGroup.updateMatrixWorld(true) @@ -156,6 +156,7 @@ onMounted(async () => { } // Reanchor the root position track to the model origin reanchorRootPositionTrack(clip) + // rotateRotationTracksInClip(clip, quaternion) // play animation vrmAnimationMixer.value = new AnimationMixer(_vrm.scene) @@ -171,8 +172,9 @@ onMounted(async () => { disposeBeforeRenderLoop = onBeforeRender(({ delta }) => { vrmAnimationMixer.value?.update(delta) vrm.value?.update(delta) + vrm.value?.lookAt?.update?.(delta) blink.update(vrm.value, delta) - idleEyeSaccades.update(vrm.value, delta) + idleEyeSaccades.update(vrm.value, cameraPosition, delta) vrmEmote.value?.update(delta) }).off } @@ -204,7 +206,9 @@ defineExpose({ vrmEmote.value?.setEmotionWithResetAfter(expression, 1000) }, scene: computed(() => vrm.value?.scene), - lookAt: computed(() => vrm.value?.lookAt), + lookAtUpdate(target: { x: number, y: number, z: number }) { + idleEyeSaccades.instantUpdate(vrm.value, target) + }, }) const { pause, resume } = useLoop() diff --git a/packages/stage-ui/src/composables/vrm/animation.ts b/packages/stage-ui/src/composables/vrm/animation.ts index 6901f55ff..aa765f229 100644 --- a/packages/stage-ui/src/composables/vrm/animation.ts +++ b/packages/stage-ui/src/composables/vrm/animation.ts @@ -1,5 +1,6 @@ import type { VRMAnimation } from '@pixiv/three-vrm-animation' import type { VRMCore } from '@pixiv/three-vrm-core' +import type { Ref } from 'vue' import { createVRMAnimationClip } from '@pixiv/three-vrm-animation' import { Object3D, Vector3 } from 'three' @@ -100,42 +101,56 @@ export function useBlink() { */ export function useIdleEyeSaccades() { let nextSaccadeAfter = -1 - let fixationTarget: Vector3 | undefined + const fixationTarget = new Vector3() let timeSinceLastSaccade = 0 // Just a naive vector generator - Simulating random content on a 27in monitor at 65cm distance - function updateFixationTarget() { - if (!fixationTarget) { - fixationTarget = new Vector3(randFloat(-0.25, 0.25), randFloat(-0.2, 0.15), -0.65) - } - else { - fixationTarget.set(randFloat(-0.25, 0.25), randFloat(-0.2, 0.15), -0.65) - } + function updateFixationTarget(lookAtTarget: Ref<{ x: number, y: number, z: number }>) { + fixationTarget.set( + lookAtTarget.value.x + randFloat(-0.25, 0.25), + lookAtTarget.value.y + randFloat(-0.25, 0.25), + lookAtTarget.value.z, + ) } // Function to handle idle eye saccades - function update(vrm: VRMCore | undefined, delta: number) { + function update(vrm: VRMCore | undefined, lookAtTarget: Ref<{ x: number, y: number, z: number }>, delta: number) { if (!vrm?.expressionManager || !vrm.lookAt) return if (timeSinceLastSaccade >= nextSaccadeAfter) { - updateFixationTarget() + updateFixationTarget(lookAtTarget) timeSinceLastSaccade = 0 nextSaccadeAfter = randomSaccadeInterval() / 1000 } else if (!fixationTarget) { - updateFixationTarget() + updateFixationTarget(lookAtTarget) } if (!vrm.lookAt.target) { vrm.lookAt.target = new Object3D() } - vrm.lookAt.target.position.lerp(fixationTarget!, randFloat(0.2, 0.5)) + vrm.lookAt.target.position.lerp(fixationTarget!, 1) vrm.lookAt?.update(delta) timeSinceLastSaccade += delta } - return { update } + function instantUpdate(vrm: VRMCore | undefined, lookAtTarget: { x: number, y: number, z: number }) { + fixationTarget.set( + lookAtTarget.x, + lookAtTarget.y, + lookAtTarget.z, + ) + if (!vrm?.expressionManager || !vrm.lookAt) + return + if (!vrm.lookAt.target) { + vrm.lookAt.target = new Object3D() + } + vrm.lookAt.target.position.lerp(fixationTarget!, 1) + vrm.lookAt?.update(0.016) + } + + return { update, instantUpdate } } diff --git a/packages/stage-ui/src/stores/vrm.ts b/packages/stage-ui/src/stores/vrm.ts index b19c8cc06..116e457fb 100644 --- a/packages/stage-ui/src/stores/vrm.ts +++ b/packages/stage-ui/src/stores/vrm.ts @@ -23,6 +23,8 @@ export const useVRM = defineStore('vrm', () => { const modelRotationY = useLocalStorage('settings/vrm/modelRotationY', 0) const cameraDistance = useLocalStorage('settings/vrm/cameraDistance', 0) + const isTracking = useLocalStorage('settings/vrm/isTracking', false) + // Manage the object URL lifecycle to prevent memory leaks watch(modelFile, (newFile) => { if (modelObjectUrl.value) { @@ -60,5 +62,6 @@ export const useVRM = defineStore('vrm', () => { cameraPosition, modelRotationY, cameraDistance, + isTracking, } })