From 97ecbdcc9d1254e0f8d53d9b667a1818cbd63739 Mon Sep 17 00:00:00 2001 From: Lilia_Chen Date: Sun, 27 Jul 2025 14:32:01 +0100 Subject: [PATCH] feat(stage-ui): LookAt 3 functions: looking at camera; looking at mouse; looking forward (tracking disabled) (#326) --------- Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: Neko --- packages/i18n/src/locales/en/settings.yaml | 7 ++ .../i18n/src/locales/zh-Hans/settings.yaml | 7 ++ .../Scenarios/Settings/ModelSettings/VRM.vue | 23 ++++++- .../stage-ui/src/components/Scenes/VRM.vue | 65 ++++++++++++++++++- .../src/components/Scenes/VRM/Model.vue | 22 ++++++- packages/stage-ui/src/stores/vrm.ts | 6 ++ 6 files changed, 124 insertions(+), 6 deletions(-) diff --git a/packages/i18n/src/locales/en/settings.yaml b/packages/i18n/src/locales/en/settings.yaml index 292b6b6bd..73e43cdba 100644 --- a/packages/i18n/src/locales/en/settings.yaml +++ b/packages/i18n/src/locales/en/settings.yaml @@ -579,6 +579,13 @@ vrm: fov: FOV (degree) rotation-y: Rotation (Y-axis) camera-distance: Camera distance + eye-tracking-mode: + title: Looking at + options: + option: + camera: Camera + mouse: Mouse + disabled: Disabled switch-to-vrm: title: Switch to Live2D Avatar? change-to-vrm: Click here to switch to the Live2D avatar setting diff --git a/packages/i18n/src/locales/zh-Hans/settings.yaml b/packages/i18n/src/locales/zh-Hans/settings.yaml index b1efdf257..990f66748 100644 --- a/packages/i18n/src/locales/zh-Hans/settings.yaml +++ b/packages/i18n/src/locales/zh-Hans/settings.yaml @@ -549,6 +549,13 @@ vrm: fov: 视角调整（度） rotation-y: 模型朝向（Y轴旋转） camera-distance: 相机距离（画面缩放） + eye-tracking-mode: + title: 模型注视方向 + options: + option: + camera: 相机 + mouse: 鼠标 + disabled: 禁用 switch-to-vrm: title: 想切换至Live2D虚拟形象？ change-to-vrm: 切换至Live2D虚拟形象设定页面 diff --git a/packages/stage-ui/src/components/Scenarios/Settings/ModelSettings/VRM.vue b/packages/stage-ui/src/components/Scenarios/Settings/ModelSettings/VRM.vue index 73c76b191..6cfbd8089 100644 --- a/packages/stage-ui/src/components/Scenarios/Settings/ModelSettings/VRM.vue +++ b/packages/stage-ui/src/components/Scenarios/Settings/ModelSettings/VRM.vue @@ -2,7 +2,7 @@ import { Input } from '@proj-airi/ui' import { useFileDialog } from '@vueuse/core' import { storeToRefs } from 'pinia' -import { ref } from 'vue' +import { computed, ref } from 'vue' import { useI18n } from 'vue-i18n' import { useVRM } from '../../../../stores' @@ -38,8 +38,14 @@ const { selectedModel, modelRotationY, cameraDistance, + trackingMode, } = storeToRefs(vrm) const localModelUrl = ref(modelUrl.value) +const trackingOptions = computed(() => [ + { value: 'camera', label: t('settings.vrm.scale-and-position.eye-tracking-mode.options.option.camera'), class: 'col-start-3' }, + { value: 'mouse', label: t('settings.vrm.scale-and-position.eye-tracking-mode.options.option.mouse'), class: 'col-start-4' }, + { value: 'none', label: t('settings.vrm.scale-and-position.eye-tracking-mode.options.option.disabled'), class: 'col-start-5' }, +]) modelFileDialog.onChange((files) => { if (files && files.length > 0) { @@ -113,6 +119,21 @@ function urlUploadClick() { :config="{ min: -180, max: 180, step: 1, label: t('settings.vrm.scale-and-position.rotation-y') }" :label="t('settings.vrm.scale-and-position.rotation-y')" /> + + + {{ t('settings.vrm.scale-and-position.eye-tracking-mode.title') }}: + + import { TresCanvas } from '@tresjs/core' -import { useElementBounding } from '@vueuse/core' +import { useElementBounding, useMouse } from '@vueuse/core' import { storeToRefs } from 'pinia' import { onUnmounted, ref, shallowRef, watch } from 'vue' @@ -13,6 +13,9 @@ const emit = defineEmits<{ (e: 'loadModelProgress', value: number): void (e: 'error', value: unknown): void }>() + +const { x: mouseX, y: mouseY } = useMouse() + const vrmContainerRef = ref() const { width, height } = useElementBounding(vrmContainerRef) const { @@ -21,6 +24,9 @@ const { cameraPosition, cameraDistance, modelOrigin, + trackingMode, + lookAtTarget, + eyeHeight, } = storeToRefs(useVRM()) const modelRef = ref>() @@ -32,6 +38,8 @@ let isUpdatingCamera = true const controlsReady = ref(false) const modelReady = ref(false) const sceneReady = ref(false) +const raycaster = new THREE.Raycaster() +const mouse = new THREE.Vector2() watch(cameraFOV, (newFov) => { if (camera.value) { @@ -136,11 +144,62 @@ watch(cameraDistance, (newDistance) => { } isUpdatingCamera = false }) + +// Set looking target according to trackingMode +function lookAtCamera(newPosition: { x: number, y: number, z: number }) { + modelRef.value?.lookAtUpdate(newPosition) + lookAtTarget.value = newPosition +} +function lookAtMouse(mouseX: number, mouseY: number) { + mouse.x = (mouseX / window.innerWidth) * 2 - 1 + mouse.y = -(mouseY / window.innerHeight) * 2 + 1 + // Raycast from the mouse position + raycaster.setFromCamera(mouse, camera.value) + // Create a plane in front of the camera + const cameraDirection = new THREE.Vector3() + camera.value.getWorldDirection(cameraDirection) // Get camera's forward direction + const plane = new THREE.Plane() + plane.setFromNormalAndCoplanarPoint( + cameraDirection, + camera.value.position.clone().add(cameraDirection.multiplyScalar(1)), // 1 unit in front of the camera + ) + const intersection = new THREE.Vector3() + raycaster.ray.intersectPlane(plane, intersection) + lookAtTarget.value = { x: intersection.x, y: intersection.y, z: intersection.z } + // Pass the target to the model + modelRef.value?.lookAtUpdate(lookAtTarget.value) +} watch(cameraPosition, (newPosition) => { - if (modelRef.value) { - modelRef.value.lookAtUpdate(newPosition) + if (!sceneReady.value || !modelRef.value) + return + if (trackingMode.value === 'camera') { + lookAtCamera(newPosition) } }, { deep: true }) +watch([mouseX, mouseY], () => { + if (!sceneReady.value || !modelRef.value) + return + if (trackingMode.value === 'mouse') { + lookAtMouse(mouseX.value, mouseY.value) + } +}) +watch(trackingMode, (newMode) => { + if (!sceneReady.value || !modelRef.value) + return + if (newMode === 'camera') { + lookAtCamera(cameraPosition.value) + } + else if (newMode === 'mouse') { + lookAtMouse(mouseX.value, mouseY.value) + } + else { + lookAtTarget.value = { + x: 0, + y: eyeHeight.value, + z: -1000, + } + } +}) defineExpose({ setExpression: (expression: string) => { diff --git a/packages/stage-ui/src/components/Scenes/VRM/Model.vue b/packages/stage-ui/src/components/Scenes/VRM/Model.vue index ffda628c7..6b75e94ad 100644 --- a/packages/stage-ui/src/components/Scenes/VRM/Model.vue +++ b/packages/stage-ui/src/components/Scenes/VRM/Model.vue @@ -43,6 +43,9 @@ const { cameraPosition, loadingModel, modelRotationY, + lookAtTarget, + eyeHeight, + trackingMode, } = storeToRefs(vrmStore) const vrmGroup = ref() const idleEyeSaccades = useIdleEyeSaccades() @@ -156,7 +159,6 @@ onMounted(async () => { } // Reanchor the root position track to the model origin reanchorRootPositionTrack(clip) - // rotateRotationTracksInClip(clip, quaternion) // play animation vrmAnimationMixer.value = new AnimationMixer(_vrm.scene) @@ -169,12 +171,28 @@ onMounted(async () => { loadingModel.value = false emit('modelReady') + function getEyePosition(): number | null { + const eye = vrm.value?.humanoid?.getNormalizedBoneNode('head') + if (!eye) + return null + const eyePos = new Vector3() + eye.getWorldPosition(eyePos) + return eyePos.y + } + eyeHeight.value = getEyePosition() + trackingMode.value = 'none' + lookAtTarget.value = { + x: 0, + y: eyeHeight.value, + z: -1000, + } + disposeBeforeRenderLoop = onBeforeRender(({ delta }) => { vrmAnimationMixer.value?.update(delta) vrm.value?.update(delta) vrm.value?.lookAt?.update?.(delta) blink.update(vrm.value, delta) - idleEyeSaccades.update(vrm.value, cameraPosition, delta) + idleEyeSaccades.update(vrm.value, lookAtTarget, delta) vrmEmote.value?.update(delta) }).off } diff --git a/packages/stage-ui/src/stores/vrm.ts b/packages/stage-ui/src/stores/vrm.ts index 116e457fb..778d8cd1b 100644 --- a/packages/stage-ui/src/stores/vrm.ts +++ b/packages/stage-ui/src/stores/vrm.ts @@ -24,6 +24,9 @@ export const useVRM = defineStore('vrm', () => { const cameraDistance = useLocalStorage('settings/vrm/cameraDistance', 0) const isTracking = useLocalStorage('settings/vrm/isTracking', false) + const trackingMode = useLocalStorage('settings/vrm/trackingMode', 'none' as 'camera' | 'mouse' | 'none') + const lookAtTarget = useLocalStorage('settings/vrm/lookAtTarget', { x: 0, y: 0, z: 0 }) + const eyeHeight = useLocalStorage('settings/vrm/eyeHeight', 0) // Manage the object URL lifecycle to prevent memory leaks watch(modelFile, (newFile) => { @@ -63,5 +66,8 @@ export const useVRM = defineStore('vrm', () => { modelRotationY, cameraDistance, isTracking, + trackingMode, + lookAtTarget, + eyeHeight, } })