feat(stage-ui): LookAt 3 functions: looking at camera; looking at mouse; looking forward (tracking disabled) (#326)

---------

Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Co-authored-by: Neko <neko@ayaka.moe>

packages/i18n/src/locales/en/settings.yaml

@@ -579,6 +579,13 @@ vrm:
     fov: FOV (degree)
     rotation-y: Rotation (Y-axis)
     camera-distance: Camera distance
+    eye-tracking-mode:
+      title: Looking at
+      options:
+        option:
+          camera: Camera
+          mouse: Mouse
+          disabled: Disabled
   switch-to-vrm:
     title: Switch to Live2D Avatar?
     change-to-vrm: Click here to switch to the Live2D avatar setting

packages/i18n/src/locales/zh-Hans/settings.yaml

@@ -549,6 +549,13 @@ vrm:
     fov: 视角调整（度）
     rotation-y: 模型朝向（Y轴旋转）
     camera-distance: 相机距离（画面缩放）
+    eye-tracking-mode:
+      title: 模型注视方向
+      options:
+        option:
+          camera: 相机
+          mouse: 鼠标
+          disabled: 禁用
   switch-to-vrm:
     title: 想切换至Live2D虚拟形象？
     change-to-vrm: 切换至Live2D虚拟形象设定页面

packages/stage-ui/src/components/Scenarios/Settings/ModelSettings/VRM.vue

@@ -2,7 +2,7 @@
 import { Input } from '@proj-airi/ui'
 import { useFileDialog } from '@vueuse/core'
 import { storeToRefs } from 'pinia'
-import { ref } from 'vue'
+import { computed, ref } from 'vue'
 import { useI18n } from 'vue-i18n'
 
 import { useVRM } from '../../../../stores'
@@ -38,8 +38,14 @@ const {
   selectedModel,
   modelRotationY,
   cameraDistance,
+  trackingMode,
 } = storeToRefs(vrm)
 const localModelUrl = ref(modelUrl.value)
+const trackingOptions = computed(() => [
+  { value: 'camera', label: t('settings.vrm.scale-and-position.eye-tracking-mode.options.option.camera'), class: 'col-start-3' },
+  { value: 'mouse', label: t('settings.vrm.scale-and-position.eye-tracking-mode.options.option.mouse'), class: 'col-start-4' },
+  { value: 'none', label: t('settings.vrm.scale-and-position.eye-tracking-mode.options.option.disabled'), class: 'col-start-5' },
+])
 
 modelFileDialog.onChange((files) => {
   if (files && files.length > 0) {
@@ -113,6 +119,21 @@ function urlUploadClick() {
         :config="{ min: -180, max: 180, step: 1, label: t('settings.vrm.scale-and-position.rotation-y') }"
         :label="t('settings.vrm.scale-and-position.rotation-y')"
       />
+      <!-- Set eye tracking mode -->
+      <span
+        class="col-span-1 col-start-1 row-start-6 self-center text-xs leading-tight font-mono"
+      >
+        {{ t('settings.vrm.scale-and-position.eye-tracking-mode.title') }}:
+      </span>
+      <template v-for="option in trackingOptions" :key="option.value">
+        <Button
+          :class="[option.class, 'row-start-6 w-auto']"
+          size="sm"
+          :variant="trackingMode === option.value ? 'primary' : 'secondary'"
+          :label="option.label"
+          @click="trackingMode = option.value"
+        />
+      </template>
     </div>
   </Container>
   <Container

packages/stage-ui/src/components/Scenes/VRM.vue

@@ -1,6 +1,6 @@
 <script setup lang="ts">
 import { TresCanvas } from '@tresjs/core'
-import { useElementBounding } from '@vueuse/core'
+import { useElementBounding, useMouse } from '@vueuse/core'
 import { storeToRefs } from 'pinia'
 import { onUnmounted, ref, shallowRef, watch } from 'vue'
 
@@ -13,6 +13,9 @@ const emit = defineEmits<{
   (e: 'loadModelProgress', value: number): void
   (e: 'error', value: unknown): void
 }>()
+
+const { x: mouseX, y: mouseY } = useMouse()
+
 const vrmContainerRef = ref<HTMLDivElement>()
 const { width, height } = useElementBounding(vrmContainerRef)
 const {
@@ -21,6 +24,9 @@ const {
   cameraPosition,
   cameraDistance,
   modelOrigin,
+  trackingMode,
+  lookAtTarget,
+  eyeHeight,
 } = storeToRefs(useVRM())
 
 const modelRef = ref<InstanceType<typeof VRMModel>>()
@@ -32,6 +38,8 @@ let isUpdatingCamera = true
 const controlsReady = ref(false)
 const modelReady = ref(false)
 const sceneReady = ref(false)
+const raycaster = new THREE.Raycaster()
+const mouse = new THREE.Vector2()
 
 watch(cameraFOV, (newFov) => {
   if (camera.value) {
@@ -136,11 +144,62 @@ watch(cameraDistance, (newDistance) => {
   }
   isUpdatingCamera = false
 })
+
+// Set looking target according to trackingMode
+function lookAtCamera(newPosition: { x: number, y: number, z: number }) {
+  modelRef.value?.lookAtUpdate(newPosition)
+  lookAtTarget.value = newPosition
+}
+function lookAtMouse(mouseX: number, mouseY: number) {
+  mouse.x = (mouseX / window.innerWidth) * 2 - 1
+  mouse.y = -(mouseY / window.innerHeight) * 2 + 1
+  // Raycast from the mouse position
+  raycaster.setFromCamera(mouse, camera.value)
+  // Create a plane in front of the camera
+  const cameraDirection = new THREE.Vector3()
+  camera.value.getWorldDirection(cameraDirection) // Get camera's forward direction
+  const plane = new THREE.Plane()
+  plane.setFromNormalAndCoplanarPoint(
+    cameraDirection,
+    camera.value.position.clone().add(cameraDirection.multiplyScalar(1)), // 1 unit in front of the camera
+  )
+  const intersection = new THREE.Vector3()
+  raycaster.ray.intersectPlane(plane, intersection)
+  lookAtTarget.value = { x: intersection.x, y: intersection.y, z: intersection.z }
+  // Pass the target to the model
+  modelRef.value?.lookAtUpdate(lookAtTarget.value)
+}
 watch(cameraPosition, (newPosition) => {
-  if (modelRef.value) {
-    modelRef.value.lookAtUpdate(newPosition)
+  if (!sceneReady.value || !modelRef.value)
+    return
+  if (trackingMode.value === 'camera') {
+    lookAtCamera(newPosition)
   }
 }, { deep: true })
+watch([mouseX, mouseY], () => {
+  if (!sceneReady.value || !modelRef.value)
+    return
+  if (trackingMode.value === 'mouse') {
+    lookAtMouse(mouseX.value, mouseY.value)
+  }
+})
+watch(trackingMode, (newMode) => {
+  if (!sceneReady.value || !modelRef.value)
+    return
+  if (newMode === 'camera') {
+    lookAtCamera(cameraPosition.value)
+  }
+  else if (newMode === 'mouse') {
+    lookAtMouse(mouseX.value, mouseY.value)
+  }
+  else {
+    lookAtTarget.value = {
+      x: 0,
+      y: eyeHeight.value,
+      z: -1000,
+    }
+  }
+})
 
 defineExpose({
   setExpression: (expression: string) => {

packages/stage-ui/src/components/Scenes/VRM/Model.vue

@@ -43,6 +43,9 @@ const {
   cameraPosition,
   loadingModel,
   modelRotationY,
+  lookAtTarget,
+  eyeHeight,
+  trackingMode,
 } = storeToRefs(vrmStore)
 const vrmGroup = ref<Group>()
 const idleEyeSaccades = useIdleEyeSaccades()
@@ -156,7 +159,6 @@ onMounted(async () => {
     }
     // Reanchor the root position track to the model origin
     reanchorRootPositionTrack(clip)
-    // rotateRotationTracksInClip(clip, quaternion)
 
     // play animation
     vrmAnimationMixer.value = new AnimationMixer(_vrm.scene)
@@ -169,12 +171,28 @@ onMounted(async () => {
     loadingModel.value = false
     emit('modelReady')
 
+    function getEyePosition(): number | null {
+      const eye = vrm.value?.humanoid?.getNormalizedBoneNode('head')
+      if (!eye)
+        return null
+      const eyePos = new Vector3()
+      eye.getWorldPosition(eyePos)
+      return eyePos.y
+    }
+    eyeHeight.value = getEyePosition()
+    trackingMode.value = 'none'
+    lookAtTarget.value = {
+      x: 0,
+      y: eyeHeight.value,
+      z: -1000,
+    }
+
     disposeBeforeRenderLoop = onBeforeRender(({ delta }) => {
       vrmAnimationMixer.value?.update(delta)
       vrm.value?.update(delta)
       vrm.value?.lookAt?.update?.(delta)
       blink.update(vrm.value, delta)
-      idleEyeSaccades.update(vrm.value, cameraPosition, delta)
+      idleEyeSaccades.update(vrm.value, lookAtTarget, delta)
       vrmEmote.value?.update(delta)
     }).off
   }

packages/stage-ui/src/stores/vrm.ts

@@ -24,6 +24,9 @@ export const useVRM = defineStore('vrm', () => {
   const cameraDistance = useLocalStorage('settings/vrm/cameraDistance', 0)
 
   const isTracking = useLocalStorage('settings/vrm/isTracking', false)
+  const trackingMode = useLocalStorage('settings/vrm/trackingMode', 'none' as 'camera' | 'mouse' | 'none')
+  const lookAtTarget = useLocalStorage('settings/vrm/lookAtTarget', { x: 0, y: 0, z: 0 })
+  const eyeHeight = useLocalStorage('settings/vrm/eyeHeight', 0)
 
   // Manage the object URL lifecycle to prevent memory leaks
   watch(modelFile, (newFile) => {
@@ -63,5 +66,8 @@ export const useVRM = defineStore('vrm', () => {
     modelRotationY,
     cameraDistance,
     isTracking,
+    trackingMode,
+    lookAtTarget,
+    eyeHeight,
   }
 })