feat(stage-ui): animation start coordinate anchor + vrm position logic fix (#314)

---------

Co-authored-by: Neko <neko@ayaka.moe>

packages/stage-ui/src/components/Scenarios/Settings/ModelSettings/VRM.vue

@@ -2,6 +2,7 @@
 import { FieldRange, Input } from '@proj-airi/ui'
 import { useFileDialog } from '@vueuse/core'
 import { storeToRefs } from 'pinia'
+import { Vector3 } from 'three'
 import { ref } from 'vue'
 import { useI18n } from 'vue-i18n'
 
@@ -71,7 +72,12 @@ function urlUploadClick() {
 function resetCameraDistance() {
   // Calculate the camera distance that can fit the up-2/3 part of the model in the view
   const radians = (cameraFOV.value / 2 * Math.PI) / 180
-  cameraDistance.value = (modelSize.value.y / 3) / Math.tan(radians)
+  const initialCameraOffset = new Vector3(
+    modelSize.value.x / 16,
+    modelSize.value.y / 6, // default y value
+    -(modelSize.value.y / 3) / Math.tan(radians), // default z value
+  )
+  cameraDistance.value = initialCameraOffset.length()
 }
 </script>
 
@@ -208,7 +214,7 @@ function resetCameraDistance() {
       </template>
     </FieldRange>
     <FieldRange
-      v-model="modelOffset.y"
+      v-model="modelOffset.z"
       as="div"
       :min="-modelSize.z * 2"
       :max="modelSize.z * 2"
@@ -250,6 +256,7 @@ function resetCameraDistance() {
       :max="modelSize.z * 20"
       :step="modelSize.z / 100"
       :label="t('settings.vrm.scale-and-position.camera-distance')"
+      :format-value="val => val.toFixed(4)"
     >
       <template #label>
         <div flex items-center>

packages/stage-ui/src/components/Scenes/VRM.vue

@@ -20,6 +20,7 @@ const {
   cameraFOV,
   initialCameraPosition,
   cameraDistance,
+  modelOrigin,
 } = storeToRefs(useVRM())
 
 const modelRef = ref<InstanceType<typeof VRMModel>>()
@@ -54,17 +55,25 @@ watch(cameraFOV, (newFov) => {
 //     camera.value.updateProjectionMatrix()
 //   }
 // })
-
-watch(selectedModel, () => {
-  if (camera.value) {
+function handleLoadModelProgress(val: number) {
+  if (val === 100 && camera.value && controlsRef.value && controlsRef.value.controls) {
+    // Set camera pos
     camera.value.position.set(
       initialCameraPosition.value.x,
       initialCameraPosition.value.y,
       initialCameraPosition.value.z,
     )
     camera.value.updateProjectionMatrix()
+
+    // Set camera target
+    controlsRef.value.controls.target.set(
+      modelOrigin.value.x,
+      modelOrigin.value.y,
+      modelOrigin.value.z,
+    )
+    controlsRef.value.controls.update()
   }
-})
+}
 
 // Bidirectional watch between slider and OrbitControls
 watch(() => controlsRef.value?.getDistance(), (newDistance) => {
@@ -82,9 +91,11 @@ watch(() => controlsRef.value?.getDistance(), (newDistance) => {
   }
 })
 watch(cameraDistance, (newDistance) => {
-  if (camera.value && controlsRef.value) {
+  if (camera.value && controlsRef.value && controlsRef.value.controls) {
     const newPosition = new THREE.Vector3()
-    newPosition.copy(camera.value.position.normalize().multiplyScalar(newDistance))
+    const target = controlsRef.value.controls.target
+    const direction = new THREE.Vector3().subVectors(camera.value.position, target).normalize()
+    newPosition.copy(target).addScaledVector(direction, newDistance)
     camera.value.position.set(
       newPosition.x,
       newPosition.y,
@@ -119,10 +130,9 @@ defineExpose({
         :model="selectedModel"
         idle-animation="/assets/vrm/animations/idle_loop.vrma"
         :paused="false"
-        @load-model-progress="(val) => emit('loadModelProgress', val)"
+        @load-model-progress="(val) => { emit('loadModelProgress', val); handleLoadModelProgress(val); }"
         @error="(val) => emit('error', val)"
       />
-      <!-- <TresPerspectiveCamera :position="[initialCameraPosition.x, initialCameraPosition.y, initialCameraPosition.z]" :fov="cameraFOV"/> -->
     </TresCanvas>
   </div>
 </template>

packages/stage-ui/src/components/Scenes/VRM/Model.vue

@@ -5,7 +5,7 @@ import type { AnimationClip, Group } from 'three'
 import { VRMUtils } from '@pixiv/three-vrm'
 import { useLoop, useTresContext } from '@tresjs/core'
 import { storeToRefs } from 'pinia'
-import { AnimationMixer, MathUtils, Quaternion, Vector3 } from 'three'
+import { AnimationMixer, MathUtils, Quaternion, Vector3, VectorKeyframeTrack } from 'three'
 import { computed, onMounted, onUnmounted, ref, watch } from 'vue'
 
 import { clipFromVRMAnimation, loadVRMAnimation, useBlink, useIdleEyeSaccades } from '../../../composables/vrm/animation'
@@ -40,7 +40,6 @@ const {
   modelOffset,
   modelOrigin,
   modelSize,
-  position,
   initialCameraPosition,
   loadingModel,
   modelRotationY,
@@ -69,12 +68,16 @@ onMounted(async () => {
       _vrmGroup,
       modelCenter: vrmModelCenter,
       modelSize: vrmModelSize,
-      initialCameraPosition: vrmInitialCameraPosition,
+      initialCameraOffset: vrmInitialCameraOffset,
     } = _vrmInfo
 
     vrmGroup.value = _vrmGroup
     // Set initial camera position
-    initialCameraPosition.value = vrmInitialCameraPosition
+    initialCameraPosition.value = {
+      x: vrmModelCenter.x + vrmInitialCameraOffset.x,
+      y: vrmModelCenter.y + vrmInitialCameraOffset.y,
+      z: vrmModelCenter.z + vrmInitialCameraOffset.z,
+    }
 
     // Set initial positions for model
     modelOrigin.value = {
@@ -95,8 +98,8 @@ onMounted(async () => {
       const facingDirection = lookAtTarget.faceFront
       const quaternion = new Quaternion()
       quaternion.setFromUnitVectors(facingDirection.normalize(), targetDirection.normalize())
-      _vrm.scene.quaternion.premultiply(quaternion)
-      _vrm.scene.updateMatrixWorld(true)
+      _vrmGroup.quaternion.premultiply(quaternion)
+      _vrmGroup.updateMatrixWorld(true)
     }
     else {
       console.warn('No look-at target found in VRM model')
@@ -105,9 +108,41 @@ onMounted(async () => {
     modelRotationY.value = 0
 
     // Set initial positions for animation
-    function removeRootPositionTrack(clip: AnimationClip) {
-      clip.tracks = clip.tracks.filter((track) => {
-        return !track.name.endsWith('.position')
+    function reanchorRootPositionTrack(clip: AnimationClip) {
+      // Get the hips node to reanchor the root position track
+      const hipNode = _vrm.humanoid?.getNormalizedBoneNode('hips')
+      if (!hipNode) {
+        console.warn('No hips node found in VRM model.')
+        return
+      }
+      hipNode.updateMatrixWorld(true)
+      const defaultHipPos = new Vector3()
+      hipNode.getWorldPosition(defaultHipPos)
+
+      // Calculate the offset from the hips node to the hips's first frame position
+      const hipsTrack = clip.tracks.find(track =>
+        track.name.endsWith('Hips.position'),
+      )
+      if (!(hipsTrack instanceof VectorKeyframeTrack)) {
+        console.warn('No Hips.position track of type VectorKeyframeTrack found in animation.')
+        return
+      }
+
+      const animeHipPos = new Vector3(
+        hipsTrack.values[0],
+        hipsTrack.values[1],
+        hipsTrack.values[2],
+      )
+      const delta = new Vector3().subVectors(animeHipPos, defaultHipPos)
+
+      clip.tracks.forEach((track) => {
+        if (track.name.endsWith('.position') && track instanceof VectorKeyframeTrack) {
+          for (let i = 0; i < track.values.length; i += 3) {
+            track.values[i] -= delta.x
+            track.values[i + 1] -= delta.y
+            track.values[i + 2] -= delta.z
+          }
+        }
       })
     }
 
@@ -117,8 +152,8 @@ onMounted(async () => {
       console.warn('No VRM animation loaded')
       return
     }
-
-    removeRootPositionTrack(clip)
+    // Reanchor the root position track to the model origin
+    reanchorRootPositionTrack(clip)
 
     // play animation
     vrmAnimationMixer.value = new AnimationMixer(_vrm.scene)
@@ -148,9 +183,9 @@ onMounted(async () => {
 watch(modelOffset, () => {
   if (vrmGroup.value) {
     vrmGroup.value.position.set(
-      position.value.x,
-      position.value.y,
-      position.value.z,
+      modelOffset.value.x,
+      modelOffset.value.y,
+      modelOffset.value.z,
     )
   }
 }, { deep: true })

packages/stage-ui/src/composables/vrm/core.ts

@@ -16,7 +16,7 @@ export async function loadVrm(model: string, options?: {
   scene?: Scene
   lookAt?: boolean
   onProgress?: (progress: ProgressEvent<EventTarget>) => void | Promise<void>
-}): Promise<{ _vrm: VRMCore, _vrmGroup: Group, modelCenter: Vector3, modelSize: Vector3, initialCameraPosition: Vector3 } | undefined> {
+}): Promise<{ _vrm: VRMCore, _vrmGroup: Group, modelCenter: Vector3, modelSize: Vector3, initialCameraOffset: Vector3 } | undefined> {
   const loader = useVRMLoader()
   const gltf = await loader.loadAsync(model, progress => options?.onProgress?.(progress))
 
@@ -50,34 +50,22 @@ export async function loadVrm(model: string, options?: {
     options.scene.add(_vrmGroup)
   }
 
-  // Move the VRM model centre to the (0, 0, 0)
   const box = new Box3().setFromObject(_vrm.scene)
   const modelSize = new Vector3()
   const modelCenter = new Vector3()
   box.getSize(modelSize)
   box.getCenter(modelCenter)
-  modelCenter.negate()
-  modelCenter.y -= modelSize.y / 5 // Adjust pivot to align chest with the origin
+  modelCenter.y += modelSize.y / 5 // Adjust pivot to align chest with the origin
 
   // Set position
-  if (options?.positionOffset) {
-    _vrmGroup.position.set(
-      modelCenter.x + options.positionOffset[0],
-      modelCenter.y + options.positionOffset[1],
-      modelCenter.z + options.positionOffset[2],
-    )
-  }
-  else {
-    _vrmGroup.position.set(modelCenter.x, modelCenter.y, modelCenter.z)
-  }
 
   // Compute the initial camera position (once per loaded model)
   // In order to see the up-2/3 part fo the model, z = (y/3) / tan(fov/2)
   const fov = 40 // default fov = 40 degrees
   const radians = (fov / 2 * Math.PI) / 180
-  const initialCameraPosition = new Vector3(
+  const initialCameraOffset = new Vector3(
     modelSize.x / 16,
-    modelSize.y / 6,
+    modelSize.y / 6, // default y value
     -(modelSize.y / 3) / Math.tan(radians), // default z value
   )
 
@@ -86,6 +74,6 @@ export async function loadVrm(model: string, options?: {
     _vrmGroup,
     modelCenter,
     modelSize,
-    initialCameraPosition,
+    initialCameraOffset,
   }
 }

packages/stage-ui/src/stores/vrm.ts

@@ -16,16 +16,6 @@ export const useVRM = defineStore('vrm', () => {
   const modelOrigin = useLocalStorage('settings/vrm/modelOrigin', { x: 0, y: 0, z: 0 })
   const modelOffset = useLocalStorage('settings/vrm/modelOffset', { x: 0, y: 0, z: 0 })
 
-  const position = computed(() => ({
-    x: modelOrigin.value.x + modelOffset.value.x,
-    y: modelOrigin.value.y + modelOffset.value.y,
-    z: modelOrigin.value.z + modelOffset.value.z,
-  }))
-  const positionInPercentageString = computed(() => ({
-    x: `${position.value.x}%`,
-    y: `${position.value.y}%`,
-    z: `${position.value.z}%`,
-  }))
   const cameraFOV = useLocalStorage('settings/vrm/cameraFOV', 40)
   const initialCameraPosition = useLocalStorage('settings/vrm/initialCameraPosition', { x: 0, y: 0, z: -1 })
 
@@ -65,8 +55,6 @@ export const useVRM = defineStore('vrm', () => {
     scale,
     modelOrigin,
     modelOffset,
-    position,
-    positionInPercentageString,
     selectedModel,
     cameraFOV,
     initialCameraPosition,