feat(stage-*): add clickable VRM body-part reactions (#2068)

2026-07-20 16:13:03 +08:00
parent 0460a6cf14
commit 78d6588cb0
11 changed files with 321 additions and 5 deletions
@@ -668,7 +668,16 @@ describe('setupExtensionHost', () => {
    while (Date.now() < deadline && afterSessionId === beforeSession?.id) {
      await new Promise(resolve => setTimeout(resolve, 100))
      const snapshot = await invokeInspect()
-      afterSessionId = snapshot.sessions.find(session => session.extensionId === 'test-auto-reload-reload')?.id
+      const currentSessionId = snapshot.sessions.find(session => session.extensionId === 'test-auto-reload-reload')?.id
+
+      // ROOT CAUSE:
+      //
+      // Reload stops the old session before starting the replacement, so an inspect call can
+      // observe the intentional short-lived gap with no session. Treat that state as pending
+      // instead of ending the poll before the replacement session is available.
+      if (currentSessionId && currentSessionId !== beforeSession?.id) {
+        afterSessionId = currentSessionId
+      }
    }

    expect(afterSessionId).toBeDefined()
@@ -84,6 +84,16 @@ const isTransparentByThree = useThreeSceneIsTransparentAtPoint(
  relativeMouseY,
  { regionRadius: 25 },
 )
+const isTransparentByPixelsExact = useCanvasPixelIsTransparentAtPoint(
+  stageCanvas,
+  relativeMouseX,
+  relativeMouseY,
+)
+const isTransparentByThreeExact = useThreeSceneIsTransparentAtPoint(
+  widgetStageRef,
+  relativeMouseX,
+  relativeMouseY,
+)

 const settingsStore = useSettings()
 const { stageModelRenderer, stageModelSelectedUrl } = storeToRefs(settingsStore)
@@ -111,6 +121,18 @@ const isTransparent = computed(() => {

  return true
 })
+const isTransparentForMouseEvents = computed(() => {
+  if (stagePaused.value || componentStateStage.value !== 'mounted' || !fadeOnHoverEnabled.value)
+    return true
+
+  if (stageModelRenderer.value === 'vrm')
+    return shouldUseThreeTransparencyHitTest.value ? isTransparentByThreeExact.value : true
+
+  if (stageModelRenderer.value === 'live2d')
+    return isTransparentByPixelsExact.value
+
+  return true
+})

 const { isNearAnyBorder: isAroundWindowBorder } = useElectronMouseAroundWindowBorder({ threshold: 10 })
 const isAroundWindowBorderFor250Ms = refDebounced(isAroundWindowBorder, 250)
@@ -216,10 +238,13 @@ watch([isOutsideFor250Ms, isOutsideStatusIslandFor250Ms, isAroundWindowBorderFor
  }
  else {
    const fadeEnabled = fadeOnHoverEnabled.value
-    // Otherwise allow click-through while we fade UI based on transparency (when enabled)
-    isIgnoringMouseEvents.value = fadeEnabled
+    // Keep visible model pixels interactive; only the exact transparent pixel under the cursor
+    // should pass clicks through. The fuzzy transparency value above is intentionally reserved
+    // for fade stability near model edges.
+    const shouldIgnoreMouseEvents = fadeEnabled && isTransparentForMouseEvents.value
+    isIgnoringMouseEvents.value = shouldIgnoreMouseEvents
    shouldFadeOnCursorWithin.value = fadeEnabled && !isOutsideWindow.value && !isTransparent.value
-    setIgnoreMouseEvents([fadeEnabled, { forward: true }])
+    setIgnoreMouseEvents([shouldIgnoreMouseEvents, { forward: true }])
    if (fadeEnabled)
      resume()
    else
@@ -25,6 +25,7 @@ import type {
  VrmLoadHookContext,
  VrmMaterialHookContext,
 } from '../../composables/vrm/hooks'
+import type { VrmInteractionColliderSet } from '../../composables/vrm/interaction'
 import type { SceneBootstrap, TrackingMode, Vec3 } from '../../stores/model-store'
 import type { VrmLifecycleReason } from '../../trace'
 import type { ManagedVrmInstance } from './vrm-instance-cache'
@@ -74,6 +75,7 @@ import {
 } from '../../composables/vrm/animation'
 import { loadVrm } from '../../composables/vrm/core'
 import { useVRMEmote } from '../../composables/vrm/expression'
+import { createVrmInteractionColliders } from '../../composables/vrm/interaction'
 import { resolveInternalVrmHooks } from '../../composables/vrm/internal-hooks'
 import { useVRMLipSync } from '../../composables/vrm/lip-sync'
 import {
@@ -169,6 +171,8 @@ const { renderer, scene } = useTresContext()
 const vrm = shallowRef<VRM>()
 const vrmGroup = shallowRef<Group>()
 const modelLoaded = ref<boolean>(false)
+const interactionColliders = shallowRef<VrmInteractionColliderSet>()
+
 let loadSequence = 0
 // for eye tracking modes
 const raycaster = new Raycaster()
@@ -282,6 +286,7 @@ function getActiveManagedVrmInstance() {
  return createManagedVrmInstance({
    emote: vrmEmote.value,
    group: vrmGroup.value,
+    interactionColliders: interactionColliders.value!,
    mixer: vrmAnimationMixer.value,
    vrm: vrm.value,
  })
@@ -292,6 +297,7 @@ function clearActiveManagedVrmRefs() {
  vrmEmote.value = undefined
  vrm.value = undefined
  vrmGroup.value = undefined
+  interactionColliders.value = undefined
 }

 function applyManagedVrmInstance(instance: ManagedVrmInstance) {
@@ -299,6 +305,7 @@ function applyManagedVrmInstance(instance: ManagedVrmInstance) {
  vrmGroup.value = instance.group
  vrmAnimationMixer.value = instance.mixer
  vrmEmote.value = instance.emote
+  interactionColliders.value = instance.interactionColliders
 }

 function destroyManagedVrmInstance(instance?: ManagedVrmInstance) {
@@ -307,6 +314,7 @@ function destroyManagedVrmInstance(instance?: ManagedVrmInstance) {

  instance.emote.dispose()
  instance.mixer.stopAllAction()
+  instance.interactionColliders.dispose()
  disposeDetachedVrm(instance.vrm, instance.group)
 }

@@ -853,9 +861,12 @@ async function loadModel() {

    emit('sceneBootstrap', buildSceneBootstrap(_vrm, false))

+    const nextInteractionColliders = createVrmInteractionColliders(_vrm)
+
    commitManagedVrmInstance(createManagedVrmInstance({
      emote: nextVrmEmote,
      group: _vrmGroup,
+      interactionColliders: nextInteractionColliders,
      mixer: nextVrmAnimationMixer,
      vrm: _vrm,
    }))
@@ -1008,6 +1019,7 @@ if (import.meta.hot) {
 }

 defineExpose({
+  getInteractionColliders: () => interactionColliders.value?.colliders ?? [],
  setExpression(expression: string, intensity = 1) {
    vrmEmote.value?.setEmotionWithResetAfter(expression, 3000, intensity)
  },
@@ -2,6 +2,7 @@ import type { VRM } from '@pixiv/three-vrm'
 import type { AnimationMixer, Group } from 'three'

 import type { useVRMEmote } from '../../composables/vrm/expression'
+import type { VrmInteractionColliderSet } from '../../composables/vrm/interaction'

 import { getStageThreeRuntimeTraceContext, isStageThreeRuntimeTraceEnabled } from '../../trace/context'
 import { stageThreeTraceVrmCacheEvent } from '../../trace/eventa'
@@ -9,6 +10,7 @@ import { stageThreeTraceVrmCacheEvent } from '../../trace/eventa'
 export interface ManagedVrmInstance {
  emote: ReturnType<typeof useVRMEmote>
  group: Group
+  interactionColliders: VrmInteractionColliderSet
  mixer: AnimationMixer
  modelSrc: string
  scopeKey: string
@@ -11,6 +11,7 @@ import type { VRM } from '@pixiv/three-vrm'
 import type { TresContext } from '@tresjs/core'
 import type { DirectionalLight, SphericalHarmonics3, Texture, WebGLRenderer, WebGLRenderTarget } from 'three'

+import type { VrmInteractionTarget } from '../composables/vrm/interaction'
 import type { SceneBootstrap, ScenePhase, Vec3 } from '../stores/model-store'
 import type { VrmLifecycleReason } from '../trace'

@@ -25,12 +26,15 @@ import {
  Euler,
  MathUtils,
  PerspectiveCamera,
+  Raycaster,
+  Vector2,
  Vector3,
 } from 'three'
 import { computed, nextTick, onMounted, onUnmounted, ref, shallowRef, watch } from 'vue'

 // From stage-ui-three package
 import { useRenderTargetRegionAtClientPoint } from '../composables/render-target'
+import { getVrmInteractionTargetFromObjectName, isClickLikePointerGesture } from '../composables/vrm/interaction'
 // pinia store
 import { useModelStore } from '../stores/model-store'
 import {
@@ -80,6 +84,7 @@ const props = withDefaults(defineProps<{
 const emit = defineEmits<{
  (e: 'loadModelProgress', value: number): void
  (e: 'error', value: unknown): void
+  (e: 'vrmInteract', value: VrmInteractionTarget): void
 }>()

 type ModelPhase = 'no-model' | 'loading' | 'ready' | 'error'
@@ -498,6 +503,9 @@ function onSkyBoxReady(EnvPayload: {
 function onTresReady(context: TresContext) {
  tresContextRef.value = context
  canvasReady.value = true
+  context.renderer.instance.domElement.addEventListener('pointerdown', onCanvasPointerDown)
+  context.renderer.instance.domElement.addEventListener('pointerup', onCanvasPointerUp)
+  context.renderer.instance.domElement.addEventListener('pointercancel', onCanvasPointerCancel)
  emitSceneSubtreeTrace('tresCanvasRef', 'attached')
  setScenePhaseWithTrace(resolveScenePhaseAfterBinding(), 'tres:ready')
 }
@@ -521,6 +529,53 @@ function onTresRender() {
  })
 }

+const pickingRaycaster = new Raycaster()
+const pickingMouse = new Vector2()
+let activePointer: { id: number, x: number, y: number } | undefined
+
+function onCanvasPointerDown(event: PointerEvent) {
+  if (!event.isPrimary || event.button !== 0)
+    return
+  activePointer = { id: event.pointerId, x: event.clientX, y: event.clientY }
+}
+
+function onCanvasPointerCancel(event: PointerEvent) {
+  if (activePointer?.id === event.pointerId)
+    activePointer = undefined
+}
+
+function onCanvasPointerUp(event: PointerEvent) {
+  const pointer = activePointer
+  activePointer = undefined
+  if (!pointer || pointer.id !== event.pointerId || !event.isPrimary)
+    return
+  if (!isClickLikePointerGesture(pointer, { x: event.clientX, y: event.clientY }))
+    return
+  handleCanvasInteraction(event)
+}
+
+function handleCanvasInteraction(event: PointerEvent) {
+  const canvasElement = tresContextRef.value?.renderer.instance.domElement
+  if (!canvasElement || !modelRef.value)
+    return
+
+  const rect = canvasElement.getBoundingClientRect()
+  if (rect.width <= 0 || rect.height <= 0)
+    return
+
+  pickingMouse.x = ((event.clientX - rect.left) / rect.width) * 2 - 1
+  pickingMouse.y = -((event.clientY - rect.top) / rect.height) * 2 + 1
+
+  pickingRaycaster.setFromCamera(pickingMouse, camera.value)
+
+  const activeColliders = modelRef.value.getInteractionColliders?.() ?? []
+  const intersects = pickingRaycaster.intersectObjects([...activeColliders])
+
+  const target = getVrmInteractionTargetFromObjectName(intersects[0]?.object.name ?? '')
+  if (target)
+    emit('vrmInteract', target)
+}
+
 onMounted(() => {
  if (envSelect.value === 'skyBox') {
    skyBoxEnvRef.value?.reload(skyBoxSrc.value)
@@ -528,6 +583,14 @@ onMounted(() => {
 })

 onUnmounted(() => {
+  const canvas = tresContextRef.value?.renderer.instance.domElement
+  if (canvas) {
+    canvas.removeEventListener('pointerdown', onCanvasPointerDown)
+    canvas.removeEventListener('pointerup', onCanvasPointerUp)
+    canvas.removeEventListener('pointercancel', onCanvasPointerCancel)
+  }
+  activePointer = undefined
+
  invalidateBindingRevision()
  if (tresContextRef.value)
    emitSceneSubtreeTrace('tresCanvasRef', 'detached')
@@ -78,6 +78,12 @@ export function useVRMEmote(vrm: VRMCore) {
      ],
      blendDuration: 0.5,
    }],
+    ['relaxed', {
+      expression: [
+        { name: 'relaxed', value: 0.7 },
+      ],
+      blendDuration: 0.4,
+    }],
  ])

  const clearResetTimeout = () => {
@@ -121,7 +127,15 @@ export function useVRMEmote(vrm: VRMCore) {

    // Override target values for specified expressions in the emotion state
    for (const expr of emotionState.expression || []) {
-      targetExpressionValues.value.set(expr.name, expr.value * normalizedIntensity)
+      let actualName = expr.name
+      if (vrm.expressionManager) {
+        const modelNames = Object.keys(vrm.expressionManager.expressionMap)
+        const match = modelNames.find(n => n.toLowerCase() === expr.name.toLowerCase())
+        if (match) {
+          actualName = match
+        }
+      }
+      targetExpressionValues.value.set(actualName, expr.value * normalizedIntensity)
    }
  }

@@ -0,0 +1,20 @@
+import { describe, expect, it } from 'vitest'
+
+import {
+  getVrmInteractionTargetFromObjectName,
+  isClickLikePointerGesture,
+} from './interaction'
+
+describe('vrm interaction helpers', () => {
+  it('maps collider object names to typed interaction targets', () => {
+    expect(getVrmInteractionTargetFromObjectName('vrm_interaction_leftUpperArm')).toBe('leftUpperArm')
+    expect(getVrmInteractionTargetFromObjectName('vrm_interaction_rightHand')).toBe('rightHand')
+    expect(getVrmInteractionTargetFromObjectName('avatar_mesh')).toBeNull()
+    expect(getVrmInteractionTargetFromObjectName('vrm_interaction_unknown')).toBeNull()
+  })
+
+  it('accepts small pointer movement as a click', () => {
+    expect(isClickLikePointerGesture({ x: 10, y: 10 }, { x: 15, y: 13 })).toBe(true)
+    expect(isClickLikePointerGesture({ x: 10, y: 10 }, { x: 20, y: 10 })).toBe(false)
+  })
+})
@@ -0,0 +1,131 @@
+import type { VRM } from '@pixiv/three-vrm'
+
+import { Box3, BoxGeometry, Mesh, MeshBasicMaterial, Vector3 } from 'three'
+
+/** Normalized VRM humanoid regions that can produce a user interaction. */
+export const VRM_INTERACTION_TARGETS = [
+  'head',
+  'leftUpperArm',
+  'leftLowerArm',
+  'leftHand',
+  'rightUpperArm',
+  'rightLowerArm',
+  'rightHand',
+  'leftFoot',
+  'rightFoot',
+] as const
+
+/** Identifies the normalized humanoid region hit by a pointer interaction. */
+export type VrmInteractionTarget = (typeof VRM_INTERACTION_TARGETS)[number]
+
+const COLLIDER_NAME_PREFIX = 'vrm_interaction_'
+const MAX_MODEL_SCALE = 1.5
+const MIN_MODEL_SCALE = 0.65
+const REFERENCE_MODEL_HEIGHT = 1.6
+
+interface ColliderDefinition {
+  target: VrmInteractionTarget
+  bone: VrmInteractionTarget
+  size: readonly [number, number, number]
+  offset: readonly [number, number, number]
+}
+
+const COLLIDER_DEFINITIONS: readonly ColliderDefinition[] = [
+  { target: 'head', bone: 'head', size: [0.22, 0.25, 0.25], offset: [0, 0.05, 0] },
+  { target: 'leftUpperArm', bone: 'leftUpperArm', size: [0.2, 0.34, 0.2], offset: [0, -0.17, 0] },
+  { target: 'leftLowerArm', bone: 'leftLowerArm', size: [0.17, 0.3, 0.17], offset: [0, -0.15, 0] },
+  { target: 'leftHand', bone: 'leftHand', size: [0.2, 0.2, 0.2], offset: [0.06, 0, 0] },
+  { target: 'rightUpperArm', bone: 'rightUpperArm', size: [0.2, 0.34, 0.2], offset: [0, -0.17, 0] },
+  { target: 'rightLowerArm', bone: 'rightLowerArm', size: [0.17, 0.3, 0.17], offset: [0, -0.15, 0] },
+  { target: 'rightHand', bone: 'rightHand', size: [0.2, 0.2, 0.2], offset: [-0.06, 0, 0] },
+  { target: 'leftFoot', bone: 'leftFoot', size: [0.15, 0.15, 0.25], offset: [0, -0.05, -0.08] },
+  { target: 'rightFoot', bone: 'rightFoot', size: [0.15, 0.15, 0.25], offset: [0, -0.05, -0.08] },
+]
+
+export interface VrmInteractionColliderSet {
+  /** Invisible Three.js meshes used only by the interaction raycaster. */
+  readonly colliders: readonly Mesh[]
+  /** Removes the meshes from their bones and releases their GPU resources. */
+  dispose: () => void
+}
+
+function getModelScale(vrm: VRM) {
+  const bounds = new Box3().setFromObject(vrm.scene)
+  const size = new Vector3()
+  bounds.getSize(size)
+  if (!Number.isFinite(size.y) || size.y <= 0)
+    return 1
+  return Math.min(MAX_MODEL_SCALE, Math.max(MIN_MODEL_SCALE, size.y / REFERENCE_MODEL_HEIGHT))
+}
+
+/** Creates model-relative invisible hit regions attached to normalized VRM bones. */
+export function createVrmInteractionColliders(vrm: VRM): VrmInteractionColliderSet {
+  const material = new MeshBasicMaterial({ visible: false })
+  const scale = getModelScale(vrm)
+  const colliders: Mesh[] = []
+
+  for (const definition of COLLIDER_DEFINITIONS) {
+    const boneNode = vrm.humanoid?.getNormalizedBoneNode(definition.bone)
+    if (!boneNode)
+      continue
+
+    const geometry = new BoxGeometry(
+      definition.size[0] * scale,
+      definition.size[1] * scale,
+      definition.size[2] * scale,
+    )
+    const collider = new Mesh(geometry, material)
+    collider.name = `${COLLIDER_NAME_PREFIX}${definition.target}`
+    collider.position.set(
+      definition.offset[0] * scale,
+      definition.offset[1] * scale,
+      definition.offset[2] * scale,
+    )
+    boneNode.add(collider)
+    colliders.push(collider)
+  }
+
+  let disposed = false
+  return {
+    colliders,
+    dispose() {
+      if (disposed)
+        return
+      disposed = true
+      for (const collider of colliders) {
+        collider.removeFromParent()
+        collider.geometry.dispose()
+      }
+      material.dispose()
+      colliders.length = 0
+    },
+  }
+}
+
+/**
+ * Normalizes a collider object name into an interaction target.
+ *
+ * Before:
+ * - "vrm_interaction_leftUpperArm"
+ *
+ * After:
+ * - "leftUpperArm"
+ */
+export function getVrmInteractionTargetFromObjectName(name: string): VrmInteractionTarget | null {
+  if (!name.startsWith(COLLIDER_NAME_PREFIX))
+    return null
+  const target = name.slice(COLLIDER_NAME_PREFIX.length)
+  return (VRM_INTERACTION_TARGETS as readonly string[]).includes(target)
+    ? target as VrmInteractionTarget
+    : null
+}
+
+export function isClickLikePointerGesture(
+  start: { x: number, y: number },
+  end: { x: number, y: number },
+  maxDistance = 8,
+): boolean {
+  const dx = end.x - start.x
+  const dy = end.y - start.y
+  return dx * dx + dy * dy <= maxDistance * maxDistance
+}
@@ -2,6 +2,7 @@ export { default as ThreeScene } from './components/ThreeScene.vue'

 export * from './composables/hit-test'
 export * from './composables/render-target'
+export * from './composables/vrm/interaction'
 export { useModelStore } from './stores/model-store'
 export * from './stores/view-control'
 export * from './trace'
@@ -1,6 +1,7 @@
 <script setup lang="ts">
 import type { Live2DLipSync, Live2DLipSyncOptions } from '@proj-airi/model-driver-lipsync'
 import type { Profile } from '@proj-airi/model-driver-lipsync/shared/wlipsync'
+import type { VrmInteractionTarget } from '@proj-airi/stage-ui-three'
 import type { SpeechProviderWithExtraOptions } from '@xsai-ext/providers/utils'
 import type { UnElevenLabsOptions } from 'unspeech'

@@ -91,6 +92,25 @@ const { mouthOpenSize, nowSpeaking } = storeToRefs(useSpeakingStore())
 const { audioContext } = useAudioContext()
 const currentAudioSource = ref<AudioBufferSourceNode>()
 const { latestStopRequest } = storeToRefs(useSpeechOutputControlStore())
+const lastVrmInteractionAt = new Map<VrmInteractionTarget, number>()
+const VRM_INTERACTION_COOLDOWN_MS = 450
+
+function getVrmInteractionExpression(target: VrmInteractionTarget) {
+  if (target === 'head')
+    return 'happy'
+  if (target === 'leftFoot' || target === 'rightFoot')
+    return 'relaxed'
+  return 'surprised'
+}
+
+function onVRMInteract(target: VrmInteractionTarget) {
+  const now = Date.now()
+  const lastTriggeredAt = lastVrmInteractionAt.get(target) ?? 0
+  if (now - lastTriggeredAt < VRM_INTERACTION_COOLDOWN_MS)
+    return
+  lastVrmInteractionAt.set(target, now)
+  vrmViewerRef.value?.setExpression(getVrmInteractionExpression(target), 1)
+}

 const { onBeforeMessageComposed, onBeforeSend, onTokenLiteral, onTokenSpecial, onStreamEnd, onAssistantResponseEnd } = useChatOrchestratorStore()
 const chatHookCleanups: Array<() => void> = []
@@ -1012,6 +1032,7 @@ defineExpose({
        :enable-orbit-controls="props.enableOrbitControls"
        :current-audio-source="currentAudioSource"
        @error="console.error"
+        @vrm-interact="onVRMInteract"
      />
      <SpineScene
        v-if="stageModelRenderer === 'spine' && showStage"
@@ -105,4 +105,22 @@ describe('isCanvasRegionTransparent', () => {

    expect(result).toBe(true)
  })
+
+  it('uses only the exact cursor pixel when the radius is zero', () => {
+    const gl = createGlMock({ hotPixel: { x: 50, y: 49, alpha: 255 } })
+
+    const result = isCanvasRegionTransparent({
+      gl,
+      clientX: 50,
+      clientY: 50,
+      left: 0,
+      top: 0,
+      width: 100,
+      height: 100,
+      radius: 0,
+      threshold: 10,
+    })
+
+    expect(result).toBe(false)
+  })
 })