From 78d6588cb07ff0114ca584f4f87886820dd9c4fb Mon Sep 17 00:00:00 2001 From: 0xSelenicDove <140679517+0xSelenicDove@users.noreply.github.com> Date: Mon, 20 Jul 2026 16:13:03 +0800 Subject: [PATCH] feat(stage-*): add clickable VRM body-part reactions (#2068) --- .../main/services/airi/plugins/index.test.ts | 11 +- .../src/renderer/pages/index.vue | 31 ++++- .../src/components/Model/VRMModel.vue | 12 ++ .../components/Model/vrm-instance-cache.ts | 2 + .../src/components/ThreeScene.vue | 63 +++++++++ .../src/composables/vrm/expression.ts | 16 ++- .../src/composables/vrm/interaction.test.ts | 20 +++ .../src/composables/vrm/interaction.ts | 131 ++++++++++++++++++ packages/stage-ui-three/src/index.ts | 1 + .../stage-ui/src/components/scenes/Stage.vue | 21 +++ .../src/composables/canvas-alpha.test.ts | 18 +++ 11 files changed, 321 insertions(+), 5 deletions(-) create mode 100644 packages/stage-ui-three/src/composables/vrm/interaction.test.ts create mode 100644 packages/stage-ui-three/src/composables/vrm/interaction.ts diff --git a/apps/stage-tamagotchi/src/main/services/airi/plugins/index.test.ts b/apps/stage-tamagotchi/src/main/services/airi/plugins/index.test.ts index e736017cc..b23a2539a 100644 --- a/apps/stage-tamagotchi/src/main/services/airi/plugins/index.test.ts +++ b/apps/stage-tamagotchi/src/main/services/airi/plugins/index.test.ts @@ -668,7 +668,16 @@ describe('setupExtensionHost', () => { while (Date.now() < deadline && afterSessionId === beforeSession?.id) { await new Promise(resolve => setTimeout(resolve, 100)) const snapshot = await invokeInspect() - afterSessionId = snapshot.sessions.find(session => session.extensionId === 'test-auto-reload-reload')?.id + const currentSessionId = snapshot.sessions.find(session => session.extensionId === 'test-auto-reload-reload')?.id + + // ROOT CAUSE: + // + // Reload stops the old session before starting the replacement, so an inspect call can + // observe the intentional short-lived gap with no session. Treat that state as pending + // instead of ending the poll before the replacement session is available. + if (currentSessionId && currentSessionId !== beforeSession?.id) { + afterSessionId = currentSessionId + } } expect(afterSessionId).toBeDefined() diff --git a/apps/stage-tamagotchi/src/renderer/pages/index.vue b/apps/stage-tamagotchi/src/renderer/pages/index.vue index aadea7902..5f6268969 100644 --- a/apps/stage-tamagotchi/src/renderer/pages/index.vue +++ b/apps/stage-tamagotchi/src/renderer/pages/index.vue @@ -84,6 +84,16 @@ const isTransparentByThree = useThreeSceneIsTransparentAtPoint( relativeMouseY, { regionRadius: 25 }, ) +const isTransparentByPixelsExact = useCanvasPixelIsTransparentAtPoint( + stageCanvas, + relativeMouseX, + relativeMouseY, +) +const isTransparentByThreeExact = useThreeSceneIsTransparentAtPoint( + widgetStageRef, + relativeMouseX, + relativeMouseY, +) const settingsStore = useSettings() const { stageModelRenderer, stageModelSelectedUrl } = storeToRefs(settingsStore) @@ -111,6 +121,18 @@ const isTransparent = computed(() => { return true }) +const isTransparentForMouseEvents = computed(() => { + if (stagePaused.value || componentStateStage.value !== 'mounted' || !fadeOnHoverEnabled.value) + return true + + if (stageModelRenderer.value === 'vrm') + return shouldUseThreeTransparencyHitTest.value ? isTransparentByThreeExact.value : true + + if (stageModelRenderer.value === 'live2d') + return isTransparentByPixelsExact.value + + return true +}) const { isNearAnyBorder: isAroundWindowBorder } = useElectronMouseAroundWindowBorder({ threshold: 10 }) const isAroundWindowBorderFor250Ms = refDebounced(isAroundWindowBorder, 250) @@ -216,10 +238,13 @@ watch([isOutsideFor250Ms, isOutsideStatusIslandFor250Ms, isAroundWindowBorderFor } else { const fadeEnabled = fadeOnHoverEnabled.value - // Otherwise allow click-through while we fade UI based on transparency (when enabled) - isIgnoringMouseEvents.value = fadeEnabled + // Keep visible model pixels interactive; only the exact transparent pixel under the cursor + // should pass clicks through. The fuzzy transparency value above is intentionally reserved + // for fade stability near model edges. + const shouldIgnoreMouseEvents = fadeEnabled && isTransparentForMouseEvents.value + isIgnoringMouseEvents.value = shouldIgnoreMouseEvents shouldFadeOnCursorWithin.value = fadeEnabled && !isOutsideWindow.value && !isTransparent.value - setIgnoreMouseEvents([fadeEnabled, { forward: true }]) + setIgnoreMouseEvents([shouldIgnoreMouseEvents, { forward: true }]) if (fadeEnabled) resume() else diff --git a/packages/stage-ui-three/src/components/Model/VRMModel.vue b/packages/stage-ui-three/src/components/Model/VRMModel.vue index 64dec17d0..6e9890aaf 100644 --- a/packages/stage-ui-three/src/components/Model/VRMModel.vue +++ b/packages/stage-ui-three/src/components/Model/VRMModel.vue @@ -25,6 +25,7 @@ import type { VrmLoadHookContext, VrmMaterialHookContext, } from '../../composables/vrm/hooks' +import type { VrmInteractionColliderSet } from '../../composables/vrm/interaction' import type { SceneBootstrap, TrackingMode, Vec3 } from '../../stores/model-store' import type { VrmLifecycleReason } from '../../trace' import type { ManagedVrmInstance } from './vrm-instance-cache' @@ -74,6 +75,7 @@ import { } from '../../composables/vrm/animation' import { loadVrm } from '../../composables/vrm/core' import { useVRMEmote } from '../../composables/vrm/expression' +import { createVrmInteractionColliders } from '../../composables/vrm/interaction' import { resolveInternalVrmHooks } from '../../composables/vrm/internal-hooks' import { useVRMLipSync } from '../../composables/vrm/lip-sync' import { @@ -169,6 +171,8 @@ const { renderer, scene } = useTresContext() const vrm = shallowRef() const vrmGroup = shallowRef() const modelLoaded = ref(false) +const interactionColliders = shallowRef() + let loadSequence = 0 // for eye tracking modes const raycaster = new Raycaster() @@ -282,6 +286,7 @@ function getActiveManagedVrmInstance() { return createManagedVrmInstance({ emote: vrmEmote.value, group: vrmGroup.value, + interactionColliders: interactionColliders.value!, mixer: vrmAnimationMixer.value, vrm: vrm.value, }) @@ -292,6 +297,7 @@ function clearActiveManagedVrmRefs() { vrmEmote.value = undefined vrm.value = undefined vrmGroup.value = undefined + interactionColliders.value = undefined } function applyManagedVrmInstance(instance: ManagedVrmInstance) { @@ -299,6 +305,7 @@ function applyManagedVrmInstance(instance: ManagedVrmInstance) { vrmGroup.value = instance.group vrmAnimationMixer.value = instance.mixer vrmEmote.value = instance.emote + interactionColliders.value = instance.interactionColliders } function destroyManagedVrmInstance(instance?: ManagedVrmInstance) { @@ -307,6 +314,7 @@ function destroyManagedVrmInstance(instance?: ManagedVrmInstance) { instance.emote.dispose() instance.mixer.stopAllAction() + instance.interactionColliders.dispose() disposeDetachedVrm(instance.vrm, instance.group) } @@ -853,9 +861,12 @@ async function loadModel() { emit('sceneBootstrap', buildSceneBootstrap(_vrm, false)) + const nextInteractionColliders = createVrmInteractionColliders(_vrm) + commitManagedVrmInstance(createManagedVrmInstance({ emote: nextVrmEmote, group: _vrmGroup, + interactionColliders: nextInteractionColliders, mixer: nextVrmAnimationMixer, vrm: _vrm, })) @@ -1008,6 +1019,7 @@ if (import.meta.hot) { } defineExpose({ + getInteractionColliders: () => interactionColliders.value?.colliders ?? [], setExpression(expression: string, intensity = 1) { vrmEmote.value?.setEmotionWithResetAfter(expression, 3000, intensity) }, diff --git a/packages/stage-ui-three/src/components/Model/vrm-instance-cache.ts b/packages/stage-ui-three/src/components/Model/vrm-instance-cache.ts index 479f7f3c4..cd8c42f8d 100644 --- a/packages/stage-ui-three/src/components/Model/vrm-instance-cache.ts +++ b/packages/stage-ui-three/src/components/Model/vrm-instance-cache.ts @@ -2,6 +2,7 @@ import type { VRM } from '@pixiv/three-vrm' import type { AnimationMixer, Group } from 'three' import type { useVRMEmote } from '../../composables/vrm/expression' +import type { VrmInteractionColliderSet } from '../../composables/vrm/interaction' import { getStageThreeRuntimeTraceContext, isStageThreeRuntimeTraceEnabled } from '../../trace/context' import { stageThreeTraceVrmCacheEvent } from '../../trace/eventa' @@ -9,6 +10,7 @@ import { stageThreeTraceVrmCacheEvent } from '../../trace/eventa' export interface ManagedVrmInstance { emote: ReturnType group: Group + interactionColliders: VrmInteractionColliderSet mixer: AnimationMixer modelSrc: string scopeKey: string diff --git a/packages/stage-ui-three/src/components/ThreeScene.vue b/packages/stage-ui-three/src/components/ThreeScene.vue index 6c25082de..e3b24151d 100644 --- a/packages/stage-ui-three/src/components/ThreeScene.vue +++ b/packages/stage-ui-three/src/components/ThreeScene.vue @@ -11,6 +11,7 @@ import type { VRM } from '@pixiv/three-vrm' import type { TresContext } from '@tresjs/core' import type { DirectionalLight, SphericalHarmonics3, Texture, WebGLRenderer, WebGLRenderTarget } from 'three' +import type { VrmInteractionTarget } from '../composables/vrm/interaction' import type { SceneBootstrap, ScenePhase, Vec3 } from '../stores/model-store' import type { VrmLifecycleReason } from '../trace' @@ -25,12 +26,15 @@ import { Euler, MathUtils, PerspectiveCamera, + Raycaster, + Vector2, Vector3, } from 'three' import { computed, nextTick, onMounted, onUnmounted, ref, shallowRef, watch } from 'vue' // From stage-ui-three package import { useRenderTargetRegionAtClientPoint } from '../composables/render-target' +import { getVrmInteractionTargetFromObjectName, isClickLikePointerGesture } from '../composables/vrm/interaction' // pinia store import { useModelStore } from '../stores/model-store' import { @@ -80,6 +84,7 @@ const props = withDefaults(defineProps<{ const emit = defineEmits<{ (e: 'loadModelProgress', value: number): void (e: 'error', value: unknown): void + (e: 'vrmInteract', value: VrmInteractionTarget): void }>() type ModelPhase = 'no-model' | 'loading' | 'ready' | 'error' @@ -498,6 +503,9 @@ function onSkyBoxReady(EnvPayload: { function onTresReady(context: TresContext) { tresContextRef.value = context canvasReady.value = true + context.renderer.instance.domElement.addEventListener('pointerdown', onCanvasPointerDown) + context.renderer.instance.domElement.addEventListener('pointerup', onCanvasPointerUp) + context.renderer.instance.domElement.addEventListener('pointercancel', onCanvasPointerCancel) emitSceneSubtreeTrace('tresCanvasRef', 'attached') setScenePhaseWithTrace(resolveScenePhaseAfterBinding(), 'tres:ready') } @@ -521,6 +529,53 @@ function onTresRender() { }) } +const pickingRaycaster = new Raycaster() +const pickingMouse = new Vector2() +let activePointer: { id: number, x: number, y: number } | undefined + +function onCanvasPointerDown(event: PointerEvent) { + if (!event.isPrimary || event.button !== 0) + return + activePointer = { id: event.pointerId, x: event.clientX, y: event.clientY } +} + +function onCanvasPointerCancel(event: PointerEvent) { + if (activePointer?.id === event.pointerId) + activePointer = undefined +} + +function onCanvasPointerUp(event: PointerEvent) { + const pointer = activePointer + activePointer = undefined + if (!pointer || pointer.id !== event.pointerId || !event.isPrimary) + return + if (!isClickLikePointerGesture(pointer, { x: event.clientX, y: event.clientY })) + return + handleCanvasInteraction(event) +} + +function handleCanvasInteraction(event: PointerEvent) { + const canvasElement = tresContextRef.value?.renderer.instance.domElement + if (!canvasElement || !modelRef.value) + return + + const rect = canvasElement.getBoundingClientRect() + if (rect.width <= 0 || rect.height <= 0) + return + + pickingMouse.x = ((event.clientX - rect.left) / rect.width) * 2 - 1 + pickingMouse.y = -((event.clientY - rect.top) / rect.height) * 2 + 1 + + pickingRaycaster.setFromCamera(pickingMouse, camera.value) + + const activeColliders = modelRef.value.getInteractionColliders?.() ?? [] + const intersects = pickingRaycaster.intersectObjects([...activeColliders]) + + const target = getVrmInteractionTargetFromObjectName(intersects[0]?.object.name ?? '') + if (target) + emit('vrmInteract', target) +} + onMounted(() => { if (envSelect.value === 'skyBox') { skyBoxEnvRef.value?.reload(skyBoxSrc.value) @@ -528,6 +583,14 @@ onMounted(() => { }) onUnmounted(() => { + const canvas = tresContextRef.value?.renderer.instance.domElement + if (canvas) { + canvas.removeEventListener('pointerdown', onCanvasPointerDown) + canvas.removeEventListener('pointerup', onCanvasPointerUp) + canvas.removeEventListener('pointercancel', onCanvasPointerCancel) + } + activePointer = undefined + invalidateBindingRevision() if (tresContextRef.value) emitSceneSubtreeTrace('tresCanvasRef', 'detached') diff --git a/packages/stage-ui-three/src/composables/vrm/expression.ts b/packages/stage-ui-three/src/composables/vrm/expression.ts index dcfde6976..195cd96d6 100644 --- a/packages/stage-ui-three/src/composables/vrm/expression.ts +++ b/packages/stage-ui-three/src/composables/vrm/expression.ts @@ -78,6 +78,12 @@ export function useVRMEmote(vrm: VRMCore) { ], blendDuration: 0.5, }], + ['relaxed', { + expression: [ + { name: 'relaxed', value: 0.7 }, + ], + blendDuration: 0.4, + }], ]) const clearResetTimeout = () => { @@ -121,7 +127,15 @@ export function useVRMEmote(vrm: VRMCore) { // Override target values for specified expressions in the emotion state for (const expr of emotionState.expression || []) { - targetExpressionValues.value.set(expr.name, expr.value * normalizedIntensity) + let actualName = expr.name + if (vrm.expressionManager) { + const modelNames = Object.keys(vrm.expressionManager.expressionMap) + const match = modelNames.find(n => n.toLowerCase() === expr.name.toLowerCase()) + if (match) { + actualName = match + } + } + targetExpressionValues.value.set(actualName, expr.value * normalizedIntensity) } } diff --git a/packages/stage-ui-three/src/composables/vrm/interaction.test.ts b/packages/stage-ui-three/src/composables/vrm/interaction.test.ts new file mode 100644 index 000000000..791cf464b --- /dev/null +++ b/packages/stage-ui-three/src/composables/vrm/interaction.test.ts @@ -0,0 +1,20 @@ +import { describe, expect, it } from 'vitest' + +import { + getVrmInteractionTargetFromObjectName, + isClickLikePointerGesture, +} from './interaction' + +describe('vrm interaction helpers', () => { + it('maps collider object names to typed interaction targets', () => { + expect(getVrmInteractionTargetFromObjectName('vrm_interaction_leftUpperArm')).toBe('leftUpperArm') + expect(getVrmInteractionTargetFromObjectName('vrm_interaction_rightHand')).toBe('rightHand') + expect(getVrmInteractionTargetFromObjectName('avatar_mesh')).toBeNull() + expect(getVrmInteractionTargetFromObjectName('vrm_interaction_unknown')).toBeNull() + }) + + it('accepts small pointer movement as a click', () => { + expect(isClickLikePointerGesture({ x: 10, y: 10 }, { x: 15, y: 13 })).toBe(true) + expect(isClickLikePointerGesture({ x: 10, y: 10 }, { x: 20, y: 10 })).toBe(false) + }) +}) diff --git a/packages/stage-ui-three/src/composables/vrm/interaction.ts b/packages/stage-ui-three/src/composables/vrm/interaction.ts new file mode 100644 index 000000000..b9b38ca3e --- /dev/null +++ b/packages/stage-ui-three/src/composables/vrm/interaction.ts @@ -0,0 +1,131 @@ +import type { VRM } from '@pixiv/three-vrm' + +import { Box3, BoxGeometry, Mesh, MeshBasicMaterial, Vector3 } from 'three' + +/** Normalized VRM humanoid regions that can produce a user interaction. */ +export const VRM_INTERACTION_TARGETS = [ + 'head', + 'leftUpperArm', + 'leftLowerArm', + 'leftHand', + 'rightUpperArm', + 'rightLowerArm', + 'rightHand', + 'leftFoot', + 'rightFoot', +] as const + +/** Identifies the normalized humanoid region hit by a pointer interaction. */ +export type VrmInteractionTarget = (typeof VRM_INTERACTION_TARGETS)[number] + +const COLLIDER_NAME_PREFIX = 'vrm_interaction_' +const MAX_MODEL_SCALE = 1.5 +const MIN_MODEL_SCALE = 0.65 +const REFERENCE_MODEL_HEIGHT = 1.6 + +interface ColliderDefinition { + target: VrmInteractionTarget + bone: VrmInteractionTarget + size: readonly [number, number, number] + offset: readonly [number, number, number] +} + +const COLLIDER_DEFINITIONS: readonly ColliderDefinition[] = [ + { target: 'head', bone: 'head', size: [0.22, 0.25, 0.25], offset: [0, 0.05, 0] }, + { target: 'leftUpperArm', bone: 'leftUpperArm', size: [0.2, 0.34, 0.2], offset: [0, -0.17, 0] }, + { target: 'leftLowerArm', bone: 'leftLowerArm', size: [0.17, 0.3, 0.17], offset: [0, -0.15, 0] }, + { target: 'leftHand', bone: 'leftHand', size: [0.2, 0.2, 0.2], offset: [0.06, 0, 0] }, + { target: 'rightUpperArm', bone: 'rightUpperArm', size: [0.2, 0.34, 0.2], offset: [0, -0.17, 0] }, + { target: 'rightLowerArm', bone: 'rightLowerArm', size: [0.17, 0.3, 0.17], offset: [0, -0.15, 0] }, + { target: 'rightHand', bone: 'rightHand', size: [0.2, 0.2, 0.2], offset: [-0.06, 0, 0] }, + { target: 'leftFoot', bone: 'leftFoot', size: [0.15, 0.15, 0.25], offset: [0, -0.05, -0.08] }, + { target: 'rightFoot', bone: 'rightFoot', size: [0.15, 0.15, 0.25], offset: [0, -0.05, -0.08] }, +] + +export interface VrmInteractionColliderSet { + /** Invisible Three.js meshes used only by the interaction raycaster. */ + readonly colliders: readonly Mesh[] + /** Removes the meshes from their bones and releases their GPU resources. */ + dispose: () => void +} + +function getModelScale(vrm: VRM) { + const bounds = new Box3().setFromObject(vrm.scene) + const size = new Vector3() + bounds.getSize(size) + if (!Number.isFinite(size.y) || size.y <= 0) + return 1 + return Math.min(MAX_MODEL_SCALE, Math.max(MIN_MODEL_SCALE, size.y / REFERENCE_MODEL_HEIGHT)) +} + +/** Creates model-relative invisible hit regions attached to normalized VRM bones. */ +export function createVrmInteractionColliders(vrm: VRM): VrmInteractionColliderSet { + const material = new MeshBasicMaterial({ visible: false }) + const scale = getModelScale(vrm) + const colliders: Mesh[] = [] + + for (const definition of COLLIDER_DEFINITIONS) { + const boneNode = vrm.humanoid?.getNormalizedBoneNode(definition.bone) + if (!boneNode) + continue + + const geometry = new BoxGeometry( + definition.size[0] * scale, + definition.size[1] * scale, + definition.size[2] * scale, + ) + const collider = new Mesh(geometry, material) + collider.name = `${COLLIDER_NAME_PREFIX}${definition.target}` + collider.position.set( + definition.offset[0] * scale, + definition.offset[1] * scale, + definition.offset[2] * scale, + ) + boneNode.add(collider) + colliders.push(collider) + } + + let disposed = false + return { + colliders, + dispose() { + if (disposed) + return + disposed = true + for (const collider of colliders) { + collider.removeFromParent() + collider.geometry.dispose() + } + material.dispose() + colliders.length = 0 + }, + } +} + +/** + * Normalizes a collider object name into an interaction target. + * + * Before: + * - "vrm_interaction_leftUpperArm" + * + * After: + * - "leftUpperArm" + */ +export function getVrmInteractionTargetFromObjectName(name: string): VrmInteractionTarget | null { + if (!name.startsWith(COLLIDER_NAME_PREFIX)) + return null + const target = name.slice(COLLIDER_NAME_PREFIX.length) + return (VRM_INTERACTION_TARGETS as readonly string[]).includes(target) + ? target as VrmInteractionTarget + : null +} + +export function isClickLikePointerGesture( + start: { x: number, y: number }, + end: { x: number, y: number }, + maxDistance = 8, +): boolean { + const dx = end.x - start.x + const dy = end.y - start.y + return dx * dx + dy * dy <= maxDistance * maxDistance +} diff --git a/packages/stage-ui-three/src/index.ts b/packages/stage-ui-three/src/index.ts index 51a841faa..00571f4a2 100644 --- a/packages/stage-ui-three/src/index.ts +++ b/packages/stage-ui-three/src/index.ts @@ -2,6 +2,7 @@ export { default as ThreeScene } from './components/ThreeScene.vue' export * from './composables/hit-test' export * from './composables/render-target' +export * from './composables/vrm/interaction' export { useModelStore } from './stores/model-store' export * from './stores/view-control' export * from './trace' diff --git a/packages/stage-ui/src/components/scenes/Stage.vue b/packages/stage-ui/src/components/scenes/Stage.vue index 966f39b3b..f4103cecb 100644 --- a/packages/stage-ui/src/components/scenes/Stage.vue +++ b/packages/stage-ui/src/components/scenes/Stage.vue @@ -1,6 +1,7 @@