diff --git a/apps/stage-tamagotchi/package.json b/apps/stage-tamagotchi/package.json index 5a102998a..136905366 100644 --- a/apps/stage-tamagotchi/package.json +++ b/apps/stage-tamagotchi/package.json @@ -59,6 +59,8 @@ "@proj-airi/server-sdk": "workspace:*", "@proj-airi/stage-layouts": "workspace:^", "@proj-airi/stage-ui": "workspace:^", + "@proj-airi/stage-ui-live2d": "workspace:^", + "@proj-airi/stage-ui-three": "workspace:^", "@proj-airi/ui": "workspace:^", "@stdlib/string-base-kebabcase": "^0.2.2", "@tresjs/cientos": "^5.2.1", diff --git a/apps/stage-tamagotchi/src/renderer/pages/index.vue b/apps/stage-tamagotchi/src/renderer/pages/index.vue index 4e424b393..5e448b666 100644 --- a/apps/stage-tamagotchi/src/renderer/pages/index.vue +++ b/apps/stage-tamagotchi/src/renderer/pages/index.vue @@ -3,6 +3,7 @@ import type { ChatProvider } from '@xsai-ext/providers/utils' import workletUrl from '@proj-airi/stage-ui/workers/vad/process.worklet?worker&url' +import { useThreeSceneIsTransparentAtPoint } from '@proj-airi/stage-ui-three' import { WidgetStage } from '@proj-airi/stage-ui/components/scenes' import { useAudioRecorder } from '@proj-airi/stage-ui/composables/audio/audio-recorder' import { useCanvasPixelIsTransparentAtPoint } from '@proj-airi/stage-ui/composables/canvas-alpha' @@ -12,7 +13,7 @@ import { useLive2d } from '@proj-airi/stage-ui/stores/live2d' import { useConsciousnessStore } from '@proj-airi/stage-ui/stores/modules/consciousness' import { useHearingSpeechInputPipeline } from '@proj-airi/stage-ui/stores/modules/hearing' import { useProvidersStore } from '@proj-airi/stage-ui/stores/providers' -import { useSettingsAudioDevice } from '@proj-airi/stage-ui/stores/settings' +import { useSettings, useSettingsAudioDevice } from '@proj-airi/stage-ui/stores/settings' import { refDebounced, useBroadcastChannel } from '@vueuse/core' import { storeToRefs } from 'pinia' import { computed, onUnmounted, ref, toRef, watch } from 'vue' @@ -32,7 +33,7 @@ import { useControlsIslandStore } from '../stores/controls-island' import { useWindowStore } from '../stores/window' const controlsIslandRef = ref>() -const widgetStageRef = ref<{ canvasElement: () => HTMLCanvasElement }>() +const widgetStageRef = ref>() const stageCanvas = toRef(() => widgetStageRef.value?.canvasElement()) const componentStateStage = ref<'pending' | 'loading' | 'mounted'>('pending') @@ -47,9 +48,31 @@ const isOutsideFor250Ms = refDebounced(isOutside, 250) const { x: relativeMouseX, y: relativeMouseY } = useElectronRelativeMouse() // NOTICE: In real-world use cases of Fade on Hover feature, the cursor may move around the edge of the // model rapidly, causing flickering effects when checking pixel transparency strictly. -// Here we use `regionRadius` to help to detect with look-ahead strategy to relax the pixel accurate -// check / detection on hovered underlying canvas pixel, therefore inaccurate mouse movements won't cause flickering. -const isTransparent = useCanvasPixelIsTransparentAtPoint(stageCanvas, relativeMouseX, relativeMouseY, { regionRadius: 25 }) +// Here we use render-target pixel sampling to keep detection aligned with the actual render output. +const isTransparentByPixels = useCanvasPixelIsTransparentAtPoint( + stageCanvas, + relativeMouseX, + relativeMouseY, + { regionRadius: 25 }, +) +const isTransparentByThree = useThreeSceneIsTransparentAtPoint( + widgetStageRef, + relativeMouseX, + relativeMouseY, + { regionRadius: 25 }, +) + +const { stageModelRenderer } = storeToRefs(useSettings()) +const isTransparent = computed(() => { + if (stageModelRenderer.value === 'vrm') + return isTransparentByThree.value + + if (stageModelRenderer.value === 'live2d') + return isTransparentByPixels.value + + return true +}) + const { isNearAnyBorder: isAroundWindowBorder } = useElectronMouseAroundWindowBorder({ threshold: 30 }) const isAroundWindowBorderFor250Ms = refDebounced(isAroundWindowBorder, 250) diff --git a/packages/stage-ui-three/package.json b/packages/stage-ui-three/package.json index 67a9a567d..87f84c156 100644 --- a/packages/stage-ui-three/package.json +++ b/packages/stage-ui-three/package.json @@ -34,6 +34,7 @@ "@tresjs/post-processing": "^3.2.0", "@vueuse/core": "^14.1.0", "culori": "^4.0.2", + "es-toolkit": "catalog:", "pinia": "^3.0.4", "postprocessing": "^6.38.2", "three": "^0.182.0", diff --git a/packages/stage-ui-three/src/components/ThreeScene.vue b/packages/stage-ui-three/src/components/ThreeScene.vue index 3608b1266..1d4310a9f 100644 --- a/packages/stage-ui-three/src/components/ThreeScene.vue +++ b/packages/stage-ui-three/src/components/ThreeScene.vue @@ -9,7 +9,7 @@ import type { VRM } from '@pixiv/three-vrm' import type { TresContext } from '@tresjs/core' -import type { DirectionalLight, SphericalHarmonics3, Texture, WebGLRenderTarget } from 'three' +import type { DirectionalLight, SphericalHarmonics3, Texture, WebGLRenderer, WebGLRenderTarget } from 'three' import type { Vec3 } from '../stores/model-store' @@ -29,9 +29,10 @@ import { } from 'three' import { onMounted, onUnmounted, ref, shallowRef, watch } from 'vue' +// From stage-ui-three package +import { useRenderTargetRegionAtClientPoint } from '../composables/render-target' // pinia store import { useModelStore } from '../stores/model-store' -// From stage-ui-three package import { OrbitControls } from './Controls' import { SkyBox } from './Environment' import { VRMModel } from './Model' @@ -99,6 +100,12 @@ const controlsRef = shallowRef>() const tresCanvasRef = shallowRef() const skyBoxEnvRef = ref>() const dirLightRef = ref>() +const { readRenderTargetRegionAtClientPoint, disposeRenderTarget } = useRenderTargetRegionAtClientPoint({ + getRenderer: () => tresCanvasRef.value?.renderer.instance as WebGLRenderer | undefined, + getScene: () => tresCanvasRef.value?.scene.value, + getCamera: () => camera.value, + getCanvas: () => tresCanvasRef.value?.renderer.instance.domElement, +}) /* * Pinia store definition @@ -194,7 +201,9 @@ onMounted(() => { } }) -onUnmounted(() => {}) +onUnmounted(() => { + disposeRenderTarget() +}) const effectProps = { saturation: 0.3, @@ -291,6 +300,10 @@ defineExpose({ canvasElement: () => { return tresCanvasRef.value?.renderer.instance.domElement }, + camera: () => camera.value, + renderer: () => tresCanvasRef.value?.renderer.instance, + scene: () => modelRef.value?.scene, + readRenderTargetRegionAtClientPoint, }) diff --git a/packages/stage-ui-three/src/composables/hit-test.ts b/packages/stage-ui-three/src/composables/hit-test.ts new file mode 100644 index 000000000..653299cc0 --- /dev/null +++ b/packages/stage-ui-three/src/composables/hit-test.ts @@ -0,0 +1,71 @@ +import type { MaybeRefOrGetter, Ref } from 'vue' + +import type { RenderTargetRegionRead } from './render-target' + +import { toRef } from '@vueuse/core' +import { computed } from 'vue' + +interface HitTestTarget { + readRenderTargetRegionAtClientPoint?: (clientX: number, clientY: number, radius: number) => RenderTargetRegionRead | null +} + +export function useThreeSceneIsTransparentAtPoint( + scene: MaybeRefOrGetter, + pointX: MaybeRefOrGetter, + pointY: MaybeRefOrGetter, + optionsOrThreshold: number | { threshold?: number, regionRadius?: number } = 10, +): Ref { + const sceneRef = toRef(scene) + const xRef = toRef(pointX) + const yRef = toRef(pointY) + + const options = typeof optionsOrThreshold === 'number' + ? { threshold: optionsOrThreshold, regionRadius: 0 } + : optionsOrThreshold + + const threshold = options?.threshold ?? 10 + const radius = Math.max(0, options?.regionRadius ?? 0) + + return computed(() => { + const instance = sceneRef.value + if (!instance || !instance.readRenderTargetRegionAtClientPoint) + return true + + const result = instance.readRenderTargetRegionAtClientPoint(xRef.value, yRef.value, radius) + if (!result) + return true + + const { + data, + readWidth, + readHeight, + startX, + startY, + centerX, + centerY, + scaleX, + scaleY, + } = result + + const radiusSq = radius * radius + for (let y = 0; y < readHeight; y += 1) { + const gy = startY + y + const dy = (gy - centerY) / scaleY + const dySq = dy * dy + + for (let x = 0; x < readWidth; x += 1) { + const gx = startX + x + const dx = (gx - centerX) / scaleX + if (dx * dx + dySq > radiusSq) + continue + + const index = (y * readWidth + x) * 4 + const alpha = data[index + 3] + if (alpha >= threshold) + return false + } + } + + return true + }) +} diff --git a/packages/stage-ui-three/src/composables/render-target.ts b/packages/stage-ui-three/src/composables/render-target.ts new file mode 100644 index 000000000..ee130f3e6 --- /dev/null +++ b/packages/stage-ui-three/src/composables/render-target.ts @@ -0,0 +1,112 @@ +import type { Camera, Scene, WebGLRenderer } from 'three' + +import { clamp } from 'es-toolkit/math' +import { Vector2, WebGLRenderTarget } from 'three' +import { shallowRef } from 'vue' + +export interface RenderTargetRegionRead { + data: Uint8Array + readWidth: number + readHeight: number + startX: number + startY: number + centerX: number + centerY: number + scaleX: number + scaleY: number +} + +export function useRenderTargetRegionAtClientPoint(context: { + getRenderer: () => WebGLRenderer | undefined + getScene: () => Scene | undefined + getCamera: () => Camera | undefined + getCanvas: () => HTMLCanvasElement | undefined +}) { + const renderTargetRef = shallowRef() + const renderTargetSize = new Vector2() + + function ensureRenderTarget(renderer: WebGLRenderer) { + // Match the offscreen target to the current drawing buffer size (DPI + canvas resize). + // Recreate the target when size changes to avoid reading stale or cropped pixels. + renderer.getDrawingBufferSize(renderTargetSize) + const width = Math.max(1, Math.floor(renderTargetSize.x)) + const height = Math.max(1, Math.floor(renderTargetSize.y)) + + if (!renderTargetRef.value || renderTargetRef.value.width !== width || renderTargetRef.value.height !== height) { + // Dispose old target to keep GPU memory in check. + renderTargetRef.value?.dispose() + renderTargetRef.value = new WebGLRenderTarget(width, height, { depthBuffer: false }) + } + + return renderTargetRef.value + } + + function readRenderTargetRegionAtClientPoint(clientX: number, clientY: number, radius: number): RenderTargetRegionRead | null { + const renderer = context.getRenderer() + const scene = context.getScene() + const camera = context.getCamera() + const canvas = context.getCanvas() + if (!renderer || !scene || !camera || !canvas) + return null + + const rect = canvas.getBoundingClientRect() + const xIn = clientX - rect.left + const yIn = clientY - rect.top + const inCanvas = xIn >= 0 && yIn >= 0 && xIn < rect.width && yIn < rect.height + if (!inCanvas) + return null + + const renderTarget = ensureRenderTarget(renderer) + const scaleX = renderTarget.width / rect.width + const scaleY = renderTarget.height / rect.height + if (!Number.isFinite(scaleX) || !Number.isFinite(scaleY)) + return null + + const centerX = Math.floor(xIn * scaleX) + const centerY = Math.floor(renderTarget.height - 1 - yIn * scaleY) + + const radiusX = Math.ceil(radius * scaleX) + const radiusY = Math.ceil(radius * scaleY) + + const startX = clamp(centerX - radiusX, 0, renderTarget.width - 1) + const endX = clamp(centerX + radiusX, 0, renderTarget.width - 1) + const startY = clamp(centerY - radiusY, 0, renderTarget.height - 1) + const endY = clamp(centerY + radiusY, 0, renderTarget.height - 1) + + const readWidth = endX - startX + 1 + const readHeight = endY - startY + 1 + const data = new Uint8Array(readWidth * readHeight * 4) + + const prevTarget = renderer.getRenderTarget() + // Render into our offscreen target so we can read pixels from it. + // Preserve the previous target to avoid breaking the main render pipeline. + renderer.setRenderTarget(renderTarget) + renderer.clear() + renderer.render(scene, camera) + renderer.readRenderTargetPixels(renderTarget, startX, startY, readWidth, readHeight, data) + // Restore the original target so downstream renders keep working. + renderer.setRenderTarget(prevTarget) + + return { + data, + readWidth, + readHeight, + startX, + startY, + centerX, + centerY, + scaleX, + scaleY, + } + } + + function disposeRenderTarget() { + renderTargetRef.value?.dispose() + renderTargetRef.value = undefined + } + + return { + readRenderTargetRegionAtClientPoint, + disposeRenderTarget, + } +} diff --git a/packages/stage-ui-three/src/index.ts b/packages/stage-ui-three/src/index.ts index 09e45f039..8d3b6126d 100644 --- a/packages/stage-ui-three/src/index.ts +++ b/packages/stage-ui-three/src/index.ts @@ -1,3 +1,5 @@ export { default as ThreeScene } from './components/ThreeScene.vue' +export * from './composables/hit-test' +export * from './composables/render-target' export { useModelStore } from './stores/model-store' export * from './utils/vrm-preview' diff --git a/packages/stage-ui/src/components/scenes/Stage.vue b/packages/stage-ui/src/components/scenes/Stage.vue index ee8eacbc9..3114c9a96 100644 --- a/packages/stage-ui/src/components/scenes/Stage.vue +++ b/packages/stage-ui/src/components/scenes/Stage.vue @@ -339,6 +339,13 @@ function canvasElement() { return vrmViewerRef.value?.canvasElement() } +function readRenderTargetRegionAtClientPoint(clientX: number, clientY: number, radius: number) { + if (stageModelRenderer.value !== 'vrm') + return null + + return vrmViewerRef.value?.readRenderTargetRegionAtClientPoint?.(clientX, clientY, radius) ?? null +} + onUnmounted(() => { if (lipSyncLoopId.value) { cancelAnimationFrame(lipSyncLoopId.value) @@ -350,6 +357,7 @@ onUnmounted(() => { defineExpose({ canvasElement, + readRenderTargetRegionAtClientPoint, }) onPlaybackFinished(() => { diff --git a/packages/stage-ui/src/composables/canvas-alpha.ts b/packages/stage-ui/src/composables/canvas-alpha.ts index 68926b838..5c57179dd 100644 --- a/packages/stage-ui/src/composables/canvas-alpha.ts +++ b/packages/stage-ui/src/composables/canvas-alpha.ts @@ -132,7 +132,7 @@ export function useCanvasPixelAtPoint( const scaleY = gl.drawingBufferHeight / height.value const pixelX = Math.floor(xIn * scaleX) // Flip Y; subtract 1 to avoid top-edge off-by-one - const pixelY = Math.floor(gl.drawingBufferHeight - 1 - yIn * scaleY) + const pixelY = Math.floor(gl.drawingBufferHeight + yIn * scaleY) const data = new Uint8Array(4) try { @@ -141,6 +141,7 @@ export function useCanvasPixelAtPoint( catch { return new Uint8Array([0, 0, 0, 0]) } + return data }) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a70534c90..942bbfb00 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -106,7 +106,7 @@ catalogs: specifier: ^8.6.0 version: 8.6.0 es-toolkit: - specifier: ^1.43.0 + specifier: 1.43.0 version: 1.43.0 hono: specifier: 4.11.3 @@ -875,6 +875,12 @@ importers: '@proj-airi/stage-ui': specifier: workspace:^ version: link:../../packages/stage-ui + '@proj-airi/stage-ui-live2d': + specifier: workspace:^ + version: link:../../packages/stage-ui-live2d + '@proj-airi/stage-ui-three': + specifier: workspace:^ + version: link:../../packages/stage-ui-three '@proj-airi/ui': specifier: workspace:^ version: link:../../packages/ui @@ -2600,6 +2606,9 @@ importers: culori: specifier: ^4.0.2 version: 4.0.2 + es-toolkit: + specifier: 'catalog:' + version: 1.43.0 pinia: specifier: ^3.0.4 version: 3.0.4(typescript@5.9.3)(vue@3.5.25(typescript@5.9.3)) diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 60527ffa2..31ac927d4 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -56,7 +56,7 @@ catalog: builder-util-runtime: ^9.3.1 drizzle-valibot: ^0.4.2 embla-carousel-vue: ^8.6.0 - es-toolkit: ^1.43.0 + es-toolkit: 1.43.0 hono: 4.11.3 injeca: ^0.1.5 is-network-error: ^1.3.0