feat(model-driver-mediapipe): integrate MediaPipe as a motion capture tool (#828)
--------- Co-authored-by: Makito <i@maki.to>
This commit is contained in:
Lilia_Chen
@@ -110,3 +110,6 @@ twitter-session.json
|
|||||||
result*
|
result*
|
||||||
# Nix develop outputs
|
# Nix develop outputs
|
||||||
outputs/
|
outputs/
|
||||||
|
|
||||||
|
# MediaPipe task assets
|
||||||
|
packages/model-driver-mediapipe/tasks/assets/*
|
||||||
|
|||||||
@@ -28,6 +28,7 @@
|
|||||||
"@proj-airi/font-cjkfonts-allseto": "workspace:^",
|
"@proj-airi/font-cjkfonts-allseto": "workspace:^",
|
||||||
"@proj-airi/font-xiaolai": "workspace:^",
|
"@proj-airi/font-xiaolai": "workspace:^",
|
||||||
"@proj-airi/i18n": "workspace:^",
|
"@proj-airi/i18n": "workspace:^",
|
||||||
|
"@proj-airi/model-driver-mediapipe": "workspace:^",
|
||||||
"@proj-airi/server-sdk": "workspace:^",
|
"@proj-airi/server-sdk": "workspace:^",
|
||||||
"@proj-airi/stage-ui": "workspace:^",
|
"@proj-airi/stage-ui": "workspace:^",
|
||||||
"@proj-airi/stage-ui-three": "workspace:^",
|
"@proj-airi/stage-ui-three": "workspace:^",
|
||||||
|
|||||||
@@ -0,0 +1,485 @@
|
|||||||
|
<script setup lang="ts">
|
||||||
|
import type { PerceptionState, VrmPoseTargets } from '@proj-airi/model-driver-mediapipe'
|
||||||
|
import type { Vector3Like } from 'three'
|
||||||
|
|
||||||
|
import { createMediaPipeBackend, createMocapEngine, createVrmPoseApplier, drawOverlay, poseToVrmTargets } from '@proj-airi/model-driver-mediapipe'
|
||||||
|
import { ThreeScene } from '@proj-airi/stage-ui-three'
|
||||||
|
import { animations } from '@proj-airi/stage-ui-three/assets/vrm'
|
||||||
|
import { useSettings } from '@proj-airi/stage-ui/stores/settings'
|
||||||
|
import { Checkbox } from '@proj-airi/ui'
|
||||||
|
import { storeToRefs } from 'pinia'
|
||||||
|
import { computed, onMounted, onUnmounted, ref, toRaw, watch } from 'vue'
|
||||||
|
|
||||||
|
const status = ref<'idle' | 'starting' | 'running' | 'error'>('idle')
|
||||||
|
const errorMessage = ref('')
|
||||||
|
const pipelineEnabled = ref(true)
|
||||||
|
const syncingToggleState = ref(false)
|
||||||
|
const ignoreErrorsUntil = ref(0)
|
||||||
|
|
||||||
|
const videoRef = ref<HTMLVideoElement>()
|
||||||
|
const canvasRef = ref<HTMLCanvasElement>()
|
||||||
|
const sceneRef = ref<InstanceType<typeof ThreeScene>>()
|
||||||
|
let stream: MediaStream | undefined
|
||||||
|
let engine: ReturnType<typeof createMocapEngine> | undefined
|
||||||
|
|
||||||
|
// config on the page
|
||||||
|
const config = ref({
|
||||||
|
enabled: {
|
||||||
|
pose: true,
|
||||||
|
hands: true,
|
||||||
|
face: true,
|
||||||
|
},
|
||||||
|
hz: {
|
||||||
|
pose: 30,
|
||||||
|
hands: 30,
|
||||||
|
face: 30,
|
||||||
|
},
|
||||||
|
maxPeople: 1 as const, // Fixed to 1 for simplicity
|
||||||
|
})
|
||||||
|
|
||||||
|
const vrmMapping = ref({
|
||||||
|
flipX: true,
|
||||||
|
flipY: true,
|
||||||
|
flipZ: false,
|
||||||
|
})
|
||||||
|
|
||||||
|
const poseFiltering = ref({
|
||||||
|
minVisibility: 0.5,
|
||||||
|
})
|
||||||
|
|
||||||
|
// MediaPipe assets config
|
||||||
|
const latestState = ref<PerceptionState>()
|
||||||
|
const latestPoseTargets = ref<VrmPoseTargets>()
|
||||||
|
const prevPoseTargets = ref<VrmPoseTargets>()
|
||||||
|
const prevPoseForward = ref<Vector3Like>()
|
||||||
|
|
||||||
|
// VRM pose applier
|
||||||
|
const vrmPoseApplier = createVrmPoseApplier({ alpha: 1 })
|
||||||
|
function onVrmFrame(vrm: Parameters<typeof vrmPoseApplier.applyPoseDirectionsToVrm>[0]) {
|
||||||
|
const targets = latestPoseTargets.value
|
||||||
|
if (!targets)
|
||||||
|
return
|
||||||
|
vrmPoseApplier.applyPoseTargetsToVrm(vrm, targets)
|
||||||
|
}
|
||||||
|
const vrmFrameHook = (vrm: Parameters<typeof vrmPoseApplier.applyPoseDirectionsToVrm>[0]) => onVrmFrame(vrm)
|
||||||
|
|
||||||
|
const settingsStore = useSettings()
|
||||||
|
const { stageModelRenderer, stageModelSelected, stageModelSelectedUrl, stageViewControlsEnabled } = storeToRefs(settingsStore)
|
||||||
|
|
||||||
|
// Snapshot summary of the running state
|
||||||
|
const summary = computed(() => {
|
||||||
|
const enabled = Object.entries(config.value.enabled)
|
||||||
|
.filter(([, v]) => v)
|
||||||
|
.map(([k]) => k)
|
||||||
|
.join(', ') || 'none'
|
||||||
|
|
||||||
|
const fps = latestState.value?.quality.fps
|
||||||
|
const latency = latestState.value?.quality.latencyMs
|
||||||
|
const dropped = latestState.value?.quality.droppedFrames
|
||||||
|
|
||||||
|
return [
|
||||||
|
`enabled: ${enabled}`,
|
||||||
|
`hz: pose ${config.value.hz.pose}, hands ${config.value.hz.hands}, face ${config.value.hz.face}`,
|
||||||
|
fps != null ? `fps ${fps.toFixed(1)}` : null,
|
||||||
|
latency != null ? `latency ${latency.toFixed(1)}ms` : null,
|
||||||
|
dropped != null ? `dropped ${dropped}` : null,
|
||||||
|
].filter(Boolean).join(' | ')
|
||||||
|
})
|
||||||
|
|
||||||
|
const poseVisibilityDebug = computed(() => {
|
||||||
|
const pose = latestState.value?.pose
|
||||||
|
const lm = pose?.landmarks2d
|
||||||
|
const world = pose?.worldLandmarks
|
||||||
|
if (!lm?.length)
|
||||||
|
return 'pose: (no landmarks)'
|
||||||
|
|
||||||
|
const pick = (i: number) => {
|
||||||
|
const v = lm[i]?.visibility
|
||||||
|
return v == null || !Number.isFinite(v) ? 'na' : v.toFixed(2)
|
||||||
|
}
|
||||||
|
|
||||||
|
const withVis = lm.filter(p => p.visibility != null && Number.isFinite(p.visibility)).length
|
||||||
|
const worldWithVis = world?.filter(p => p.visibility != null && Number.isFinite(p.visibility)).length ?? 0
|
||||||
|
return [
|
||||||
|
`pose 2d vis ${withVis}/${lm.length}`,
|
||||||
|
`pose 3d vis ${worldWithVis}/${world?.length ?? 0}`,
|
||||||
|
`LS ${pick(11)} RS ${pick(12)} LE ${pick(13)} RE ${pick(14)}`,
|
||||||
|
`LW ${pick(15)} RW ${pick(16)} LH ${pick(23)} RH ${pick(24)}`,
|
||||||
|
].join(' | ')
|
||||||
|
})
|
||||||
|
|
||||||
|
// Start camera and pipeline
|
||||||
|
async function startCamera() {
|
||||||
|
if (status.value === 'starting' || status.value === 'running')
|
||||||
|
return
|
||||||
|
|
||||||
|
status.value = 'starting'
|
||||||
|
errorMessage.value = ''
|
||||||
|
|
||||||
|
try {
|
||||||
|
stop()
|
||||||
|
stream = await navigator.mediaDevices.getUserMedia({ video: true, audio: false })
|
||||||
|
if (!videoRef.value)
|
||||||
|
throw new Error('video element not mounted')
|
||||||
|
|
||||||
|
videoRef.value.srcObject = stream
|
||||||
|
await videoRef.value.play()
|
||||||
|
|
||||||
|
status.value = 'running'
|
||||||
|
await startPipeline()
|
||||||
|
}
|
||||||
|
catch (err) {
|
||||||
|
status.value = 'error'
|
||||||
|
errorMessage.value = err instanceof Error ? err.message : String(err)
|
||||||
|
console.error('Failed to start camera or pipeline:', err)
|
||||||
|
|
||||||
|
syncingToggleState.value = true
|
||||||
|
pipelineEnabled.value = false
|
||||||
|
syncingToggleState.value = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function startPipeline() {
|
||||||
|
if (!videoRef.value)
|
||||||
|
return
|
||||||
|
if (engine)
|
||||||
|
return
|
||||||
|
|
||||||
|
const backend = createMediaPipeBackend()
|
||||||
|
engine = createMocapEngine(backend, toRaw(config.value))
|
||||||
|
await engine.init()
|
||||||
|
|
||||||
|
engine.start(
|
||||||
|
{ getFrame: () => videoRef.value as HTMLVideoElement },
|
||||||
|
(state) => {
|
||||||
|
latestState.value = state
|
||||||
|
const axis = {
|
||||||
|
x: vrmMapping.value.flipX ? -1 : 1,
|
||||||
|
y: vrmMapping.value.flipY ? -1 : 1,
|
||||||
|
z: vrmMapping.value.flipZ ? -1 : 1,
|
||||||
|
} as const
|
||||||
|
|
||||||
|
const poseTargets = (config.value.enabled.pose && state.pose?.worldLandmarks?.length)
|
||||||
|
? poseToVrmTargets(state.pose, {
|
||||||
|
axis,
|
||||||
|
confidence: { minVisibility: poseFiltering.value.minVisibility },
|
||||||
|
stabilize: {
|
||||||
|
previousTargets: prevPoseTargets.value,
|
||||||
|
previousForward: prevPoseForward.value,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
: {}
|
||||||
|
|
||||||
|
const hasAny = Object.keys(poseTargets).length > 0
|
||||||
|
latestPoseTargets.value = hasAny ? poseTargets : undefined
|
||||||
|
if (hasAny) {
|
||||||
|
prevPoseTargets.value = poseTargets
|
||||||
|
const derivedForward = poseTargets.hips?.pole ?? poseTargets.spine?.pole
|
||||||
|
if (derivedForward)
|
||||||
|
prevPoseForward.value = derivedForward
|
||||||
|
}
|
||||||
|
|
||||||
|
const canvas = canvasRef.value
|
||||||
|
const video = videoRef.value
|
||||||
|
if (!canvas || !video)
|
||||||
|
return
|
||||||
|
|
||||||
|
const w = video.videoWidth || 640
|
||||||
|
const h = video.videoHeight || 480
|
||||||
|
if (canvas.width !== w)
|
||||||
|
canvas.width = w
|
||||||
|
if (canvas.height !== h)
|
||||||
|
canvas.height = h
|
||||||
|
|
||||||
|
const ctx = canvas.getContext('2d')
|
||||||
|
if (!ctx)
|
||||||
|
return
|
||||||
|
|
||||||
|
drawOverlay(ctx, state, config.value.enabled)
|
||||||
|
},
|
||||||
|
{
|
||||||
|
onError: (err) => {
|
||||||
|
if (!pipelineEnabled.value || Date.now() < ignoreErrorsUntil.value) {
|
||||||
|
console.warn('Ignored pipeline error during stop:', err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
errorMessage.value = err instanceof Error ? err.message : String(err)
|
||||||
|
// Ensure resources are released, but keep the error status visible.
|
||||||
|
stop()
|
||||||
|
status.value = 'error'
|
||||||
|
console.error('Pipeline error:', err)
|
||||||
|
},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
function stopPipeline() {
|
||||||
|
engine?.stop()
|
||||||
|
engine = undefined
|
||||||
|
latestState.value = undefined
|
||||||
|
latestPoseTargets.value = undefined
|
||||||
|
prevPoseTargets.value = undefined
|
||||||
|
prevPoseForward.value = undefined
|
||||||
|
}
|
||||||
|
|
||||||
|
function stop() {
|
||||||
|
// During stop, MediaPipe may still be processing an in-flight frame; ignore transient errors.
|
||||||
|
ignoreErrorsUntil.value = Date.now() + 1500
|
||||||
|
canvasRef.value?.getContext('2d')?.clearRect(0, 0, canvasRef.value.width, canvasRef.value.height)
|
||||||
|
stopPipeline()
|
||||||
|
|
||||||
|
try {
|
||||||
|
stream?.getTracks().forEach(t => t.stop())
|
||||||
|
}
|
||||||
|
catch {}
|
||||||
|
|
||||||
|
stream = undefined
|
||||||
|
|
||||||
|
if (videoRef.value)
|
||||||
|
videoRef.value.srcObject = null
|
||||||
|
|
||||||
|
status.value = 'idle'
|
||||||
|
}
|
||||||
|
|
||||||
|
watch(config, (val) => {
|
||||||
|
engine?.updateConfig(toRaw(val))
|
||||||
|
}, { deep: true })
|
||||||
|
|
||||||
|
watch(sceneRef, (scene, prev) => {
|
||||||
|
prev?.setVrmFrameHook(undefined)
|
||||||
|
scene?.setVrmFrameHook(vrmFrameHook)
|
||||||
|
}, { immediate: true })
|
||||||
|
|
||||||
|
watch(pipelineEnabled, async (enabled) => {
|
||||||
|
if (syncingToggleState.value)
|
||||||
|
return
|
||||||
|
|
||||||
|
if (enabled)
|
||||||
|
await startCamera()
|
||||||
|
else
|
||||||
|
stop()
|
||||||
|
})
|
||||||
|
|
||||||
|
onMounted(() => {
|
||||||
|
// Ensure a VRM model is selected for the viewer (preserve existing selection if already VRM).
|
||||||
|
const needsFallback = !stageModelSelectedUrl.value || stageModelRenderer.value !== 'vrm'
|
||||||
|
if (needsFallback)
|
||||||
|
stageModelSelected.value = 'preset-vrm-1'
|
||||||
|
|
||||||
|
settingsStore.updateStageModel().catch((err) => {
|
||||||
|
console.error('Failed to init VRM model:', err)
|
||||||
|
})
|
||||||
|
|
||||||
|
// Autostart for convenience
|
||||||
|
if (pipelineEnabled.value)
|
||||||
|
startCamera()
|
||||||
|
})
|
||||||
|
|
||||||
|
onUnmounted(() => {
|
||||||
|
sceneRef.value?.setVrmFrameHook(undefined)
|
||||||
|
stop()
|
||||||
|
})
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<template>
|
||||||
|
<div :class="['p-4', 'space-y-4']">
|
||||||
|
<div>
|
||||||
|
<div :class="['text-lg', 'font-600']">
|
||||||
|
MediaPipe Workshop Playground
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- Top config -->
|
||||||
|
<div :class="['rounded-2xl', 'border', 'border-neutral-300/40', 'dark:border-neutral-700/40', 'p-3', 'space-y-3']">
|
||||||
|
<div :class="['flex', 'items-start', 'justify-between', 'gap-3', 'flex-wrap']">
|
||||||
|
<div :class="['space-y-1']">
|
||||||
|
<div :class="['font-600']">
|
||||||
|
Config
|
||||||
|
</div>
|
||||||
|
<div :class="['text-xs', 'text-neutral-500']">
|
||||||
|
{{ summary }}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<label :class="['flex', 'items-center', 'gap-3']">
|
||||||
|
<div :class="['text-sm', 'text-neutral-600', 'dark:text-neutral-300']">
|
||||||
|
{{ pipelineEnabled ? 'Running' : 'Stopped' }}
|
||||||
|
</div>
|
||||||
|
<Checkbox v-model="pipelineEnabled" />
|
||||||
|
</label>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div :class="['grid', 'gap-3', 'lg:grid-cols-3']">
|
||||||
|
<div :class="['flex', 'items-center', 'justify-between', 'gap-3']">
|
||||||
|
<label :class="['flex', 'items-center', 'gap-2', 'text-sm']">
|
||||||
|
<input v-model="config.enabled.pose" type="checkbox">
|
||||||
|
Pose
|
||||||
|
</label>
|
||||||
|
<label :class="['flex', 'items-center', 'gap-2']">
|
||||||
|
<div :class="['text-xs', 'text-neutral-500']">
|
||||||
|
Hz
|
||||||
|
</div>
|
||||||
|
<input
|
||||||
|
v-model.number="config.hz.pose"
|
||||||
|
type="number"
|
||||||
|
min="1"
|
||||||
|
max="60"
|
||||||
|
:class="['w-24', 'rounded-lg', 'border', 'border-neutral-300/60', 'bg-white', 'px-2', 'py-1', 'text-sm', 'dark:bg-neutral-900/60', 'dark:border-neutral-700/60']"
|
||||||
|
>
|
||||||
|
</label>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div :class="['flex', 'items-center', 'justify-between', 'gap-3']">
|
||||||
|
<label :class="['flex', 'items-center', 'gap-2', 'text-sm']">
|
||||||
|
<input v-model="config.enabled.hands" type="checkbox">
|
||||||
|
Hands
|
||||||
|
</label>
|
||||||
|
<label :class="['flex', 'items-center', 'gap-2']">
|
||||||
|
<div :class="['text-xs', 'text-neutral-500']">
|
||||||
|
Hz
|
||||||
|
</div>
|
||||||
|
<input
|
||||||
|
v-model.number="config.hz.hands"
|
||||||
|
type="number"
|
||||||
|
min="1"
|
||||||
|
max="60"
|
||||||
|
:class="['w-24', 'rounded-lg', 'border', 'border-neutral-300/60', 'bg-white', 'px-2', 'py-1', 'text-sm', 'dark:bg-neutral-900/60', 'dark:border-neutral-700/60']"
|
||||||
|
>
|
||||||
|
</label>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div :class="['flex', 'items-center', 'justify-between', 'gap-3']">
|
||||||
|
<label :class="['flex', 'items-center', 'gap-2', 'text-sm']">
|
||||||
|
<input v-model="config.enabled.face" type="checkbox">
|
||||||
|
Face
|
||||||
|
</label>
|
||||||
|
<label :class="['flex', 'items-center', 'gap-2']">
|
||||||
|
<div :class="['text-xs', 'text-neutral-500']">
|
||||||
|
Hz
|
||||||
|
</div>
|
||||||
|
<input
|
||||||
|
v-model.number="config.hz.face"
|
||||||
|
type="number"
|
||||||
|
min="1"
|
||||||
|
max="60"
|
||||||
|
:class="['w-24', 'rounded-lg', 'border', 'border-neutral-300/60', 'bg-white', 'px-2', 'py-1', 'text-sm', 'dark:bg-neutral-900/60', 'dark:border-neutral-700/60']"
|
||||||
|
>
|
||||||
|
</label>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div :class="['flex', 'items-center', 'justify-between', 'gap-4', 'flex-wrap']">
|
||||||
|
<div :class="['text-sm', 'text-neutral-600', 'dark:text-neutral-300']">
|
||||||
|
VRM Mapping
|
||||||
|
</div>
|
||||||
|
<div :class="['flex', 'items-center', 'gap-6', 'flex-wrap']">
|
||||||
|
<label :class="['flex', 'items-center', 'gap-3']">
|
||||||
|
<div :class="['text-sm', 'text-neutral-600', 'dark:text-neutral-300']">
|
||||||
|
Flip X
|
||||||
|
</div>
|
||||||
|
<Checkbox v-model="vrmMapping.flipX" />
|
||||||
|
</label>
|
||||||
|
<label :class="['flex', 'items-center', 'gap-3']">
|
||||||
|
<div :class="['text-sm', 'text-neutral-600', 'dark:text-neutral-300']">
|
||||||
|
Flip Y
|
||||||
|
</div>
|
||||||
|
<Checkbox v-model="vrmMapping.flipY" />
|
||||||
|
</label>
|
||||||
|
<label :class="['flex', 'items-center', 'gap-3']">
|
||||||
|
<div :class="['text-sm', 'text-neutral-600', 'dark:text-neutral-300']">
|
||||||
|
Flip Z
|
||||||
|
</div>
|
||||||
|
<Checkbox v-model="vrmMapping.flipZ" />
|
||||||
|
</label>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div :class="['flex', 'items-center', 'justify-between', 'gap-4', 'flex-wrap']">
|
||||||
|
<div :class="['text-sm', 'text-neutral-600', 'dark:text-neutral-300']">
|
||||||
|
Pose Filtering
|
||||||
|
</div>
|
||||||
|
<label :class="['flex', 'items-center', 'gap-3']">
|
||||||
|
<div :class="['text-sm', 'text-neutral-600', 'dark:text-neutral-300']">
|
||||||
|
Min Visibility
|
||||||
|
</div>
|
||||||
|
<input
|
||||||
|
v-model.number="poseFiltering.minVisibility"
|
||||||
|
type="number"
|
||||||
|
min="0"
|
||||||
|
max="1"
|
||||||
|
step="0.05"
|
||||||
|
:class="['w-24', 'rounded-lg', 'border', 'border-neutral-300/60', 'bg-white', 'px-2', 'py-1', 'text-sm', 'dark:bg-neutral-900/60', 'dark:border-neutral-700/60']"
|
||||||
|
>
|
||||||
|
</label>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div :class="['text-xs', 'text-neutral-500']">
|
||||||
|
Note: `@mediapipe/tasks-vision` runs sync and may block the main thread. This workshop drops frames when busy to keep UI responsive.
|
||||||
|
</div>
|
||||||
|
<div :class="['text-xs', 'text-neutral-500', 'break-words']">
|
||||||
|
{{ poseVisibilityDebug }}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<!-- Main: camera + VRM -->
|
||||||
|
<div :class="['grid', 'gap-4', 'lg:grid-cols-2']">
|
||||||
|
<div :class="['rounded-2xl', 'border', 'border-neutral-300/40', 'dark:border-neutral-700/40', 'overflow-hidden']">
|
||||||
|
<div :class="['relative', 'aspect-video', 'bg-black']">
|
||||||
|
<video
|
||||||
|
ref="videoRef"
|
||||||
|
muted
|
||||||
|
playsinline
|
||||||
|
:class="['absolute', 'inset-0', 'h-full', 'w-full', 'object-cover', 'opacity-70']"
|
||||||
|
/>
|
||||||
|
<canvas
|
||||||
|
ref="canvasRef"
|
||||||
|
:class="['absolute', 'inset-0', 'h-full', 'w-full', 'object-cover', 'opacity-70']"
|
||||||
|
/>
|
||||||
|
|
||||||
|
<div
|
||||||
|
:class="[
|
||||||
|
'absolute',
|
||||||
|
'left-3',
|
||||||
|
'top-3',
|
||||||
|
'rounded-lg',
|
||||||
|
'bg-black/50',
|
||||||
|
'px-2',
|
||||||
|
'py-1',
|
||||||
|
'text-xs',
|
||||||
|
'text-white',
|
||||||
|
'backdrop-blur',
|
||||||
|
]"
|
||||||
|
>
|
||||||
|
<div>Status: {{ status }}</div>
|
||||||
|
<div v-if="status === 'error'" :class="['text-red-300']">
|
||||||
|
{{ errorMessage }}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div :class="['rounded-2xl', 'border', 'border-neutral-300/40', 'dark:border-neutral-700/40', 'overflow-hidden']">
|
||||||
|
<div :class="['h-full', 'min-h-80']">
|
||||||
|
<ThreeScene
|
||||||
|
v-if="stageModelRenderer === 'vrm'"
|
||||||
|
ref="sceneRef"
|
||||||
|
:model-src="stageModelSelectedUrl"
|
||||||
|
:idle-animation="animations.idleLoop.toString()"
|
||||||
|
:show-axes="stageViewControlsEnabled"
|
||||||
|
:paused="false"
|
||||||
|
@error="console.error"
|
||||||
|
/>
|
||||||
|
<div v-else :class="['p-4', 'text-sm', 'text-red-500']">
|
||||||
|
请选择 VRM 模型（当前模型类型不支持）。
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</template>
|
||||||
|
|
||||||
|
<route lang="yaml">
|
||||||
|
meta:
|
||||||
|
layout: plain
|
||||||
|
</route>
|
||||||
@@ -80,6 +80,12 @@ const menu = computed(() => [
|
|||||||
icon: 'i-solar:sledgehammer-bold-duotone',
|
icon: 'i-solar:sledgehammer-bold-duotone',
|
||||||
to: '/devtools/performance-playground',
|
to: '/devtools/performance-playground',
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
title: 'MediaPipe Workshop',
|
||||||
|
description: 'Single-person mocap playground (MediaPipe backend) with scheduling knobs',
|
||||||
|
icon: 'i-solar:sledgehammer-bold-duotone',
|
||||||
|
to: '/devtools/model-driver-mediapipe',
|
||||||
|
},
|
||||||
])
|
])
|
||||||
</script>
|
</script>
|
||||||
|
|
||||||
|
|||||||
@@ -162,6 +162,7 @@ words:
|
|||||||
- matchall
|
- matchall
|
||||||
- mdit
|
- mdit
|
||||||
- mediabunny
|
- mediabunny
|
||||||
|
- mediapipe
|
||||||
- MeshToonMaterial
|
- MeshToonMaterial
|
||||||
- micvad
|
- micvad
|
||||||
- mineflayer
|
- mineflayer
|
||||||
|
|||||||
@@ -0,0 +1,35 @@
|
|||||||
|
# model-driver-mediapipe Agent Notes
|
||||||
|
|
||||||
|
Scope: `packages/model-driver-mediapipe/**`
|
||||||
|
|
||||||
|
## Intent
|
||||||
|
|
||||||
|
This package is an experimental single-person mocap pipeline for stage-web devtools.
|
||||||
|
|
||||||
|
`camera frame` → `@mediapipe/tasks-vision` → `PerceptionState` → `overlay`
|
||||||
|
|
||||||
|
## Style / Conventions
|
||||||
|
|
||||||
|
- Prefer functional programming (FP) and pure functions where practical.
|
||||||
|
- Use factory functions + closures for stateful modules (example: `createMocapEngine()`).
|
||||||
|
- Avoid classes unless extending browser APIs or required by external libraries.
|
||||||
|
- Keep the backend boundary clean:
|
||||||
|
- Engine/scheduler should not import `@mediapipe/tasks-vision`.
|
||||||
|
- MediaPipe specifics live under `src/backends/`.
|
||||||
|
- Keep types stable and narrow:
|
||||||
|
- Stage consumers depend on `src/types.ts` as the contract.
|
||||||
|
- New fields should be optional and backwards compatible.
|
||||||
|
|
||||||
|
## Key Files
|
||||||
|
|
||||||
|
- `src/types.ts`: middle-layer contract (`PerceptionState`, config types)
|
||||||
|
- `src/engine.ts`: scheduling + dropped-frame policy + partial merge (FP)
|
||||||
|
- `src/backends/mediapipe.ts`: MediaPipe Tasks Vision adapter (sync `detectForVideo`)
|
||||||
|
- `src/overlay.ts`: debug overlay renderer (points + connectors)
|
||||||
|
- `references/tasks-vision-api.md`: minimal upstream API notes
|
||||||
|
|
||||||
|
## Performance Notes
|
||||||
|
|
||||||
|
- `detectForVideo()` is synchronous; avoid blocking UI:
|
||||||
|
- Engine drops frames when backend reports `isBusy()`.
|
||||||
|
- Scheduler controls per-task rates (`hz`) to cap work.
|
||||||
@@ -0,0 +1,35 @@
|
|||||||
|
# @proj-airi/model-driver-mediapipe
|
||||||
|
|
||||||
|
Single-person motion capture workshop package.
|
||||||
|
|
||||||
|
**Goal**
|
||||||
|
|
||||||
|
Provide a minimal closed loop that stage-web can consume:
|
||||||
|
|
||||||
|
`camera frame` → `MediaPipe Tasks Vision` → `PerceptionState` → `canvas overlay`
|
||||||
|
|
||||||
|
**Where to try it**
|
||||||
|
|
||||||
|
- Devtools page: `apps/stage-web/src/pages/devtools/model-driver-mediapipe.vue`
|
||||||
|
- Menu entry: Settings → System → Developer → “MediaPipe Workshop”
|
||||||
|
|
||||||
|
**Key files**
|
||||||
|
|
||||||
|
- `packages/model-driver-mediapipe/src/types.ts`: middle-layer contract (`PerceptionState`)
|
||||||
|
- `packages/model-driver-mediapipe/src/engine.ts`: scheduler + dropped-frame policy + state merge
|
||||||
|
- `packages/model-driver-mediapipe/src/backends/mediapipe.ts`: `@mediapipe/tasks-vision` integration
|
||||||
|
- `packages/model-driver-mediapipe/src/utils/overlay.ts`: canvas overlay renderer
|
||||||
|
|
||||||
|
**Backend assumptions**
|
||||||
|
|
||||||
|
- Single person (`maxPeople: 1`)
|
||||||
|
- Running mode: `VIDEO`
|
||||||
|
- Landmarks are normalized (`x`/`y` in `[0..1]`) and drawn onto the overlay canvas
|
||||||
|
|
||||||
|
**Docs**
|
||||||
|
|
||||||
|
Keep upstream docs/snippets in `packages/model-driver-mediapipe/references/`.
|
||||||
|
|
||||||
|
The minimal API surface this package uses is summarized in:
|
||||||
|
|
||||||
|
- `packages/model-driver-mediapipe/references/tasks-vision-api.md`
|
||||||
@@ -0,0 +1,36 @@
|
|||||||
|
{
|
||||||
|
"name": "@proj-airi/model-driver-mediapipe",
|
||||||
|
"type": "module",
|
||||||
|
"private": true,
|
||||||
|
"description": "MediaPipe motion capture workshop (experimental) for Project AIRI",
|
||||||
|
"author": {
|
||||||
|
"name": "Moeru AI Project AIRI Team",
|
||||||
|
"email": "airi@moeru.ai",
|
||||||
|
"url": "https://github.com/moeru-ai"
|
||||||
|
},
|
||||||
|
"license": "MIT",
|
||||||
|
"repository": {
|
||||||
|
"type": "git",
|
||||||
|
"url": "https://github.com/moeru-ai/airi.git",
|
||||||
|
"directory": "packages/model-driver-mediapipe"
|
||||||
|
},
|
||||||
|
"exports": {
|
||||||
|
".": "./src/index.ts",
|
||||||
|
"./types": "./src/types.ts"
|
||||||
|
},
|
||||||
|
"scripts": {
|
||||||
|
"typecheck": "tsc --noEmit",
|
||||||
|
"postinstall": "tsx ./tasks/prepare-tasks.ts"
|
||||||
|
},
|
||||||
|
"dependencies": {
|
||||||
|
"@mediapipe/tasks-vision": "^0.10.0",
|
||||||
|
"@pixiv/three-vrm": "^3.4.4",
|
||||||
|
"es-toolkit": "catalog:",
|
||||||
|
"three": "^0.182.0"
|
||||||
|
},
|
||||||
|
"devDependencies": {
|
||||||
|
"@types/three": "^0.182.0",
|
||||||
|
"ofetch": "catalog:",
|
||||||
|
"tsx": "catalog:"
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
# References
|
||||||
|
|
||||||
|
Put MediaPipe Tasks Vision API docs / snippets / notes here so the implementation can follow the exact API surface you’re using.
|
||||||
@@ -0,0 +1,44 @@
|
|||||||
|
# MediaPipe Tasks Vision (minimal API notes)
|
||||||
|
|
||||||
|
This package targets `@mediapipe/tasks-vision` (web, JS/TS).
|
||||||
|
|
||||||
|
## Core init
|
||||||
|
|
||||||
|
```ts
|
||||||
|
import { FilesetResolver, PoseLandmarker } from '@mediapipe/tasks-vision'
|
||||||
|
|
||||||
|
const vision = await FilesetResolver.forVisionTasks(wasmRoot)
|
||||||
|
const pose = await PoseLandmarker.createFromOptions(vision, {
|
||||||
|
baseOptions: { modelAssetPath: poseModelUrl },
|
||||||
|
runningMode: 'VIDEO',
|
||||||
|
numPoses: 1,
|
||||||
|
})
|
||||||
|
```
|
||||||
|
|
||||||
|
## Video inference
|
||||||
|
|
||||||
|
MediaPipe Tasks Vision `detectForVideo()` runs synchronously and can block the main thread.
|
||||||
|
|
||||||
|
```ts
|
||||||
|
const nowMs = performance.now()
|
||||||
|
const res = pose.detectForVideo(videoEl, nowMs)
|
||||||
|
```
|
||||||
|
|
||||||
|
## Result shapes (single-person usage)
|
||||||
|
|
||||||
|
### Pose
|
||||||
|
|
||||||
|
- `res.landmarks`: `NormalizedLandmark[][]` (take `[0]` for single person)
|
||||||
|
- each landmark includes `{ x, y, z, visibility?, presence? }` with `x/y` in `[0..1]`
|
||||||
|
|
||||||
|
### Hands
|
||||||
|
|
||||||
|
- `res.landmarks`: `NormalizedLandmark[][]` (each entry is 21 landmarks for one hand)
|
||||||
|
- `res.handedness`: `Category[][]` aligned with `landmarks`
|
||||||
|
- `handedness[i][0].categoryName` is typically `'Left' | 'Right'`
|
||||||
|
- `handedness[i][0].score` is confidence
|
||||||
|
|
||||||
|
### Face (optional)
|
||||||
|
|
||||||
|
- `res.faceLandmarks`: `NormalizedLandmark[][]` (468 landmarks, heavy)
|
||||||
|
- For the workshop we treat this as presence-only by default.
|
||||||
@@ -0,0 +1,158 @@
|
|||||||
|
import type {
|
||||||
|
Category,
|
||||||
|
FaceLandmarker,
|
||||||
|
FaceLandmarkerResult,
|
||||||
|
HandLandmarker,
|
||||||
|
HandLandmarkerResult,
|
||||||
|
Landmark,
|
||||||
|
NormalizedLandmark,
|
||||||
|
PoseLandmarker,
|
||||||
|
PoseLandmarkerResult,
|
||||||
|
} from '@mediapipe/tasks-vision'
|
||||||
|
|
||||||
|
import type { MocapBackend, MocapConfig, MocapJob, PerceptionPartial, VisionTaskModule, VisionTaskWasmFileset } from '../types'
|
||||||
|
|
||||||
|
import { Semaphore } from 'es-toolkit'
|
||||||
|
|
||||||
|
import { visionTaskAssets, visionTaskWasmRoot } from '../../tasks/tasks'
|
||||||
|
|
||||||
|
export function createMediaPipeBackend(): MocapBackend {
|
||||||
|
const semaphore = new Semaphore(1)
|
||||||
|
let busy = false
|
||||||
|
let config: MocapConfig | undefined
|
||||||
|
let tasksVision: VisionTaskModule | undefined
|
||||||
|
let vision: VisionTaskWasmFileset | undefined
|
||||||
|
|
||||||
|
let poseLandmarker: PoseLandmarker | undefined
|
||||||
|
let handLandmarker: HandLandmarker | undefined
|
||||||
|
let faceLandmarker: FaceLandmarker | undefined
|
||||||
|
|
||||||
|
async function init(nextConfig: MocapConfig) {
|
||||||
|
config = nextConfig
|
||||||
|
|
||||||
|
if (!tasksVision)
|
||||||
|
tasksVision = await import('@mediapipe/tasks-vision')
|
||||||
|
|
||||||
|
if (!vision) {
|
||||||
|
const { FilesetResolver } = tasksVision
|
||||||
|
vision = await FilesetResolver.forVisionTasks(visionTaskWasmRoot)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function isBusy() {
|
||||||
|
return busy
|
||||||
|
}
|
||||||
|
|
||||||
|
async function ensurePoseLandmarker() {
|
||||||
|
if (poseLandmarker)
|
||||||
|
return poseLandmarker
|
||||||
|
|
||||||
|
const { PoseLandmarker } = tasksVision!
|
||||||
|
poseLandmarker = await PoseLandmarker.createFromOptions(vision!, {
|
||||||
|
baseOptions: { modelAssetPath: visionTaskAssets.pose },
|
||||||
|
runningMode: 'VIDEO',
|
||||||
|
numPoses: 1,
|
||||||
|
})
|
||||||
|
|
||||||
|
return poseLandmarker
|
||||||
|
}
|
||||||
|
|
||||||
|
async function ensureHandLandmarker() {
|
||||||
|
if (handLandmarker)
|
||||||
|
return handLandmarker
|
||||||
|
|
||||||
|
const { HandLandmarker } = tasksVision!
|
||||||
|
handLandmarker = await HandLandmarker.createFromOptions(vision!, {
|
||||||
|
baseOptions: { modelAssetPath: visionTaskAssets.hands },
|
||||||
|
runningMode: 'VIDEO',
|
||||||
|
numHands: 2,
|
||||||
|
})
|
||||||
|
|
||||||
|
return handLandmarker
|
||||||
|
}
|
||||||
|
|
||||||
|
async function ensureFaceLandmarker() {
|
||||||
|
if (faceLandmarker)
|
||||||
|
return faceLandmarker
|
||||||
|
|
||||||
|
const { FaceLandmarker } = tasksVision!
|
||||||
|
faceLandmarker = await FaceLandmarker.createFromOptions(vision!, {
|
||||||
|
baseOptions: { modelAssetPath: visionTaskAssets.face },
|
||||||
|
runningMode: 'VIDEO',
|
||||||
|
numFaces: 1,
|
||||||
|
})
|
||||||
|
|
||||||
|
return faceLandmarker
|
||||||
|
}
|
||||||
|
|
||||||
|
async function run(frame: TexImageSource, jobs: MocapJob[], nowMs: number): Promise<PerceptionPartial> {
|
||||||
|
if (!config)
|
||||||
|
throw new Error('MediaPipe backend not initialized (call init() first)')
|
||||||
|
|
||||||
|
await semaphore.acquire()
|
||||||
|
busy = true
|
||||||
|
try {
|
||||||
|
const partial: PerceptionPartial = {}
|
||||||
|
|
||||||
|
for (const job of jobs) {
|
||||||
|
if (!config.enabled[job])
|
||||||
|
continue
|
||||||
|
|
||||||
|
if (job === 'pose') {
|
||||||
|
const landmarker = await ensurePoseLandmarker()
|
||||||
|
const res: PoseLandmarkerResult = landmarker.detectForVideo(frame, nowMs)
|
||||||
|
const firstPose: NormalizedLandmark[] = res.landmarks[0] ?? []
|
||||||
|
const firstWorld: Landmark[] = res.worldLandmarks[0] ?? []
|
||||||
|
partial.pose = {
|
||||||
|
landmarks2d: firstPose,
|
||||||
|
worldLandmarks: firstWorld.map(p => ({
|
||||||
|
x: p.x,
|
||||||
|
y: p.y,
|
||||||
|
z: p.z,
|
||||||
|
visibility: p.visibility,
|
||||||
|
})),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else if (job === 'hands') {
|
||||||
|
const landmarker = await ensureHandLandmarker()
|
||||||
|
const res: HandLandmarkerResult = landmarker.detectForVideo(frame, nowMs)
|
||||||
|
const landmarks: NormalizedLandmark[][] = res.landmarks ?? []
|
||||||
|
const handedness: Category[][] = res.handedness ?? []
|
||||||
|
|
||||||
|
partial.hands = landmarks.map((lm, i) => {
|
||||||
|
const mostLikelyCategory = handedness[i]?.[0]
|
||||||
|
const categoryName = mostLikelyCategory?.categoryName
|
||||||
|
const score = mostLikelyCategory?.score
|
||||||
|
const handed = categoryName === 'Left' || categoryName === 'Right' ? categoryName : 'Right'
|
||||||
|
return {
|
||||||
|
handedness: handed,
|
||||||
|
landmarks2d: lm,
|
||||||
|
score,
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
else if (job === 'face') {
|
||||||
|
const landmarker = await ensureFaceLandmarker()
|
||||||
|
const res: FaceLandmarkerResult = landmarker.detectForVideo(frame, nowMs)
|
||||||
|
const firstFace: NormalizedLandmark[] = res.faceLandmarks?.[0] ?? []
|
||||||
|
partial.face = {
|
||||||
|
hasFace: firstFace.length > 0,
|
||||||
|
landmarks2d: firstFace,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return partial
|
||||||
|
}
|
||||||
|
finally {
|
||||||
|
busy = false
|
||||||
|
semaphore.release()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
init,
|
||||||
|
isBusy,
|
||||||
|
run,
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,156 @@
|
|||||||
|
import type { FrameSource, MocapBackend, MocapConfig, MocapEngine, MocapJob, PerceptionPartial, PerceptionState } from './types'
|
||||||
|
|
||||||
|
export function createStats() {
|
||||||
|
let lastTs = 0
|
||||||
|
let smoothedFps = 0
|
||||||
|
|
||||||
|
function tick(nowMs: number) {
|
||||||
|
if (!lastTs) {
|
||||||
|
lastTs = nowMs
|
||||||
|
smoothedFps = 0
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
const dt = nowMs - lastTs
|
||||||
|
lastTs = nowMs
|
||||||
|
|
||||||
|
if (dt <= 0)
|
||||||
|
return smoothedFps
|
||||||
|
|
||||||
|
const fps = 1000 / dt
|
||||||
|
smoothedFps = smoothedFps ? (smoothedFps * 0.9 + fps * 0.1) : fps
|
||||||
|
|
||||||
|
return smoothedFps
|
||||||
|
}
|
||||||
|
|
||||||
|
return { tick }
|
||||||
|
}
|
||||||
|
|
||||||
|
export function createScheduler(initialConfig: MocapConfig) {
|
||||||
|
let config = initialConfig
|
||||||
|
const lastRun: Record<MocapJob, number> = { pose: 0, hands: 0, face: 0 }
|
||||||
|
|
||||||
|
function updateConfig(next: MocapConfig) {
|
||||||
|
config = next
|
||||||
|
}
|
||||||
|
|
||||||
|
function plan(nowMs: number): MocapJob[] {
|
||||||
|
const jobs: MocapJob[] = []
|
||||||
|
|
||||||
|
for (const job of ['pose', 'hands', 'face'] as const) {
|
||||||
|
if (!config.enabled[job])
|
||||||
|
continue
|
||||||
|
|
||||||
|
const hz = config.hz[job]
|
||||||
|
if (!hz || hz <= 0)
|
||||||
|
continue
|
||||||
|
|
||||||
|
if (nowMs - lastRun[job] >= (1000 / hz))
|
||||||
|
jobs.push(job)
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const j of jobs)
|
||||||
|
lastRun[j] = nowMs
|
||||||
|
|
||||||
|
return jobs
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
plan,
|
||||||
|
updateConfig,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function createMocapEngine(backend: MocapBackend, initialConfig: MocapConfig): MocapEngine {
|
||||||
|
let config = initialConfig
|
||||||
|
const scheduler = createScheduler(config)
|
||||||
|
const stats = createStats()
|
||||||
|
|
||||||
|
let running = false
|
||||||
|
let droppedFrames = 0
|
||||||
|
let lastPartial: PerceptionPartial = {}
|
||||||
|
let rafId: number | undefined
|
||||||
|
|
||||||
|
async function init() {
|
||||||
|
await backend.init(config)
|
||||||
|
}
|
||||||
|
|
||||||
|
function updateConfig(next: MocapConfig) {
|
||||||
|
config = next
|
||||||
|
scheduler.updateConfig(next)
|
||||||
|
}
|
||||||
|
|
||||||
|
function resetState() {
|
||||||
|
lastPartial = {}
|
||||||
|
droppedFrames = 0
|
||||||
|
}
|
||||||
|
|
||||||
|
function start(
|
||||||
|
source: FrameSource,
|
||||||
|
onState: (state: PerceptionState) => void,
|
||||||
|
options?: { onError?: (error: unknown) => void },
|
||||||
|
) {
|
||||||
|
running = true
|
||||||
|
resetState()
|
||||||
|
|
||||||
|
const tick = async () => {
|
||||||
|
try {
|
||||||
|
if (!running)
|
||||||
|
return
|
||||||
|
|
||||||
|
const frame = source.getFrame()
|
||||||
|
const now = performance.now()
|
||||||
|
|
||||||
|
// Skip this frame if the backend is still busy.
|
||||||
|
if (backend.isBusy()) {
|
||||||
|
droppedFrames++
|
||||||
|
rafId = requestAnimationFrame(tick)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
const jobs = scheduler.plan(now)
|
||||||
|
const t0 = performance.now()
|
||||||
|
const partial = jobs.length > 0 ? await backend.run(frame, jobs, now) : {}
|
||||||
|
const latencyMs = performance.now() - t0
|
||||||
|
|
||||||
|
lastPartial = { ...lastPartial, ...partial }
|
||||||
|
|
||||||
|
onState({
|
||||||
|
t: now,
|
||||||
|
...lastPartial,
|
||||||
|
quality: {
|
||||||
|
fps: stats.tick(now),
|
||||||
|
latencyMs,
|
||||||
|
droppedFrames,
|
||||||
|
backend: 'mediapipe',
|
||||||
|
mode: 'split-tasks',
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
rafId = requestAnimationFrame(tick)
|
||||||
|
}
|
||||||
|
catch (err) {
|
||||||
|
// Stop the loop to avoid spamming errors; consumers can restart.
|
||||||
|
stop()
|
||||||
|
options?.onError?.(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
rafId = requestAnimationFrame(tick)
|
||||||
|
}
|
||||||
|
|
||||||
|
function stop() {
|
||||||
|
running = false
|
||||||
|
if (rafId != null)
|
||||||
|
cancelAnimationFrame(rafId)
|
||||||
|
rafId = undefined
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
init,
|
||||||
|
start,
|
||||||
|
stop,
|
||||||
|
updateConfig,
|
||||||
|
resetState,
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,5 @@
|
|||||||
|
export * from './backends/mediapipe'
|
||||||
|
export * from './engine'
|
||||||
|
export * from './three'
|
||||||
|
export * from './types'
|
||||||
|
export * from './utils/overlay'
|
||||||
@@ -0,0 +1,361 @@
|
|||||||
|
import type { VRM } from '@pixiv/three-vrm'
|
||||||
|
|
||||||
|
import type { VrmPoseDirections, VrmPoseTarget, VrmPoseTargets } from './pose-to-vrm'
|
||||||
|
|
||||||
|
import { Matrix4, Quaternion, Vector3 } from 'three'
|
||||||
|
|
||||||
|
export interface VrmPoseApplyOptions {
|
||||||
|
/**
|
||||||
|
* Slerp factor per frame in [0..1]. Higher = snappier, lower = smoother.
|
||||||
|
*/
|
||||||
|
alpha?: number
|
||||||
|
/**
|
||||||
|
* Reject sudden flips: if the target direction is close to the opposite of the
|
||||||
|
* current bone direction (in world space), skip this update.
|
||||||
|
*
|
||||||
|
* `minDot` in [-1..1]. Example: `-0.2` rejects angles > ~101°.
|
||||||
|
*/
|
||||||
|
minDotBeforeReject?: number
|
||||||
|
/**
|
||||||
|
* Similar to `minDotBeforeReject`, but applied to the pole vector when using
|
||||||
|
* pole-based orientation. Helps avoid 180° roll/yaw flips when the pole sign
|
||||||
|
* becomes ambiguous.
|
||||||
|
*/
|
||||||
|
minPoleDotBeforeReject?: number
|
||||||
|
}
|
||||||
|
|
||||||
|
type BoneKey = keyof VrmPoseDirections
|
||||||
|
|
||||||
|
interface BoneChain {
|
||||||
|
bone: string
|
||||||
|
childCandidates: readonly string[]
|
||||||
|
}
|
||||||
|
|
||||||
|
const DEFAULT_ALPHA = 0.35
|
||||||
|
|
||||||
|
const CHAINS: Readonly<Record<BoneKey, BoneChain>> = {
|
||||||
|
hips: {
|
||||||
|
bone: 'hips',
|
||||||
|
childCandidates: ['spine'],
|
||||||
|
},
|
||||||
|
spine: {
|
||||||
|
bone: 'spine',
|
||||||
|
childCandidates: ['chest', 'upperChest', 'neck'],
|
||||||
|
},
|
||||||
|
chest: {
|
||||||
|
bone: 'chest',
|
||||||
|
childCandidates: ['upperChest', 'neck'],
|
||||||
|
},
|
||||||
|
leftShoulder: {
|
||||||
|
bone: 'leftShoulder',
|
||||||
|
childCandidates: ['leftUpperArm'],
|
||||||
|
},
|
||||||
|
rightShoulder: {
|
||||||
|
bone: 'rightShoulder',
|
||||||
|
childCandidates: ['rightUpperArm'],
|
||||||
|
},
|
||||||
|
leftUpperArm: {
|
||||||
|
bone: 'leftUpperArm',
|
||||||
|
childCandidates: ['leftLowerArm'],
|
||||||
|
},
|
||||||
|
leftLowerArm: {
|
||||||
|
bone: 'leftLowerArm',
|
||||||
|
childCandidates: ['leftHand'],
|
||||||
|
},
|
||||||
|
rightUpperArm: {
|
||||||
|
bone: 'rightUpperArm',
|
||||||
|
childCandidates: ['rightLowerArm'],
|
||||||
|
},
|
||||||
|
rightLowerArm: {
|
||||||
|
bone: 'rightLowerArm',
|
||||||
|
childCandidates: ['rightHand'],
|
||||||
|
},
|
||||||
|
leftUpperLeg: {
|
||||||
|
bone: 'leftUpperLeg',
|
||||||
|
childCandidates: ['leftLowerLeg'],
|
||||||
|
},
|
||||||
|
leftLowerLeg: {
|
||||||
|
bone: 'leftLowerLeg',
|
||||||
|
childCandidates: ['leftFoot'],
|
||||||
|
},
|
||||||
|
rightUpperLeg: {
|
||||||
|
bone: 'rightUpperLeg',
|
||||||
|
childCandidates: ['rightLowerLeg'],
|
||||||
|
},
|
||||||
|
rightLowerLeg: {
|
||||||
|
bone: 'rightLowerLeg',
|
||||||
|
childCandidates: ['rightFoot'],
|
||||||
|
},
|
||||||
|
} as const
|
||||||
|
|
||||||
|
const POLE_KEYS: ReadonlySet<BoneKey> = new Set(Object.keys(CHAINS) as BoneKey[])
|
||||||
|
const LIMB_POLE_KEYS: ReadonlySet<BoneKey> = new Set([
|
||||||
|
'leftUpperArm',
|
||||||
|
'rightUpperArm',
|
||||||
|
'leftUpperLeg',
|
||||||
|
'rightUpperLeg',
|
||||||
|
])
|
||||||
|
|
||||||
|
function isFiniteVec3(v: { x: number, y: number, z?: number } | undefined): v is { x: number, y: number, z?: number } {
|
||||||
|
if (!v)
|
||||||
|
return false
|
||||||
|
return Number.isFinite(v.x) && Number.isFinite(v.y) && Number.isFinite(v.z ?? 0)
|
||||||
|
}
|
||||||
|
|
||||||
|
function firstExistingBone(vrm: VRM, candidates: readonly string[]) {
|
||||||
|
for (const name of candidates) {
|
||||||
|
const node = vrm.humanoid?.getNormalizedBoneNode(name as any)
|
||||||
|
if (node)
|
||||||
|
return node
|
||||||
|
}
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
|
||||||
|
function grandchildCandidatesFor(key: BoneKey): readonly string[] {
|
||||||
|
if (key === 'leftUpperArm')
|
||||||
|
return ['leftHand']
|
||||||
|
if (key === 'rightUpperArm')
|
||||||
|
return ['rightHand']
|
||||||
|
if (key === 'leftUpperLeg')
|
||||||
|
return ['leftFoot']
|
||||||
|
if (key === 'rightUpperLeg')
|
||||||
|
return ['rightFoot']
|
||||||
|
return []
|
||||||
|
}
|
||||||
|
|
||||||
|
function orthonormalizePole(dir: Vector3, pole: Vector3): Vector3 | null {
|
||||||
|
const poleOrtho = pole.clone().addScaledVector(dir, -pole.dot(dir))
|
||||||
|
if (poleOrtho.lengthSq() <= 1e-12)
|
||||||
|
return null
|
||||||
|
return poleOrtho.normalize()
|
||||||
|
}
|
||||||
|
|
||||||
|
export function createVrmPoseApplier(options?: VrmPoseApplyOptions) {
|
||||||
|
const alpha = options?.alpha ?? DEFAULT_ALPHA
|
||||||
|
const minDotBeforeReject = options?.minDotBeforeReject ?? -0.2
|
||||||
|
const minPoleDotBeforeReject = options?.minPoleDotBeforeReject ?? -0.2
|
||||||
|
|
||||||
|
const restDirLocal: Partial<Record<BoneKey, Vector3>> = {}
|
||||||
|
const restPoleLocal: Partial<Record<BoneKey, Vector3>> = {}
|
||||||
|
const lastTargetDirWorld: Partial<Record<BoneKey, Vector3>> = {}
|
||||||
|
const lastTargetPoleWorld: Partial<Record<BoneKey, Vector3>> = {}
|
||||||
|
|
||||||
|
const tmpBoneWorldQ = new Quaternion()
|
||||||
|
const tmpParentWorldQ = new Quaternion()
|
||||||
|
const tmpDeltaQ = new Quaternion()
|
||||||
|
const tmpNewWorldQ = new Quaternion()
|
||||||
|
const tmpNewLocalQ = new Quaternion()
|
||||||
|
|
||||||
|
const tmpRestM = new Matrix4()
|
||||||
|
const tmpTargetM = new Matrix4()
|
||||||
|
const tmpRotM = new Matrix4()
|
||||||
|
|
||||||
|
const tmpBoneWorldPos = new Vector3()
|
||||||
|
const tmpChildWorldPos = new Vector3()
|
||||||
|
const tmpGrandWorldPos = new Vector3()
|
||||||
|
const tmpDirWorld = new Vector3()
|
||||||
|
const tmpDirLocal = new Vector3()
|
||||||
|
const tmpCurrentDirWorld = new Vector3()
|
||||||
|
const tmpTargetDirWorld = new Vector3()
|
||||||
|
const tmpTargetPoleWorld = new Vector3()
|
||||||
|
const tmpRestDirLocal = new Vector3()
|
||||||
|
const tmpRestPoleLocal = new Vector3()
|
||||||
|
const tmpRestYLocal = new Vector3()
|
||||||
|
const tmpTargetYWorld = new Vector3()
|
||||||
|
const tmpWorldForward = new Vector3(0, 0, -1)
|
||||||
|
|
||||||
|
function ensureRestDirection(vrm: VRM, key: BoneKey): Vector3 | null {
|
||||||
|
const existing = restDirLocal[key]
|
||||||
|
if (existing)
|
||||||
|
return existing
|
||||||
|
|
||||||
|
const chain = CHAINS[key]
|
||||||
|
const bone = vrm.humanoid?.getNormalizedBoneNode(chain.bone as any)
|
||||||
|
if (!bone)
|
||||||
|
return null
|
||||||
|
|
||||||
|
const child = firstExistingBone(vrm, chain.childCandidates)
|
||||||
|
if (!child)
|
||||||
|
return null
|
||||||
|
|
||||||
|
bone.updateMatrixWorld(true)
|
||||||
|
child.updateMatrixWorld(true)
|
||||||
|
|
||||||
|
bone.getWorldQuaternion(tmpBoneWorldQ)
|
||||||
|
bone.getWorldPosition(tmpBoneWorldPos)
|
||||||
|
child.getWorldPosition(tmpChildWorldPos)
|
||||||
|
|
||||||
|
tmpDirWorld.copy(tmpChildWorldPos).sub(tmpBoneWorldPos).normalize()
|
||||||
|
tmpDirLocal.copy(tmpDirWorld).applyQuaternion(tmpBoneWorldQ.clone().invert()).normalize()
|
||||||
|
|
||||||
|
const stored = tmpDirLocal.clone()
|
||||||
|
restDirLocal[key] = stored
|
||||||
|
return stored
|
||||||
|
}
|
||||||
|
|
||||||
|
function ensureRestPole(vrm: VRM, key: BoneKey): Vector3 | null {
|
||||||
|
if (!POLE_KEYS.has(key))
|
||||||
|
return null
|
||||||
|
|
||||||
|
const existing = restPoleLocal[key]
|
||||||
|
if (existing)
|
||||||
|
return existing
|
||||||
|
|
||||||
|
const chain = CHAINS[key]
|
||||||
|
const bone = vrm.humanoid?.getNormalizedBoneNode(chain.bone as any)
|
||||||
|
if (!bone)
|
||||||
|
return null
|
||||||
|
|
||||||
|
bone.updateMatrixWorld(true)
|
||||||
|
bone.getWorldQuaternion(tmpBoneWorldQ)
|
||||||
|
|
||||||
|
if (LIMB_POLE_KEYS.has(key)) {
|
||||||
|
const child = firstExistingBone(vrm, chain.childCandidates)
|
||||||
|
if (!child)
|
||||||
|
return null
|
||||||
|
|
||||||
|
const grandCandidates = grandchildCandidatesFor(key)
|
||||||
|
const grand = grandCandidates.length ? firstExistingBone(vrm, grandCandidates) : null
|
||||||
|
if (!grand)
|
||||||
|
return null
|
||||||
|
|
||||||
|
child.updateMatrixWorld(true)
|
||||||
|
grand.updateMatrixWorld(true)
|
||||||
|
|
||||||
|
bone.getWorldPosition(tmpBoneWorldPos)
|
||||||
|
child.getWorldPosition(tmpChildWorldPos)
|
||||||
|
grand.getWorldPosition(tmpGrandWorldPos)
|
||||||
|
|
||||||
|
const dir1 = tmpChildWorldPos.clone().sub(tmpBoneWorldPos)
|
||||||
|
const dir2 = tmpGrandWorldPos.clone().sub(tmpChildWorldPos)
|
||||||
|
tmpDirWorld.copy(dir1).cross(dir2)
|
||||||
|
if (tmpDirWorld.lengthSq() <= 1e-12)
|
||||||
|
return null
|
||||||
|
tmpDirWorld.normalize()
|
||||||
|
|
||||||
|
tmpDirLocal.copy(tmpDirWorld).applyQuaternion(tmpBoneWorldQ.clone().invert()).normalize()
|
||||||
|
const stored = tmpDirLocal.clone()
|
||||||
|
restPoleLocal[key] = stored
|
||||||
|
return stored
|
||||||
|
}
|
||||||
|
|
||||||
|
// For torso: use "world forward" as rest pole, transformed into bone local.
|
||||||
|
tmpDirLocal.copy(tmpWorldForward).applyQuaternion(tmpBoneWorldQ.clone().invert()).normalize()
|
||||||
|
const stored = tmpDirLocal.clone()
|
||||||
|
restPoleLocal[key] = stored
|
||||||
|
return stored
|
||||||
|
}
|
||||||
|
|
||||||
|
function applyOne(vrm: VRM, key: BoneKey, target: VrmPoseTarget) {
|
||||||
|
const chain = CHAINS[key]
|
||||||
|
const bone = vrm.humanoid?.getNormalizedBoneNode(chain.bone as any)
|
||||||
|
if (!bone)
|
||||||
|
return
|
||||||
|
|
||||||
|
const rest = ensureRestDirection(vrm, key)
|
||||||
|
if (!rest)
|
||||||
|
return
|
||||||
|
|
||||||
|
tmpTargetDirWorld.set(target.dir.x, target.dir.y, target.dir.z ?? 0)
|
||||||
|
if (tmpTargetDirWorld.lengthSq() <= 1e-12)
|
||||||
|
return
|
||||||
|
tmpTargetDirWorld.normalize()
|
||||||
|
|
||||||
|
// Reject near-180° instant flips based on previous target (not current bone pose).
|
||||||
|
// Using the current bone direction is unreliable when tracking reacquires or when
|
||||||
|
// the bind pose differs from the mocap space.
|
||||||
|
const prevDir = lastTargetDirWorld[key]
|
||||||
|
if (prevDir && prevDir.dot(tmpTargetDirWorld) < minDotBeforeReject)
|
||||||
|
return
|
||||||
|
|
||||||
|
bone.updateMatrixWorld(true)
|
||||||
|
bone.getWorldQuaternion(tmpBoneWorldQ)
|
||||||
|
|
||||||
|
const parent = bone.parent
|
||||||
|
if (parent) {
|
||||||
|
parent.updateMatrixWorld(true)
|
||||||
|
parent.getWorldQuaternion(tmpParentWorldQ)
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
tmpParentWorldQ.identity()
|
||||||
|
}
|
||||||
|
|
||||||
|
const restPole = target.pole ? ensureRestPole(vrm, key) : null
|
||||||
|
const usePole = !!(target.pole && restPole)
|
||||||
|
|
||||||
|
if (usePole) {
|
||||||
|
tmpRestDirLocal.copy(rest).normalize()
|
||||||
|
tmpRestPoleLocal.copy(restPole!).normalize()
|
||||||
|
tmpRestPoleLocal.addScaledVector(tmpRestDirLocal, -tmpRestPoleLocal.dot(tmpRestDirLocal)).normalize()
|
||||||
|
tmpRestYLocal.copy(tmpRestPoleLocal).cross(tmpRestDirLocal).normalize()
|
||||||
|
|
||||||
|
tmpTargetPoleWorld.set(target.pole!.x, target.pole!.y, target.pole!.z ?? 0)
|
||||||
|
if (tmpTargetPoleWorld.lengthSq() <= 1e-12)
|
||||||
|
return
|
||||||
|
tmpTargetPoleWorld.normalize()
|
||||||
|
const poleOrtho = orthonormalizePole(tmpTargetDirWorld, tmpTargetPoleWorld)
|
||||||
|
if (!poleOrtho)
|
||||||
|
return
|
||||||
|
tmpTargetPoleWorld.copy(poleOrtho)
|
||||||
|
tmpTargetYWorld.copy(tmpTargetPoleWorld).cross(tmpTargetDirWorld).normalize()
|
||||||
|
|
||||||
|
// Reject near-180° pole flips based on previous target pole.
|
||||||
|
const prevPole = lastTargetPoleWorld[key]
|
||||||
|
if (prevPole && prevPole.dot(tmpTargetPoleWorld) < minPoleDotBeforeReject)
|
||||||
|
return
|
||||||
|
|
||||||
|
tmpRestM.makeBasis(tmpRestDirLocal, tmpRestYLocal, tmpRestPoleLocal)
|
||||||
|
tmpTargetM.makeBasis(tmpTargetDirWorld, tmpTargetYWorld, tmpTargetPoleWorld)
|
||||||
|
|
||||||
|
tmpRotM.copy(tmpTargetM).multiply(tmpRestM.clone().invert())
|
||||||
|
tmpNewWorldQ.setFromRotationMatrix(tmpRotM)
|
||||||
|
tmpNewLocalQ.copy(tmpParentWorldQ).invert().multiply(tmpNewWorldQ)
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
tmpCurrentDirWorld.copy(rest).applyQuaternion(tmpBoneWorldQ).normalize()
|
||||||
|
tmpDeltaQ.setFromUnitVectors(tmpCurrentDirWorld, tmpTargetDirWorld)
|
||||||
|
tmpNewWorldQ.copy(tmpDeltaQ).multiply(tmpBoneWorldQ)
|
||||||
|
tmpNewLocalQ.copy(tmpParentWorldQ).invert().multiply(tmpNewWorldQ)
|
||||||
|
}
|
||||||
|
|
||||||
|
if (alpha >= 1)
|
||||||
|
bone.quaternion.copy(tmpNewLocalQ)
|
||||||
|
else
|
||||||
|
bone.quaternion.slerp(tmpNewLocalQ, alpha)
|
||||||
|
|
||||||
|
// Update last targets only after a successful apply.
|
||||||
|
lastTargetDirWorld[key] = tmpTargetDirWorld.clone()
|
||||||
|
if (usePole)
|
||||||
|
lastTargetPoleWorld[key] = tmpTargetPoleWorld.clone()
|
||||||
|
else
|
||||||
|
delete lastTargetPoleWorld[key]
|
||||||
|
}
|
||||||
|
|
||||||
|
function applyPoseDirectionsToVrm(vrm: VRM, directions: VrmPoseDirections) {
|
||||||
|
if (!vrm.humanoid)
|
||||||
|
return
|
||||||
|
|
||||||
|
(Object.keys(CHAINS) as BoneKey[]).forEach((key) => {
|
||||||
|
const d = directions[key]
|
||||||
|
if (!isFiniteVec3(d))
|
||||||
|
return
|
||||||
|
applyOne(vrm, key, { dir: d })
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
function applyPoseTargetsToVrm(vrm: VRM, targets: VrmPoseTargets) {
|
||||||
|
if (!vrm.humanoid)
|
||||||
|
return
|
||||||
|
|
||||||
|
(Object.keys(CHAINS) as BoneKey[]).forEach((key) => {
|
||||||
|
const t = targets[key]
|
||||||
|
if (!t || !isFiniteVec3(t.dir))
|
||||||
|
return
|
||||||
|
if (t.pole && !isFiniteVec3(t.pole))
|
||||||
|
return
|
||||||
|
applyOne(vrm, key, t)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
return { applyPoseDirectionsToVrm, applyPoseTargetsToVrm }
|
||||||
|
}
|
||||||
@@ -0,0 +1,2 @@
|
|||||||
|
export * from './apply-pose-to-vrm'
|
||||||
|
export * from './pose-to-vrm'
|
||||||
@@ -0,0 +1,347 @@
|
|||||||
|
import type { Vector3Like } from 'three'
|
||||||
|
|
||||||
|
import type { PoseState } from '../types'
|
||||||
|
|
||||||
|
export interface PoseToVrmOptions {
|
||||||
|
/**
|
||||||
|
* Axis remap from MediaPipe world to three/VRM-ish space.
|
||||||
|
* This is a pragmatic default; adjust if you see mirrored or inverted motion.
|
||||||
|
*/
|
||||||
|
axis?: {
|
||||||
|
x: 1 | -1
|
||||||
|
y: 1 | -1
|
||||||
|
z: 1 | -1
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Confidence gating based on MediaPipe pose landmark `visibility` / `presence`.
|
||||||
|
* When a landmark is not confident (e.g. off-screen), we skip emitting targets that depend on it.
|
||||||
|
*/
|
||||||
|
confidence?: {
|
||||||
|
/**
|
||||||
|
* Min `visibility` in [0..1]. Only enforced when the field exists on the landmark.
|
||||||
|
*/
|
||||||
|
minVisibility?: number
|
||||||
|
/**
|
||||||
|
* Min `presence` in [0..1]. Only enforced when the field exists on the landmark.
|
||||||
|
*/
|
||||||
|
minPresence?: number
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Use previous frame information to avoid sudden 180° flips caused by ambiguous poles.
|
||||||
|
*/
|
||||||
|
stabilize?: {
|
||||||
|
previousTargets?: VrmPoseTargets
|
||||||
|
previousForward?: Vector3Like
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export type VrmPoseDirections = Partial<Record<
|
||||||
|
| 'hips'
|
||||||
|
| 'spine'
|
||||||
|
| 'chest'
|
||||||
|
| 'leftShoulder'
|
||||||
|
| 'rightShoulder'
|
||||||
|
| 'leftUpperArm'
|
||||||
|
| 'leftLowerArm'
|
||||||
|
| 'rightUpperArm'
|
||||||
|
| 'rightLowerArm'
|
||||||
|
| 'leftUpperLeg'
|
||||||
|
| 'leftLowerLeg'
|
||||||
|
| 'rightUpperLeg'
|
||||||
|
| 'rightLowerLeg',
|
||||||
|
Vector3Like
|
||||||
|
>>
|
||||||
|
|
||||||
|
export interface VrmPoseTarget {
|
||||||
|
dir: Vector3Like
|
||||||
|
pole?: Vector3Like
|
||||||
|
}
|
||||||
|
|
||||||
|
export type VrmPoseTargets = Partial<Record<keyof VrmPoseDirections, VrmPoseTarget>>
|
||||||
|
|
||||||
|
const DEFAULT_AXIS = { x: 1 as const, y: 1 as const, z: 1 as const }
|
||||||
|
const DEFAULT_MIN_VISIBILITY = 0.5
|
||||||
|
const DEFAULT_MIN_PRESENCE = 0
|
||||||
|
|
||||||
|
// TODO: Consider consolidating these vector helpers into a shared math utility if more drivers need them.
|
||||||
|
function vSub(a: Vector3Like, b: Vector3Like): Vector3Like {
|
||||||
|
return { x: a.x - b.x, y: a.y - b.y, z: (a.z ?? 0) - (b.z ?? 0) }
|
||||||
|
}
|
||||||
|
|
||||||
|
function vAdd(a: Vector3Like, b: Vector3Like): Vector3Like {
|
||||||
|
return { x: a.x + b.x, y: a.y + b.y, z: (a.z ?? 0) + (b.z ?? 0) }
|
||||||
|
}
|
||||||
|
|
||||||
|
function vScale(v: Vector3Like, s: number): Vector3Like {
|
||||||
|
return { x: v.x * s, y: v.y * s, z: (v.z ?? 0) * s }
|
||||||
|
}
|
||||||
|
|
||||||
|
function vLen(v: Vector3Like): number {
|
||||||
|
return Math.hypot(v.x, v.y, v.z ?? 0)
|
||||||
|
}
|
||||||
|
|
||||||
|
function vNormalize(v: Vector3Like): Vector3Like | null {
|
||||||
|
const len = vLen(v)
|
||||||
|
if (!Number.isFinite(len) || len <= 1e-6)
|
||||||
|
return null
|
||||||
|
return vScale(v, 1 / len)
|
||||||
|
}
|
||||||
|
|
||||||
|
function vRemapAxis(v: Vector3Like, axis: { x: 1 | -1, y: 1 | -1, z: 1 | -1 }): Vector3Like {
|
||||||
|
return { x: v.x * axis.x, y: v.y * axis.y, z: (v.z ?? 0) * axis.z }
|
||||||
|
}
|
||||||
|
|
||||||
|
function vCross(a: Vector3Like, b: Vector3Like): Vector3Like {
|
||||||
|
const az = a.z ?? 0
|
||||||
|
const bz = b.z ?? 0
|
||||||
|
return {
|
||||||
|
x: a.y * bz - az * b.y,
|
||||||
|
y: az * b.x - a.x * bz,
|
||||||
|
z: a.x * b.y - a.y * b.x,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function vDot(a: Vector3Like, b: Vector3Like): number {
|
||||||
|
return a.x * b.x + a.y * b.y + (a.z ?? 0) * (b.z ?? 0)
|
||||||
|
}
|
||||||
|
|
||||||
|
function vNeg(v: Vector3Like): Vector3Like {
|
||||||
|
return { x: -v.x, y: -v.y, z: -(v.z ?? 0) }
|
||||||
|
}
|
||||||
|
|
||||||
|
function safePole(dir: Vector3Like, pole: Vector3Like, threshold = 0.85): Vector3Like | null {
|
||||||
|
const d = Math.abs(vDot(dir, pole))
|
||||||
|
if (!Number.isFinite(d))
|
||||||
|
return null
|
||||||
|
return d > threshold ? null : pole
|
||||||
|
}
|
||||||
|
|
||||||
|
function get(points: Vector3Like[], index: number): Vector3Like | null {
|
||||||
|
const p = points[index]
|
||||||
|
if (!p)
|
||||||
|
return null
|
||||||
|
if (!Number.isFinite(p.x) || !Number.isFinite(p.y))
|
||||||
|
return null
|
||||||
|
return { x: p.x, y: p.y, z: p.z ?? 0 }
|
||||||
|
}
|
||||||
|
|
||||||
|
// NOTICE: mediapipe doesn't provide this type correctly, so we define it here.
|
||||||
|
interface LandmarkWithPresence { presence?: number }
|
||||||
|
interface LandmarkWithVisibility { visibility?: number }
|
||||||
|
|
||||||
|
function getOptionalPresence(landmark: unknown): number | undefined {
|
||||||
|
if (landmark && typeof landmark === 'object' && 'presence' in landmark)
|
||||||
|
return (landmark as LandmarkWithPresence).presence
|
||||||
|
return undefined
|
||||||
|
}
|
||||||
|
|
||||||
|
function getOptionalVisibility(landmark: unknown): number | undefined {
|
||||||
|
if (landmark && typeof landmark === 'object' && 'visibility' in landmark)
|
||||||
|
return (landmark as LandmarkWithVisibility).visibility
|
||||||
|
return undefined
|
||||||
|
}
|
||||||
|
|
||||||
|
function isConfident(pose: PoseState, index: number, thresholds: { minVisibility: number, minPresence: number }): boolean {
|
||||||
|
// User requirement: do not output anything when `visibility` is missing.
|
||||||
|
// Prefer 2D landmarks for visibility/presence (they are more consistently populated),
|
||||||
|
// but still allow fallback to world landmarks when needed.
|
||||||
|
const lm2d = pose.landmarks2d?.[index]
|
||||||
|
const lm3d = pose.worldLandmarks?.[index]
|
||||||
|
if (!lm2d && !lm3d)
|
||||||
|
return false
|
||||||
|
|
||||||
|
const visibility = getOptionalVisibility(lm2d) ?? getOptionalVisibility(lm3d)
|
||||||
|
if (visibility == null || !Number.isFinite(visibility))
|
||||||
|
return false
|
||||||
|
if (visibility < thresholds.minVisibility)
|
||||||
|
return false
|
||||||
|
|
||||||
|
if (thresholds.minPresence > 0) {
|
||||||
|
const presence = getOptionalPresence(lm2d) ?? getOptionalPresence(lm3d)
|
||||||
|
if (presence != null && Number.isFinite(presence) && presence < thresholds.minPresence)
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
function mid(a: Vector3Like, b: Vector3Like): Vector3Like {
|
||||||
|
return vScale(vAdd(a, b), 0.5)
|
||||||
|
}
|
||||||
|
|
||||||
|
export function poseToVrmTargets(pose: PoseState, options?: PoseToVrmOptions): VrmPoseTargets {
|
||||||
|
// Prefer world landmarks when available; fallback to normalized landmarks to keep the pipeline usable.
|
||||||
|
const points = pose.worldLandmarks
|
||||||
|
if (!points?.length)
|
||||||
|
return {}
|
||||||
|
|
||||||
|
const axis = options?.axis ?? DEFAULT_AXIS
|
||||||
|
const thresholds = {
|
||||||
|
minVisibility: options?.confidence?.minVisibility ?? DEFAULT_MIN_VISIBILITY,
|
||||||
|
minPresence: options?.confidence?.minPresence ?? DEFAULT_MIN_PRESENCE,
|
||||||
|
}
|
||||||
|
|
||||||
|
const getC = (index: number) => (isConfident(pose, index, thresholds) ? get(points, index) : null)
|
||||||
|
|
||||||
|
const leftShoulder = getC(11)
|
||||||
|
const rightShoulder = getC(12)
|
||||||
|
const leftElbow = getC(13)
|
||||||
|
const rightElbow = getC(14)
|
||||||
|
const leftWrist = getC(15)
|
||||||
|
const rightWrist = getC(16)
|
||||||
|
const leftHip = getC(23)
|
||||||
|
const rightHip = getC(24)
|
||||||
|
const leftKnee = getC(25)
|
||||||
|
const rightKnee = getC(26)
|
||||||
|
const leftAnkle = getC(27)
|
||||||
|
const rightAnkle = getC(28)
|
||||||
|
|
||||||
|
const out: VrmPoseTargets = {}
|
||||||
|
|
||||||
|
const shoulderCenter = leftShoulder && rightShoulder ? mid(leftShoulder, rightShoulder) : null
|
||||||
|
const hipCenter = leftHip && rightHip ? mid(leftHip, rightHip) : null
|
||||||
|
|
||||||
|
const prevTargets = options?.stabilize?.previousTargets
|
||||||
|
const stabilizePole = (key: keyof VrmPoseTargets, pole: Vector3Like): Vector3Like => {
|
||||||
|
const prev = prevTargets?.[key]?.pole
|
||||||
|
if (prev && vDot(prev, pole) < 0)
|
||||||
|
return vNeg(pole)
|
||||||
|
return pole
|
||||||
|
}
|
||||||
|
|
||||||
|
// Torso basis:
|
||||||
|
// - up: hipCenter -> shoulderCenter (dir)
|
||||||
|
// - right: leftShoulder -> rightShoulder
|
||||||
|
// - forward: cross(right, up)
|
||||||
|
//
|
||||||
|
// NOTICE: In apply-pose-to-vrm.ts, pole is used as the Z axis in `makeBasis(X=dir, Y=..., Z=pole)`,
|
||||||
|
// so torso pole must be "forward-like" (body facing), not "right-like" (shoulder line).
|
||||||
|
let torsoForward: Vector3Like | null = null
|
||||||
|
if (hipCenter && shoulderCenter && leftShoulder && rightShoulder) {
|
||||||
|
const rightRaw = vSub(rightShoulder, leftShoulder)
|
||||||
|
const upRaw = vSub(shoulderCenter, hipCenter)
|
||||||
|
const right = vNormalize(vRemapAxis(rightRaw, axis))
|
||||||
|
const up = vNormalize(vRemapAxis(upRaw, axis))
|
||||||
|
if (right && up) {
|
||||||
|
// Base forward from right×up, then fix sign so that (up×forward) matches observed shoulder-right.
|
||||||
|
let fw = vNormalize(vCross(right, up))
|
||||||
|
if (fw) {
|
||||||
|
const rightFromBasis = vNormalize(vCross(fw, up))
|
||||||
|
if (rightFromBasis && vDot(rightFromBasis, right) < 0)
|
||||||
|
fw = vNeg(fw)
|
||||||
|
|
||||||
|
const prevForward = options?.stabilize?.previousForward ?? prevTargets?.hips?.pole ?? prevTargets?.spine?.pole ?? prevTargets?.chest?.pole
|
||||||
|
torsoForward = prevForward && vDot(prevForward, fw) < 0 ? vNeg(fw) : fw
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Hips/spine/chest: drive "up" with forward pole so the avatar can turn with you.
|
||||||
|
if (hipCenter && shoulderCenter) {
|
||||||
|
const up = vNormalize(vRemapAxis(vSub(shoulderCenter, hipCenter), axis))
|
||||||
|
if (up) {
|
||||||
|
if (torsoForward)
|
||||||
|
out.hips = { dir: up, pole: stabilizePole('hips', torsoForward) }
|
||||||
|
else
|
||||||
|
out.hips = { dir: up }
|
||||||
|
|
||||||
|
if (torsoForward) {
|
||||||
|
out.spine = { dir: up, pole: stabilizePole('spine', torsoForward) }
|
||||||
|
out.chest = { dir: up, pole: stabilizePole('chest', torsoForward) }
|
||||||
|
}
|
||||||
|
else {
|
||||||
|
out.spine = { dir: up }
|
||||||
|
out.chest = { dir: up }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Shoulder (clavicle-ish): shoulder center -> shoulder
|
||||||
|
if (shoulderCenter && leftShoulder) {
|
||||||
|
const d = vNormalize(vRemapAxis(vSub(leftShoulder, shoulderCenter), axis))
|
||||||
|
if (d)
|
||||||
|
out.leftShoulder = { dir: d }
|
||||||
|
}
|
||||||
|
if (shoulderCenter && rightShoulder) {
|
||||||
|
const d = vNormalize(vRemapAxis(vSub(rightShoulder, shoulderCenter), axis))
|
||||||
|
if (d)
|
||||||
|
out.rightShoulder = { dir: d }
|
||||||
|
}
|
||||||
|
|
||||||
|
// Arms (with pole from elbow bend plane)
|
||||||
|
if (leftShoulder && leftElbow) {
|
||||||
|
const upper = vNormalize(vRemapAxis(vSub(leftElbow, leftShoulder), axis))
|
||||||
|
if (upper) {
|
||||||
|
const poleRaw = leftWrist ? vCross(vSub(leftElbow, leftShoulder), vSub(leftWrist, leftElbow)) : null
|
||||||
|
const pole = poleRaw ? vNormalize(vRemapAxis(poleRaw, axis)) : null
|
||||||
|
out.leftUpperArm = pole ? { dir: upper, pole } : { dir: upper }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (leftElbow && leftWrist) {
|
||||||
|
const lower = vNormalize(vRemapAxis(vSub(leftWrist, leftElbow), axis))
|
||||||
|
if (lower)
|
||||||
|
out.leftLowerArm = { dir: lower }
|
||||||
|
}
|
||||||
|
if (rightShoulder && rightElbow) {
|
||||||
|
const upper = vNormalize(vRemapAxis(vSub(rightElbow, rightShoulder), axis))
|
||||||
|
if (upper) {
|
||||||
|
const poleRaw = rightWrist ? vCross(vSub(rightElbow, rightShoulder), vSub(rightWrist, rightElbow)) : null
|
||||||
|
const pole = poleRaw ? vNormalize(vRemapAxis(poleRaw, axis)) : null
|
||||||
|
out.rightUpperArm = pole ? { dir: upper, pole } : { dir: upper }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (rightElbow && rightWrist) {
|
||||||
|
const lower = vNormalize(vRemapAxis(vSub(rightWrist, rightElbow), axis))
|
||||||
|
if (lower)
|
||||||
|
out.rightLowerArm = { dir: lower }
|
||||||
|
}
|
||||||
|
|
||||||
|
// Legs: require ankle to reduce hallucinated flips when lower body is off-screen
|
||||||
|
if (leftHip && leftKnee && leftAnkle) {
|
||||||
|
const upper = vNormalize(vRemapAxis(vSub(leftKnee, leftHip), axis))
|
||||||
|
if (upper) {
|
||||||
|
const poleRaw = vCross(vSub(leftKnee, leftHip), vSub(leftAnkle, leftKnee))
|
||||||
|
let pole = vNormalize(vRemapAxis(poleRaw, axis))
|
||||||
|
if (pole)
|
||||||
|
pole = stabilizePole('leftUpperLeg', pole)
|
||||||
|
if (pole)
|
||||||
|
pole = safePole(upper, pole)
|
||||||
|
out.leftUpperLeg = pole ? { dir: upper, pole } : { dir: upper }
|
||||||
|
}
|
||||||
|
|
||||||
|
const lower = vNormalize(vRemapAxis(vSub(leftAnkle, leftKnee), axis))
|
||||||
|
if (lower)
|
||||||
|
out.leftLowerLeg = { dir: lower }
|
||||||
|
}
|
||||||
|
|
||||||
|
if (rightHip && rightKnee && rightAnkle) {
|
||||||
|
const upper = vNormalize(vRemapAxis(vSub(rightKnee, rightHip), axis))
|
||||||
|
if (upper) {
|
||||||
|
const poleRaw = vCross(vSub(rightKnee, rightHip), vSub(rightAnkle, rightKnee))
|
||||||
|
let pole = vNormalize(vRemapAxis(poleRaw, axis))
|
||||||
|
if (pole)
|
||||||
|
pole = stabilizePole('rightUpperLeg', pole)
|
||||||
|
if (pole)
|
||||||
|
pole = safePole(upper, pole)
|
||||||
|
out.rightUpperLeg = pole ? { dir: upper, pole } : { dir: upper }
|
||||||
|
}
|
||||||
|
|
||||||
|
const lower = vNormalize(vRemapAxis(vSub(rightAnkle, rightKnee), axis))
|
||||||
|
if (lower)
|
||||||
|
out.rightLowerLeg = { dir: lower }
|
||||||
|
}
|
||||||
|
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
export function poseToVrmDirections(pose: PoseState, options?: PoseToVrmOptions): VrmPoseDirections {
|
||||||
|
const targets = poseToVrmTargets(pose, options)
|
||||||
|
const out: VrmPoseDirections = {}
|
||||||
|
;(Object.keys(targets) as (keyof VrmPoseDirections)[]).forEach((k) => {
|
||||||
|
const t = targets[k]
|
||||||
|
if (t)
|
||||||
|
out[k] = t.dir
|
||||||
|
})
|
||||||
|
return out
|
||||||
|
}
|
||||||
@@ -0,0 +1,74 @@
|
|||||||
|
import type { FilesetResolver, Landmark, NormalizedLandmark } from '@mediapipe/tasks-vision'
|
||||||
|
|
||||||
|
export type VisionTaskModule = typeof import('@mediapipe/tasks-vision')
|
||||||
|
|
||||||
|
// Indirect export from @mediapipe/tasks-vision
|
||||||
|
export type VisionTaskWasmFileset = Awaited<ReturnType<typeof FilesetResolver.forVisionTasks>>
|
||||||
|
|
||||||
|
export type Landmark2D = NormalizedLandmark
|
||||||
|
export type Landmark3D = Landmark
|
||||||
|
|
||||||
|
export interface PoseState {
|
||||||
|
landmarks2d?: Landmark2D[]
|
||||||
|
worldLandmarks?: Landmark3D[]
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface HandState {
|
||||||
|
handedness: 'Left' | 'Right'
|
||||||
|
landmarks2d: Landmark2D[]
|
||||||
|
score?: number
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface FaceState {
|
||||||
|
hasFace?: boolean
|
||||||
|
landmarks2d?: Landmark2D[]
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface PerceptionQuality {
|
||||||
|
fps: number
|
||||||
|
latencyMs?: number
|
||||||
|
droppedFrames?: number
|
||||||
|
backend: 'mediapipe'
|
||||||
|
mode: 'split-tasks'
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface PerceptionPartial {
|
||||||
|
pose?: PoseState
|
||||||
|
hands?: HandState[]
|
||||||
|
face?: FaceState
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface PerceptionState extends PerceptionPartial {
|
||||||
|
t: number
|
||||||
|
quality: PerceptionQuality
|
||||||
|
}
|
||||||
|
|
||||||
|
export type MocapJob = 'pose' | 'hands' | 'face'
|
||||||
|
|
||||||
|
export interface MocapConfig {
|
||||||
|
enabled: Record<MocapJob, boolean>
|
||||||
|
hz: Record<MocapJob, number>
|
||||||
|
maxPeople: 1
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface FrameSource {
|
||||||
|
getFrame: () => TexImageSource
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface MocapBackend {
|
||||||
|
init: (config: MocapConfig) => Promise<void>
|
||||||
|
isBusy: () => boolean
|
||||||
|
run: (frame: TexImageSource, jobs: MocapJob[], nowMs: number) => Promise<PerceptionPartial>
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface MocapEngine {
|
||||||
|
init: () => Promise<void>
|
||||||
|
start: (
|
||||||
|
source: FrameSource,
|
||||||
|
onState: (state: PerceptionState) => void,
|
||||||
|
options?: { onError?: (error: unknown) => void },
|
||||||
|
) => void
|
||||||
|
stop: () => void
|
||||||
|
updateConfig: (config: MocapConfig) => void
|
||||||
|
resetState: () => void
|
||||||
|
}
|
||||||
@@ -0,0 +1,93 @@
|
|||||||
|
import type { FaceState, HandState, Landmark2D, PerceptionState, PoseState } from '../types'
|
||||||
|
|
||||||
|
import { DrawingUtils, HandLandmarker, PoseLandmarker } from '@mediapipe/tasks-vision'
|
||||||
|
|
||||||
|
const POSE_CONNECTIONS: Readonly<{ start: number, end: number }[]> = PoseLandmarker.POSE_CONNECTIONS
|
||||||
|
const HAND_CONNECTIONS: Readonly<{ start: number, end: number }[]> = HandLandmarker.HAND_CONNECTIONS
|
||||||
|
|
||||||
|
// NOTICE: Palette inspired by https://github.com/proj-airi/webai-examples (see review link in PR).
|
||||||
|
const OVERLAY_PALETTE = [
|
||||||
|
{ point: 'rgba(80, 200, 255, 0.95)', connector: 'rgba(80, 200, 255, 0.55)' },
|
||||||
|
{ point: 'rgba(120, 255, 140, 0.95)', connector: 'rgba(120, 255, 140, 0.55)' },
|
||||||
|
{ point: 'rgba(255, 180, 80, 0.95)', connector: 'rgba(255, 180, 80, 0.55)' },
|
||||||
|
{ point: 'rgba(180, 120, 255, 0.55)', connector: 'rgba(180, 120, 255, 0.55)' },
|
||||||
|
]
|
||||||
|
|
||||||
|
// NOTICE: Tuned for devtools readability; see https://ai.google.dev/edge/api/mediapipe/js/tasks-vision.drawingutils.
|
||||||
|
const OVERLAY_STYLES = {
|
||||||
|
connectorLineWidth: 2.5,
|
||||||
|
pointRadius: 6,
|
||||||
|
facePointRadius: 2,
|
||||||
|
pointLineWidth: 1.5,
|
||||||
|
}
|
||||||
|
|
||||||
|
const paletteFor = (index: number) => OVERLAY_PALETTE[index % OVERLAY_PALETTE.length]
|
||||||
|
|
||||||
|
function drawFace(drawing: DrawingUtils, face: FaceState) {
|
||||||
|
if (!face.landmarks2d?.length)
|
||||||
|
return
|
||||||
|
|
||||||
|
// Face has 468 points; keep them small to reduce clutter.
|
||||||
|
const colors = paletteFor(3)
|
||||||
|
drawing.drawLandmarks(face.landmarks2d as Landmark2D[], {
|
||||||
|
color: colors.point,
|
||||||
|
radius: OVERLAY_STYLES.facePointRadius,
|
||||||
|
lineWidth: OVERLAY_STYLES.pointLineWidth,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
function drawPose(drawing: DrawingUtils, pose: PoseState) {
|
||||||
|
if (!pose.landmarks2d?.length)
|
||||||
|
return
|
||||||
|
|
||||||
|
const colors = paletteFor(0)
|
||||||
|
drawing.drawConnectors(pose.landmarks2d as Landmark2D[], POSE_CONNECTIONS.map(({ start, end }) => ({ start, end })), {
|
||||||
|
color: colors.connector,
|
||||||
|
lineWidth: OVERLAY_STYLES.connectorLineWidth,
|
||||||
|
})
|
||||||
|
drawing.drawLandmarks(pose.landmarks2d as Landmark2D[], {
|
||||||
|
color: colors.point,
|
||||||
|
radius: OVERLAY_STYLES.pointRadius,
|
||||||
|
lineWidth: OVERLAY_STYLES.pointLineWidth,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
function drawHands(drawing: DrawingUtils, hands: HandState[]) {
|
||||||
|
hands.forEach((hand) => {
|
||||||
|
const handIndex = hand.handedness === 'Left' ? 1 : 2
|
||||||
|
const colors = paletteFor(handIndex)
|
||||||
|
drawing.drawConnectors(hand.landmarks2d as Landmark2D[], HAND_CONNECTIONS.map(({ start, end }) => ({ start, end })), {
|
||||||
|
color: colors.connector,
|
||||||
|
lineWidth: OVERLAY_STYLES.connectorLineWidth,
|
||||||
|
})
|
||||||
|
drawing.drawLandmarks(hand.landmarks2d as Landmark2D[], {
|
||||||
|
color: colors.point,
|
||||||
|
radius: OVERLAY_STYLES.pointRadius,
|
||||||
|
lineWidth: OVERLAY_STYLES.pointLineWidth,
|
||||||
|
})
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
export function drawOverlay(
|
||||||
|
ctx: CanvasRenderingContext2D,
|
||||||
|
state: PerceptionState,
|
||||||
|
enabled?: Partial<Record<'pose' | 'hands' | 'face', boolean>>,
|
||||||
|
) {
|
||||||
|
const { width, height } = ctx.canvas
|
||||||
|
ctx.clearRect(0, 0, width, height)
|
||||||
|
|
||||||
|
const drawing = new DrawingUtils(ctx)
|
||||||
|
|
||||||
|
const showPose = enabled?.pose ?? true
|
||||||
|
const showHands = enabled?.hands ?? true
|
||||||
|
const showFace = enabled?.face ?? true
|
||||||
|
|
||||||
|
if (state.face && showFace)
|
||||||
|
drawFace(drawing, state.face)
|
||||||
|
|
||||||
|
if (state.pose && showPose)
|
||||||
|
drawPose(drawing, state.pose)
|
||||||
|
|
||||||
|
if (state.hands?.length && showHands)
|
||||||
|
drawHands(drawing, state.hands)
|
||||||
|
}
|
||||||
@@ -0,0 +1,58 @@
|
|||||||
|
/* eslint-disable antfu/no-top-level-await */
|
||||||
|
/* eslint-disable no-console */
|
||||||
|
|
||||||
|
import type { VisionTaskAssets } from './tasks'
|
||||||
|
|
||||||
|
import fs from 'node:fs/promises'
|
||||||
|
|
||||||
|
import { Buffer } from 'node:buffer'
|
||||||
|
import { fileURLToPath } from 'node:url'
|
||||||
|
|
||||||
|
import { ofetch } from 'ofetch'
|
||||||
|
|
||||||
|
import { visionTaskAssets } from './tasks'
|
||||||
|
|
||||||
|
const taskSources: Record<keyof VisionTaskAssets, string> = {
|
||||||
|
pose: 'https://storage.googleapis.com/mediapipe-models/pose_landmarker/pose_landmarker_lite/float16/1/pose_landmarker_lite.task',
|
||||||
|
hands: 'https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/1/hand_landmarker.task',
|
||||||
|
face: 'https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task',
|
||||||
|
}
|
||||||
|
|
||||||
|
await fs.mkdir(fileURLToPath(new URL('./assets', import.meta.url)), { recursive: true })
|
||||||
|
|
||||||
|
await Promise.all(Object.entries(taskSources).map(
|
||||||
|
async ([key, url]) => {
|
||||||
|
console.log(`Downloading MediaPipe vision task asset for ${key} from ${url}...`)
|
||||||
|
const res = await ofetch(url, { responseType: 'arrayBuffer' })
|
||||||
|
const outputPath = fileURLToPath(visionTaskAssets[key as keyof VisionTaskAssets])
|
||||||
|
await fs.writeFile(outputPath, Buffer.from(res))
|
||||||
|
console.log(`MediaPipe vision task asset for ${key} saved to ${outputPath}`)
|
||||||
|
},
|
||||||
|
))
|
||||||
|
|
||||||
|
const wasmSourceDir = fileURLToPath(new URL('../node_modules/@mediapipe/tasks-vision/wasm', import.meta.url))
|
||||||
|
const wasmOutputDir = fileURLToPath(new URL('./assets/wasm', import.meta.url))
|
||||||
|
await fs.mkdir(wasmOutputDir, { recursive: true })
|
||||||
|
await fs.cp(wasmSourceDir, wasmOutputDir, { recursive: true, force: true })
|
||||||
|
|
||||||
|
await Promise.all(Object.entries(visionTaskAssets).map(
|
||||||
|
async ([key, url]) => {
|
||||||
|
const path = fileURLToPath(url)
|
||||||
|
try {
|
||||||
|
await fs.access(path, fs.constants.R_OK)
|
||||||
|
}
|
||||||
|
catch (err) {
|
||||||
|
throw new Error(`Failed to ensure MediaPipe vision task asset for ${key}: ${err}`)
|
||||||
|
}
|
||||||
|
const stat = await fs.stat(path)
|
||||||
|
if (!stat.isFile()) {
|
||||||
|
throw new Error(`Failed to ensure MediaPipe vision task asset for ${key}: not a file: ${path}`)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
))
|
||||||
|
|
||||||
|
const wasmEntries = await fs.readdir(wasmOutputDir)
|
||||||
|
if (!wasmEntries.length)
|
||||||
|
throw new Error(`Failed to ensure MediaPipe WASM assets: ${wasmOutputDir} is empty`)
|
||||||
|
|
||||||
|
console.log('All MediaPipe vision task assets are prepared.')
|
||||||
@@ -0,0 +1,13 @@
|
|||||||
|
export interface VisionTaskAssets {
|
||||||
|
pose: string
|
||||||
|
hands: string
|
||||||
|
face: string
|
||||||
|
}
|
||||||
|
|
||||||
|
export const visionTaskAssets: VisionTaskAssets = {
|
||||||
|
pose: new URL('./assets/pose_landmarker_lite.task', import.meta.url).href,
|
||||||
|
hands: new URL('./assets/hand_landmarker.task', import.meta.url).href,
|
||||||
|
face: new URL('./assets/face_landmarker.task', import.meta.url).href,
|
||||||
|
}
|
||||||
|
|
||||||
|
export const visionTaskWasmRoot = new URL('./assets/wasm', import.meta.url).href
|
||||||
@@ -0,0 +1,30 @@
|
|||||||
|
{
|
||||||
|
"compilerOptions": {
|
||||||
|
"target": "ESNext",
|
||||||
|
"lib": [
|
||||||
|
"DOM",
|
||||||
|
"ESNext"
|
||||||
|
],
|
||||||
|
"module": "ESNext",
|
||||||
|
"moduleResolution": "Bundler",
|
||||||
|
"resolveJsonModule": true,
|
||||||
|
"strict": true,
|
||||||
|
"strictNullChecks": true,
|
||||||
|
"noUnusedLocals": true,
|
||||||
|
"noEmit": true,
|
||||||
|
"esModuleInterop": true,
|
||||||
|
"forceConsistentCasingInFileNames": true,
|
||||||
|
"isolatedModules": true,
|
||||||
|
"verbatimModuleSyntax": true,
|
||||||
|
"skipLibCheck": true
|
||||||
|
},
|
||||||
|
"include": [
|
||||||
|
"src/**/*.ts",
|
||||||
|
"src/**/*.d.ts",
|
||||||
|
"tasks/**/*.ts"
|
||||||
|
],
|
||||||
|
"exclude": [
|
||||||
|
"dist",
|
||||||
|
"node_modules"
|
||||||
|
]
|
||||||
|
}
|
||||||
@@ -159,6 +159,9 @@ let stopCameraWatch: WatchStopHandle | undefined
|
|||||||
// Animation related ref
|
// Animation related ref
|
||||||
const vrmAnimationMixer = ref<AnimationMixer>()
|
const vrmAnimationMixer = ref<AnimationMixer>()
|
||||||
const { onBeforeRender, stop, start } = useLoop()
|
const { onBeforeRender, stop, start } = useLoop()
|
||||||
|
|
||||||
|
type VrmFrameHook = (vrm: VRM, delta: number) => void
|
||||||
|
const vrmFrameHook = shallowRef<VrmFrameHook>()
|
||||||
let disposeBeforeRenderLoop: (() => void | undefined)
|
let disposeBeforeRenderLoop: (() => void | undefined)
|
||||||
|
|
||||||
// Expressions
|
// Expressions
|
||||||
@@ -417,13 +420,23 @@ async function loadModel() {
|
|||||||
// Clean up & animation setting
|
// Clean up & animation setting
|
||||||
disposeBeforeRenderLoop = onBeforeRender(({ delta }) => {
|
disposeBeforeRenderLoop = onBeforeRender(({ delta }) => {
|
||||||
vrmAnimationMixer.value?.update(delta)
|
vrmAnimationMixer.value?.update(delta)
|
||||||
vrm.value?.update(delta)
|
const activeVrm = vrm.value
|
||||||
vrm.value?.lookAt?.update?.(delta)
|
if (activeVrm && vrmFrameHook.value) {
|
||||||
blink.update(vrm.value, delta)
|
try {
|
||||||
idleEyeSaccades.update(vrm.value, lookAtTarget, delta)
|
vrmFrameHook.value(activeVrm, delta)
|
||||||
|
}
|
||||||
|
catch (err) {
|
||||||
|
console.error(err)
|
||||||
|
emit('error', err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
activeVrm?.humanoid.update()
|
||||||
|
activeVrm?.lookAt?.update?.(delta)
|
||||||
|
blink.update(activeVrm, delta)
|
||||||
|
idleEyeSaccades.update(activeVrm, lookAtTarget, delta)
|
||||||
vrmEmote.value?.update(delta)
|
vrmEmote.value?.update(delta)
|
||||||
vrmLipSync.update(vrm.value, delta)
|
vrmLipSync.update(activeVrm, delta)
|
||||||
vrm.value?.springBoneManager?.update(delta)
|
activeVrm?.springBoneManager?.update(delta)
|
||||||
}).off
|
}).off
|
||||||
|
|
||||||
// update the 'last model src'
|
// update the 'last model src'
|
||||||
@@ -562,6 +575,9 @@ defineExpose({
|
|||||||
setExpression(expression: string) {
|
setExpression(expression: string) {
|
||||||
vrmEmote.value?.setEmotionWithResetAfter(expression, 1000)
|
vrmEmote.value?.setEmotionWithResetAfter(expression, 1000)
|
||||||
},
|
},
|
||||||
|
setVrmFrameHook(hook?: VrmFrameHook) {
|
||||||
|
vrmFrameHook.value = hook
|
||||||
|
},
|
||||||
scene: computed(() => vrm.value?.scene),
|
scene: computed(() => vrm.value?.scene),
|
||||||
lookAtUpdate(target: Vec3) {
|
lookAtUpdate(target: Vec3) {
|
||||||
idleEyeSaccades.instantUpdate(vrm.value, target)
|
idleEyeSaccades.instantUpdate(vrm.value, target)
|
||||||
|
|||||||
@@ -7,6 +7,7 @@
|
|||||||
* - Src of model is obtained from stage-ui via props, which is NOT a part of stage-ui-three package
|
* - Src of model is obtained from stage-ui via props, which is NOT a part of stage-ui-three package
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
import type { VRM } from '@pixiv/three-vrm'
|
||||||
import type { TresContext } from '@tresjs/core'
|
import type { TresContext } from '@tresjs/core'
|
||||||
import type { DirectionalLight, SphericalHarmonics3, Texture, WebGLRenderTarget } from 'three'
|
import type { DirectionalLight, SphericalHarmonics3, Texture, WebGLRenderTarget } from 'three'
|
||||||
|
|
||||||
@@ -198,6 +199,12 @@ const effectProps = {
|
|||||||
blendFunction: BlendFunction.SRC,
|
blendFunction: BlendFunction.SRC,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const vrmFrameHook = shallowRef<((vrm: VRM, delta: number) => void) | undefined>(undefined)
|
||||||
|
function applyVrmFrameHook() {
|
||||||
|
modelRef.value?.setVrmFrameHook(vrmFrameHook.value)
|
||||||
|
}
|
||||||
|
watch(modelRef, () => applyVrmFrameHook(), { immediate: true })
|
||||||
|
|
||||||
// === Directional Light ===
|
// === Directional Light ===
|
||||||
// TODO: wrap <TresDirectionalLight> to integrate all the below code
|
// TODO: wrap <TresDirectionalLight> to integrate all the below code
|
||||||
const sceneReady = ref(false)
|
const sceneReady = ref(false)
|
||||||
@@ -270,6 +277,10 @@ defineExpose({
|
|||||||
setExpression: (expression: string) => {
|
setExpression: (expression: string) => {
|
||||||
modelRef.value?.setExpression(expression)
|
modelRef.value?.setExpression(expression)
|
||||||
},
|
},
|
||||||
|
setVrmFrameHook: (hook?: (vrm: VRM, delta: number) => void) => {
|
||||||
|
vrmFrameHook.value = hook
|
||||||
|
applyVrmFrameHook()
|
||||||
|
},
|
||||||
canvasElement: () => {
|
canvasElement: () => {
|
||||||
return tresCanvasRef.value?.renderer.instance.domElement
|
return tresCanvasRef.value?.renderer.instance.domElement
|
||||||
},
|
},
|
||||||
|
|||||||
Generated
+39
@@ -72,9 +72,15 @@ catalogs:
|
|||||||
nanoid:
|
nanoid:
|
||||||
specifier: ^5.1.6
|
specifier: ^5.1.6
|
||||||
version: 5.1.6
|
version: 5.1.6
|
||||||
|
ofetch:
|
||||||
|
specifier: ^1.5.1
|
||||||
|
version: 1.5.1
|
||||||
posthog-js:
|
posthog-js:
|
||||||
specifier: 1.306.1
|
specifier: 1.306.1
|
||||||
version: 1.306.1
|
version: 1.306.1
|
||||||
|
tsx:
|
||||||
|
specifier: ^4.21.0
|
||||||
|
version: 4.21.0
|
||||||
uncrypto:
|
uncrypto:
|
||||||
specifier: ^0.1.3
|
specifier: ^0.1.3
|
||||||
version: 0.1.3
|
version: 0.1.3
|
||||||
@@ -739,6 +745,9 @@ importers:
|
|||||||
'@proj-airi/i18n':
|
'@proj-airi/i18n':
|
||||||
specifier: workspace:^
|
specifier: workspace:^
|
||||||
version: link:../../packages/i18n
|
version: link:../../packages/i18n
|
||||||
|
'@proj-airi/model-driver-mediapipe':
|
||||||
|
specifier: workspace:^
|
||||||
|
version: link:../../packages/model-driver-mediapipe
|
||||||
'@proj-airi/server-sdk':
|
'@proj-airi/server-sdk':
|
||||||
specifier: workspace:^
|
specifier: workspace:^
|
||||||
version: link:../../packages/server-sdk
|
version: link:../../packages/server-sdk
|
||||||
@@ -1273,6 +1282,31 @@ importers:
|
|||||||
specifier: ^1.3.0
|
specifier: ^1.3.0
|
||||||
version: 1.3.0
|
version: 1.3.0
|
||||||
|
|
||||||
|
packages/model-driver-mediapipe:
|
||||||
|
dependencies:
|
||||||
|
'@mediapipe/tasks-vision':
|
||||||
|
specifier: ^0.10.0
|
||||||
|
version: 0.10.21
|
||||||
|
'@pixiv/three-vrm':
|
||||||
|
specifier: ^3.4.4
|
||||||
|
version: 3.4.4(three@0.182.0)
|
||||||
|
es-toolkit:
|
||||||
|
specifier: 'catalog:'
|
||||||
|
version: 1.43.0
|
||||||
|
three:
|
||||||
|
specifier: ^0.182.0
|
||||||
|
version: 0.182.0
|
||||||
|
devDependencies:
|
||||||
|
'@types/three':
|
||||||
|
specifier: ^0.182.0
|
||||||
|
version: 0.182.0
|
||||||
|
ofetch:
|
||||||
|
specifier: 'catalog:'
|
||||||
|
version: 1.5.1
|
||||||
|
tsx:
|
||||||
|
specifier: 'catalog:'
|
||||||
|
version: 4.21.0
|
||||||
|
|
||||||
packages/pipelines-audio:
|
packages/pipelines-audio:
|
||||||
dependencies:
|
dependencies:
|
||||||
'@moeru/std':
|
'@moeru/std':
|
||||||
@@ -4972,6 +5006,9 @@ packages:
|
|||||||
markdown-it:
|
markdown-it:
|
||||||
optional: true
|
optional: true
|
||||||
|
|
||||||
|
'@mediapipe/tasks-vision@0.10.21':
|
||||||
|
resolution: {integrity: sha512-TuhKH+credq4zLksGbYrnvJ1aLIWMc5r0UHwzxzql4BHECJwIAoBR61ZrqwGOW6ZmSBIzU1t4VtKj8hbxFaKeA==}
|
||||||
|
|
||||||
'@modelcontextprotocol/sdk@1.25.1':
|
'@modelcontextprotocol/sdk@1.25.1':
|
||||||
resolution: {integrity: sha512-yO28oVFFC7EBoiKdAn+VqRm+plcfv4v0xp6osG/VsCB0NlPZWi87ajbCZZ8f/RvOFLEu7//rSRmuZZ7lMoe3gQ==}
|
resolution: {integrity: sha512-yO28oVFFC7EBoiKdAn+VqRm+plcfv4v0xp6osG/VsCB0NlPZWi87ajbCZZ8f/RvOFLEu7//rSRmuZZ7lMoe3gQ==}
|
||||||
engines: {node: '>=18'}
|
engines: {node: '>=18'}
|
||||||
@@ -17749,6 +17786,8 @@ snapshots:
|
|||||||
optionalDependencies:
|
optionalDependencies:
|
||||||
markdown-it: 14.1.0
|
markdown-it: 14.1.0
|
||||||
|
|
||||||
|
'@mediapipe/tasks-vision@0.10.21': {}
|
||||||
|
|
||||||
'@modelcontextprotocol/sdk@1.25.1(@cfworker/json-schema@4.1.1)(hono@4.11.3)(zod@4.2.1)':
|
'@modelcontextprotocol/sdk@1.25.1(@cfworker/json-schema@4.1.1)(hono@4.11.3)(zod@4.2.1)':
|
||||||
dependencies:
|
dependencies:
|
||||||
'@hono/node-server': 1.19.7(hono@4.11.3)
|
'@hono/node-server': 1.19.7(hono@4.11.3)
|
||||||
|
|||||||
+2
-2
@@ -1,7 +1,6 @@
|
|||||||
catalogMode: prefer
|
catalogMode: prefer
|
||||||
|
|
||||||
shellEmulator: true
|
shellEmulator: true
|
||||||
|
|
||||||
packages:
|
packages:
|
||||||
- packages/**
|
- packages/**
|
||||||
- plugins/**
|
- plugins/**
|
||||||
@@ -20,7 +19,6 @@ overrides:
|
|||||||
safer-buffer: npm:@nolyfill/safer-buffer@^1.0.44
|
safer-buffer: npm:@nolyfill/safer-buffer@^1.0.44
|
||||||
side-channel: npm:@nolyfill/side-channel@^1.0.44
|
side-channel: npm:@nolyfill/side-channel@^1.0.44
|
||||||
string.prototype.matchall: npm:@nolyfill/string.prototype.matchall@^1.0.44
|
string.prototype.matchall: npm:@nolyfill/string.prototype.matchall@^1.0.44
|
||||||
|
|
||||||
catalog:
|
catalog:
|
||||||
'@guiiai/logg': ^1.2.11
|
'@guiiai/logg': ^1.2.11
|
||||||
'@moeru/eslint-config': 0.1.0-beta.14
|
'@moeru/eslint-config': 0.1.0-beta.14
|
||||||
@@ -44,7 +42,9 @@ catalog:
|
|||||||
es-toolkit: 1.43.0
|
es-toolkit: 1.43.0
|
||||||
injeca: ^0.1.5
|
injeca: ^0.1.5
|
||||||
nanoid: ^5.1.6
|
nanoid: ^5.1.6
|
||||||
|
ofetch: ^1.5.1
|
||||||
posthog-js: 1.306.1
|
posthog-js: 1.306.1
|
||||||
|
tsx: ^4.21.0
|
||||||
uncrypto: ^0.1.3
|
uncrypto: ^0.1.3
|
||||||
unplugin-info: 1.2.4
|
unplugin-info: 1.2.4
|
||||||
xsschema: ^0.4.0-beta.12
|
xsschema: ^0.4.0-beta.12
|
||||||
|
|||||||
Reference in New Issue
Block a user