diff --git a/.gitignore b/.gitignore index a9a6a5d2d..ef9b570e9 100644 --- a/.gitignore +++ b/.gitignore @@ -110,3 +110,6 @@ twitter-session.json result* # Nix develop outputs outputs/ + +# MediaPipe task assets +packages/model-driver-mediapipe/tasks/assets/* diff --git a/apps/stage-web/package.json b/apps/stage-web/package.json index 2abb0e13f..cdeea2f5a 100644 --- a/apps/stage-web/package.json +++ b/apps/stage-web/package.json @@ -28,6 +28,7 @@ "@proj-airi/font-cjkfonts-allseto": "workspace:^", "@proj-airi/font-xiaolai": "workspace:^", "@proj-airi/i18n": "workspace:^", + "@proj-airi/model-driver-mediapipe": "workspace:^", "@proj-airi/server-sdk": "workspace:^", "@proj-airi/stage-ui": "workspace:^", "@proj-airi/stage-ui-three": "workspace:^", diff --git a/apps/stage-web/src/pages/devtools/model-driver-mediapipe.vue b/apps/stage-web/src/pages/devtools/model-driver-mediapipe.vue new file mode 100644 index 000000000..2429a4ab5 --- /dev/null +++ b/apps/stage-web/src/pages/devtools/model-driver-mediapipe.vue @@ -0,0 +1,485 @@ + + + + + +meta: + layout: plain + diff --git a/apps/stage-web/src/pages/settings/system/developer.vue b/apps/stage-web/src/pages/settings/system/developer.vue index f8c9750ff..adf8109af 100644 --- a/apps/stage-web/src/pages/settings/system/developer.vue +++ b/apps/stage-web/src/pages/settings/system/developer.vue @@ -80,6 +80,12 @@ const menu = computed(() => [ icon: 'i-solar:sledgehammer-bold-duotone', to: '/devtools/performance-playground', }, + { + title: 'MediaPipe Workshop', + description: 'Single-person mocap playground (MediaPipe backend) with scheduling knobs', + icon: 'i-solar:sledgehammer-bold-duotone', + to: '/devtools/model-driver-mediapipe', + }, ]) diff --git a/cspell.config.yaml b/cspell.config.yaml index 571fe398c..3c7b86a53 100644 --- a/cspell.config.yaml +++ b/cspell.config.yaml @@ -162,6 +162,7 @@ words: - matchall - mdit - mediabunny + - mediapipe - MeshToonMaterial - micvad - mineflayer diff --git a/packages/model-driver-mediapipe/AGENTS.md b/packages/model-driver-mediapipe/AGENTS.md new file mode 100644 index 000000000..11d9b7f9b --- /dev/null +++ b/packages/model-driver-mediapipe/AGENTS.md @@ -0,0 +1,35 @@ +# model-driver-mediapipe Agent Notes + +Scope: `packages/model-driver-mediapipe/**` + +## Intent + +This package is an experimental single-person mocap pipeline for stage-web devtools. + +`camera frame` → `@mediapipe/tasks-vision` → `PerceptionState` → `overlay` + +## Style / Conventions + +- Prefer functional programming (FP) and pure functions where practical. + - Use factory functions + closures for stateful modules (example: `createMocapEngine()`). + - Avoid classes unless extending browser APIs or required by external libraries. +- Keep the backend boundary clean: + - Engine/scheduler should not import `@mediapipe/tasks-vision`. + - MediaPipe specifics live under `src/backends/`. +- Keep types stable and narrow: + - Stage consumers depend on `src/types.ts` as the contract. + - New fields should be optional and backwards compatible. + +## Key Files + +- `src/types.ts`: middle-layer contract (`PerceptionState`, config types) +- `src/engine.ts`: scheduling + dropped-frame policy + partial merge (FP) +- `src/backends/mediapipe.ts`: MediaPipe Tasks Vision adapter (sync `detectForVideo`) +- `src/overlay.ts`: debug overlay renderer (points + connectors) +- `references/tasks-vision-api.md`: minimal upstream API notes + +## Performance Notes + +- `detectForVideo()` is synchronous; avoid blocking UI: + - Engine drops frames when backend reports `isBusy()`. + - Scheduler controls per-task rates (`hz`) to cap work. diff --git a/packages/model-driver-mediapipe/README.md b/packages/model-driver-mediapipe/README.md new file mode 100644 index 000000000..cd898d617 --- /dev/null +++ b/packages/model-driver-mediapipe/README.md @@ -0,0 +1,35 @@ +# @proj-airi/model-driver-mediapipe + +Single-person motion capture workshop package. + +**Goal** + +Provide a minimal closed loop that stage-web can consume: + +`camera frame` → `MediaPipe Tasks Vision` → `PerceptionState` → `canvas overlay` + +**Where to try it** + +- Devtools page: `apps/stage-web/src/pages/devtools/model-driver-mediapipe.vue` +- Menu entry: Settings → System → Developer → “MediaPipe Workshop” + +**Key files** + +- `packages/model-driver-mediapipe/src/types.ts`: middle-layer contract (`PerceptionState`) +- `packages/model-driver-mediapipe/src/engine.ts`: scheduler + dropped-frame policy + state merge +- `packages/model-driver-mediapipe/src/backends/mediapipe.ts`: `@mediapipe/tasks-vision` integration +- `packages/model-driver-mediapipe/src/utils/overlay.ts`: canvas overlay renderer + +**Backend assumptions** + +- Single person (`maxPeople: 1`) +- Running mode: `VIDEO` +- Landmarks are normalized (`x`/`y` in `[0..1]`) and drawn onto the overlay canvas + +**Docs** + +Keep upstream docs/snippets in `packages/model-driver-mediapipe/references/`. + +The minimal API surface this package uses is summarized in: + +- `packages/model-driver-mediapipe/references/tasks-vision-api.md` diff --git a/packages/model-driver-mediapipe/package.json b/packages/model-driver-mediapipe/package.json new file mode 100644 index 000000000..083f0f097 --- /dev/null +++ b/packages/model-driver-mediapipe/package.json @@ -0,0 +1,36 @@ +{ + "name": "@proj-airi/model-driver-mediapipe", + "type": "module", + "private": true, + "description": "MediaPipe motion capture workshop (experimental) for Project AIRI", + "author": { + "name": "Moeru AI Project AIRI Team", + "email": "airi@moeru.ai", + "url": "https://github.com/moeru-ai" + }, + "license": "MIT", + "repository": { + "type": "git", + "url": "https://github.com/moeru-ai/airi.git", + "directory": "packages/model-driver-mediapipe" + }, + "exports": { + ".": "./src/index.ts", + "./types": "./src/types.ts" + }, + "scripts": { + "typecheck": "tsc --noEmit", + "postinstall": "tsx ./tasks/prepare-tasks.ts" + }, + "dependencies": { + "@mediapipe/tasks-vision": "^0.10.0", + "@pixiv/three-vrm": "^3.4.4", + "es-toolkit": "catalog:", + "three": "^0.182.0" + }, + "devDependencies": { + "@types/three": "^0.182.0", + "ofetch": "catalog:", + "tsx": "catalog:" + } +} diff --git a/packages/model-driver-mediapipe/references/README.md b/packages/model-driver-mediapipe/references/README.md new file mode 100644 index 000000000..c3627d9e2 --- /dev/null +++ b/packages/model-driver-mediapipe/references/README.md @@ -0,0 +1,3 @@ +# References + +Put MediaPipe Tasks Vision API docs / snippets / notes here so the implementation can follow the exact API surface you’re using. diff --git a/packages/model-driver-mediapipe/references/tasks-vision-api.md b/packages/model-driver-mediapipe/references/tasks-vision-api.md new file mode 100644 index 000000000..fdc52ee86 --- /dev/null +++ b/packages/model-driver-mediapipe/references/tasks-vision-api.md @@ -0,0 +1,44 @@ +# MediaPipe Tasks Vision (minimal API notes) + +This package targets `@mediapipe/tasks-vision` (web, JS/TS). + +## Core init + +```ts +import { FilesetResolver, PoseLandmarker } from '@mediapipe/tasks-vision' + +const vision = await FilesetResolver.forVisionTasks(wasmRoot) +const pose = await PoseLandmarker.createFromOptions(vision, { + baseOptions: { modelAssetPath: poseModelUrl }, + runningMode: 'VIDEO', + numPoses: 1, +}) +``` + +## Video inference + +MediaPipe Tasks Vision `detectForVideo()` runs synchronously and can block the main thread. + +```ts +const nowMs = performance.now() +const res = pose.detectForVideo(videoEl, nowMs) +``` + +## Result shapes (single-person usage) + +### Pose + +- `res.landmarks`: `NormalizedLandmark[][]` (take `[0]` for single person) +- each landmark includes `{ x, y, z, visibility?, presence? }` with `x/y` in `[0..1]` + +### Hands + +- `res.landmarks`: `NormalizedLandmark[][]` (each entry is 21 landmarks for one hand) +- `res.handedness`: `Category[][]` aligned with `landmarks` + - `handedness[i][0].categoryName` is typically `'Left' | 'Right'` + - `handedness[i][0].score` is confidence + +### Face (optional) + +- `res.faceLandmarks`: `NormalizedLandmark[][]` (468 landmarks, heavy) +- For the workshop we treat this as presence-only by default. diff --git a/packages/model-driver-mediapipe/src/backends/mediapipe.ts b/packages/model-driver-mediapipe/src/backends/mediapipe.ts new file mode 100644 index 000000000..f971bf67d --- /dev/null +++ b/packages/model-driver-mediapipe/src/backends/mediapipe.ts @@ -0,0 +1,158 @@ +import type { + Category, + FaceLandmarker, + FaceLandmarkerResult, + HandLandmarker, + HandLandmarkerResult, + Landmark, + NormalizedLandmark, + PoseLandmarker, + PoseLandmarkerResult, +} from '@mediapipe/tasks-vision' + +import type { MocapBackend, MocapConfig, MocapJob, PerceptionPartial, VisionTaskModule, VisionTaskWasmFileset } from '../types' + +import { Semaphore } from 'es-toolkit' + +import { visionTaskAssets, visionTaskWasmRoot } from '../../tasks/tasks' + +export function createMediaPipeBackend(): MocapBackend { + const semaphore = new Semaphore(1) + let busy = false + let config: MocapConfig | undefined + let tasksVision: VisionTaskModule | undefined + let vision: VisionTaskWasmFileset | undefined + + let poseLandmarker: PoseLandmarker | undefined + let handLandmarker: HandLandmarker | undefined + let faceLandmarker: FaceLandmarker | undefined + + async function init(nextConfig: MocapConfig) { + config = nextConfig + + if (!tasksVision) + tasksVision = await import('@mediapipe/tasks-vision') + + if (!vision) { + const { FilesetResolver } = tasksVision + vision = await FilesetResolver.forVisionTasks(visionTaskWasmRoot) + } + } + + function isBusy() { + return busy + } + + async function ensurePoseLandmarker() { + if (poseLandmarker) + return poseLandmarker + + const { PoseLandmarker } = tasksVision! + poseLandmarker = await PoseLandmarker.createFromOptions(vision!, { + baseOptions: { modelAssetPath: visionTaskAssets.pose }, + runningMode: 'VIDEO', + numPoses: 1, + }) + + return poseLandmarker + } + + async function ensureHandLandmarker() { + if (handLandmarker) + return handLandmarker + + const { HandLandmarker } = tasksVision! + handLandmarker = await HandLandmarker.createFromOptions(vision!, { + baseOptions: { modelAssetPath: visionTaskAssets.hands }, + runningMode: 'VIDEO', + numHands: 2, + }) + + return handLandmarker + } + + async function ensureFaceLandmarker() { + if (faceLandmarker) + return faceLandmarker + + const { FaceLandmarker } = tasksVision! + faceLandmarker = await FaceLandmarker.createFromOptions(vision!, { + baseOptions: { modelAssetPath: visionTaskAssets.face }, + runningMode: 'VIDEO', + numFaces: 1, + }) + + return faceLandmarker + } + + async function run(frame: TexImageSource, jobs: MocapJob[], nowMs: number): Promise { + if (!config) + throw new Error('MediaPipe backend not initialized (call init() first)') + + await semaphore.acquire() + busy = true + try { + const partial: PerceptionPartial = {} + + for (const job of jobs) { + if (!config.enabled[job]) + continue + + if (job === 'pose') { + const landmarker = await ensurePoseLandmarker() + const res: PoseLandmarkerResult = landmarker.detectForVideo(frame, nowMs) + const firstPose: NormalizedLandmark[] = res.landmarks[0] ?? [] + const firstWorld: Landmark[] = res.worldLandmarks[0] ?? [] + partial.pose = { + landmarks2d: firstPose, + worldLandmarks: firstWorld.map(p => ({ + x: p.x, + y: p.y, + z: p.z, + visibility: p.visibility, + })), + } + } + else if (job === 'hands') { + const landmarker = await ensureHandLandmarker() + const res: HandLandmarkerResult = landmarker.detectForVideo(frame, nowMs) + const landmarks: NormalizedLandmark[][] = res.landmarks ?? [] + const handedness: Category[][] = res.handedness ?? [] + + partial.hands = landmarks.map((lm, i) => { + const mostLikelyCategory = handedness[i]?.[0] + const categoryName = mostLikelyCategory?.categoryName + const score = mostLikelyCategory?.score + const handed = categoryName === 'Left' || categoryName === 'Right' ? categoryName : 'Right' + return { + handedness: handed, + landmarks2d: lm, + score, + } + }) + } + else if (job === 'face') { + const landmarker = await ensureFaceLandmarker() + const res: FaceLandmarkerResult = landmarker.detectForVideo(frame, nowMs) + const firstFace: NormalizedLandmark[] = res.faceLandmarks?.[0] ?? [] + partial.face = { + hasFace: firstFace.length > 0, + landmarks2d: firstFace, + } + } + } + + return partial + } + finally { + busy = false + semaphore.release() + } + } + + return { + init, + isBusy, + run, + } +} diff --git a/packages/model-driver-mediapipe/src/engine.ts b/packages/model-driver-mediapipe/src/engine.ts new file mode 100644 index 000000000..7a433564b --- /dev/null +++ b/packages/model-driver-mediapipe/src/engine.ts @@ -0,0 +1,156 @@ +import type { FrameSource, MocapBackend, MocapConfig, MocapEngine, MocapJob, PerceptionPartial, PerceptionState } from './types' + +export function createStats() { + let lastTs = 0 + let smoothedFps = 0 + + function tick(nowMs: number) { + if (!lastTs) { + lastTs = nowMs + smoothedFps = 0 + return 0 + } + + const dt = nowMs - lastTs + lastTs = nowMs + + if (dt <= 0) + return smoothedFps + + const fps = 1000 / dt + smoothedFps = smoothedFps ? (smoothedFps * 0.9 + fps * 0.1) : fps + + return smoothedFps + } + + return { tick } +} + +export function createScheduler(initialConfig: MocapConfig) { + let config = initialConfig + const lastRun: Record = { pose: 0, hands: 0, face: 0 } + + function updateConfig(next: MocapConfig) { + config = next + } + + function plan(nowMs: number): MocapJob[] { + const jobs: MocapJob[] = [] + + for (const job of ['pose', 'hands', 'face'] as const) { + if (!config.enabled[job]) + continue + + const hz = config.hz[job] + if (!hz || hz <= 0) + continue + + if (nowMs - lastRun[job] >= (1000 / hz)) + jobs.push(job) + } + + for (const j of jobs) + lastRun[j] = nowMs + + return jobs + } + + return { + plan, + updateConfig, + } +} + +export function createMocapEngine(backend: MocapBackend, initialConfig: MocapConfig): MocapEngine { + let config = initialConfig + const scheduler = createScheduler(config) + const stats = createStats() + + let running = false + let droppedFrames = 0 + let lastPartial: PerceptionPartial = {} + let rafId: number | undefined + + async function init() { + await backend.init(config) + } + + function updateConfig(next: MocapConfig) { + config = next + scheduler.updateConfig(next) + } + + function resetState() { + lastPartial = {} + droppedFrames = 0 + } + + function start( + source: FrameSource, + onState: (state: PerceptionState) => void, + options?: { onError?: (error: unknown) => void }, + ) { + running = true + resetState() + + const tick = async () => { + try { + if (!running) + return + + const frame = source.getFrame() + const now = performance.now() + + // Skip this frame if the backend is still busy. + if (backend.isBusy()) { + droppedFrames++ + rafId = requestAnimationFrame(tick) + return + } + + const jobs = scheduler.plan(now) + const t0 = performance.now() + const partial = jobs.length > 0 ? await backend.run(frame, jobs, now) : {} + const latencyMs = performance.now() - t0 + + lastPartial = { ...lastPartial, ...partial } + + onState({ + t: now, + ...lastPartial, + quality: { + fps: stats.tick(now), + latencyMs, + droppedFrames, + backend: 'mediapipe', + mode: 'split-tasks', + }, + }) + + rafId = requestAnimationFrame(tick) + } + catch (err) { + // Stop the loop to avoid spamming errors; consumers can restart. + stop() + options?.onError?.(err) + } + } + + rafId = requestAnimationFrame(tick) + } + + function stop() { + running = false + if (rafId != null) + cancelAnimationFrame(rafId) + rafId = undefined + } + + return { + init, + start, + stop, + updateConfig, + resetState, + } +} diff --git a/packages/model-driver-mediapipe/src/index.ts b/packages/model-driver-mediapipe/src/index.ts new file mode 100644 index 000000000..5bcd6c673 --- /dev/null +++ b/packages/model-driver-mediapipe/src/index.ts @@ -0,0 +1,5 @@ +export * from './backends/mediapipe' +export * from './engine' +export * from './three' +export * from './types' +export * from './utils/overlay' diff --git a/packages/model-driver-mediapipe/src/three/apply-pose-to-vrm.ts b/packages/model-driver-mediapipe/src/three/apply-pose-to-vrm.ts new file mode 100644 index 000000000..6d2c3de79 --- /dev/null +++ b/packages/model-driver-mediapipe/src/three/apply-pose-to-vrm.ts @@ -0,0 +1,361 @@ +import type { VRM } from '@pixiv/three-vrm' + +import type { VrmPoseDirections, VrmPoseTarget, VrmPoseTargets } from './pose-to-vrm' + +import { Matrix4, Quaternion, Vector3 } from 'three' + +export interface VrmPoseApplyOptions { + /** + * Slerp factor per frame in [0..1]. Higher = snappier, lower = smoother. + */ + alpha?: number + /** + * Reject sudden flips: if the target direction is close to the opposite of the + * current bone direction (in world space), skip this update. + * + * `minDot` in [-1..1]. Example: `-0.2` rejects angles > ~101°. + */ + minDotBeforeReject?: number + /** + * Similar to `minDotBeforeReject`, but applied to the pole vector when using + * pole-based orientation. Helps avoid 180° roll/yaw flips when the pole sign + * becomes ambiguous. + */ + minPoleDotBeforeReject?: number +} + +type BoneKey = keyof VrmPoseDirections + +interface BoneChain { + bone: string + childCandidates: readonly string[] +} + +const DEFAULT_ALPHA = 0.35 + +const CHAINS: Readonly> = { + hips: { + bone: 'hips', + childCandidates: ['spine'], + }, + spine: { + bone: 'spine', + childCandidates: ['chest', 'upperChest', 'neck'], + }, + chest: { + bone: 'chest', + childCandidates: ['upperChest', 'neck'], + }, + leftShoulder: { + bone: 'leftShoulder', + childCandidates: ['leftUpperArm'], + }, + rightShoulder: { + bone: 'rightShoulder', + childCandidates: ['rightUpperArm'], + }, + leftUpperArm: { + bone: 'leftUpperArm', + childCandidates: ['leftLowerArm'], + }, + leftLowerArm: { + bone: 'leftLowerArm', + childCandidates: ['leftHand'], + }, + rightUpperArm: { + bone: 'rightUpperArm', + childCandidates: ['rightLowerArm'], + }, + rightLowerArm: { + bone: 'rightLowerArm', + childCandidates: ['rightHand'], + }, + leftUpperLeg: { + bone: 'leftUpperLeg', + childCandidates: ['leftLowerLeg'], + }, + leftLowerLeg: { + bone: 'leftLowerLeg', + childCandidates: ['leftFoot'], + }, + rightUpperLeg: { + bone: 'rightUpperLeg', + childCandidates: ['rightLowerLeg'], + }, + rightLowerLeg: { + bone: 'rightLowerLeg', + childCandidates: ['rightFoot'], + }, +} as const + +const POLE_KEYS: ReadonlySet = new Set(Object.keys(CHAINS) as BoneKey[]) +const LIMB_POLE_KEYS: ReadonlySet = new Set([ + 'leftUpperArm', + 'rightUpperArm', + 'leftUpperLeg', + 'rightUpperLeg', +]) + +function isFiniteVec3(v: { x: number, y: number, z?: number } | undefined): v is { x: number, y: number, z?: number } { + if (!v) + return false + return Number.isFinite(v.x) && Number.isFinite(v.y) && Number.isFinite(v.z ?? 0) +} + +function firstExistingBone(vrm: VRM, candidates: readonly string[]) { + for (const name of candidates) { + const node = vrm.humanoid?.getNormalizedBoneNode(name as any) + if (node) + return node + } + return null +} + +function grandchildCandidatesFor(key: BoneKey): readonly string[] { + if (key === 'leftUpperArm') + return ['leftHand'] + if (key === 'rightUpperArm') + return ['rightHand'] + if (key === 'leftUpperLeg') + return ['leftFoot'] + if (key === 'rightUpperLeg') + return ['rightFoot'] + return [] +} + +function orthonormalizePole(dir: Vector3, pole: Vector3): Vector3 | null { + const poleOrtho = pole.clone().addScaledVector(dir, -pole.dot(dir)) + if (poleOrtho.lengthSq() <= 1e-12) + return null + return poleOrtho.normalize() +} + +export function createVrmPoseApplier(options?: VrmPoseApplyOptions) { + const alpha = options?.alpha ?? DEFAULT_ALPHA + const minDotBeforeReject = options?.minDotBeforeReject ?? -0.2 + const minPoleDotBeforeReject = options?.minPoleDotBeforeReject ?? -0.2 + + const restDirLocal: Partial> = {} + const restPoleLocal: Partial> = {} + const lastTargetDirWorld: Partial> = {} + const lastTargetPoleWorld: Partial> = {} + + const tmpBoneWorldQ = new Quaternion() + const tmpParentWorldQ = new Quaternion() + const tmpDeltaQ = new Quaternion() + const tmpNewWorldQ = new Quaternion() + const tmpNewLocalQ = new Quaternion() + + const tmpRestM = new Matrix4() + const tmpTargetM = new Matrix4() + const tmpRotM = new Matrix4() + + const tmpBoneWorldPos = new Vector3() + const tmpChildWorldPos = new Vector3() + const tmpGrandWorldPos = new Vector3() + const tmpDirWorld = new Vector3() + const tmpDirLocal = new Vector3() + const tmpCurrentDirWorld = new Vector3() + const tmpTargetDirWorld = new Vector3() + const tmpTargetPoleWorld = new Vector3() + const tmpRestDirLocal = new Vector3() + const tmpRestPoleLocal = new Vector3() + const tmpRestYLocal = new Vector3() + const tmpTargetYWorld = new Vector3() + const tmpWorldForward = new Vector3(0, 0, -1) + + function ensureRestDirection(vrm: VRM, key: BoneKey): Vector3 | null { + const existing = restDirLocal[key] + if (existing) + return existing + + const chain = CHAINS[key] + const bone = vrm.humanoid?.getNormalizedBoneNode(chain.bone as any) + if (!bone) + return null + + const child = firstExistingBone(vrm, chain.childCandidates) + if (!child) + return null + + bone.updateMatrixWorld(true) + child.updateMatrixWorld(true) + + bone.getWorldQuaternion(tmpBoneWorldQ) + bone.getWorldPosition(tmpBoneWorldPos) + child.getWorldPosition(tmpChildWorldPos) + + tmpDirWorld.copy(tmpChildWorldPos).sub(tmpBoneWorldPos).normalize() + tmpDirLocal.copy(tmpDirWorld).applyQuaternion(tmpBoneWorldQ.clone().invert()).normalize() + + const stored = tmpDirLocal.clone() + restDirLocal[key] = stored + return stored + } + + function ensureRestPole(vrm: VRM, key: BoneKey): Vector3 | null { + if (!POLE_KEYS.has(key)) + return null + + const existing = restPoleLocal[key] + if (existing) + return existing + + const chain = CHAINS[key] + const bone = vrm.humanoid?.getNormalizedBoneNode(chain.bone as any) + if (!bone) + return null + + bone.updateMatrixWorld(true) + bone.getWorldQuaternion(tmpBoneWorldQ) + + if (LIMB_POLE_KEYS.has(key)) { + const child = firstExistingBone(vrm, chain.childCandidates) + if (!child) + return null + + const grandCandidates = grandchildCandidatesFor(key) + const grand = grandCandidates.length ? firstExistingBone(vrm, grandCandidates) : null + if (!grand) + return null + + child.updateMatrixWorld(true) + grand.updateMatrixWorld(true) + + bone.getWorldPosition(tmpBoneWorldPos) + child.getWorldPosition(tmpChildWorldPos) + grand.getWorldPosition(tmpGrandWorldPos) + + const dir1 = tmpChildWorldPos.clone().sub(tmpBoneWorldPos) + const dir2 = tmpGrandWorldPos.clone().sub(tmpChildWorldPos) + tmpDirWorld.copy(dir1).cross(dir2) + if (tmpDirWorld.lengthSq() <= 1e-12) + return null + tmpDirWorld.normalize() + + tmpDirLocal.copy(tmpDirWorld).applyQuaternion(tmpBoneWorldQ.clone().invert()).normalize() + const stored = tmpDirLocal.clone() + restPoleLocal[key] = stored + return stored + } + + // For torso: use "world forward" as rest pole, transformed into bone local. + tmpDirLocal.copy(tmpWorldForward).applyQuaternion(tmpBoneWorldQ.clone().invert()).normalize() + const stored = tmpDirLocal.clone() + restPoleLocal[key] = stored + return stored + } + + function applyOne(vrm: VRM, key: BoneKey, target: VrmPoseTarget) { + const chain = CHAINS[key] + const bone = vrm.humanoid?.getNormalizedBoneNode(chain.bone as any) + if (!bone) + return + + const rest = ensureRestDirection(vrm, key) + if (!rest) + return + + tmpTargetDirWorld.set(target.dir.x, target.dir.y, target.dir.z ?? 0) + if (tmpTargetDirWorld.lengthSq() <= 1e-12) + return + tmpTargetDirWorld.normalize() + + // Reject near-180° instant flips based on previous target (not current bone pose). + // Using the current bone direction is unreliable when tracking reacquires or when + // the bind pose differs from the mocap space. + const prevDir = lastTargetDirWorld[key] + if (prevDir && prevDir.dot(tmpTargetDirWorld) < minDotBeforeReject) + return + + bone.updateMatrixWorld(true) + bone.getWorldQuaternion(tmpBoneWorldQ) + + const parent = bone.parent + if (parent) { + parent.updateMatrixWorld(true) + parent.getWorldQuaternion(tmpParentWorldQ) + } + else { + tmpParentWorldQ.identity() + } + + const restPole = target.pole ? ensureRestPole(vrm, key) : null + const usePole = !!(target.pole && restPole) + + if (usePole) { + tmpRestDirLocal.copy(rest).normalize() + tmpRestPoleLocal.copy(restPole!).normalize() + tmpRestPoleLocal.addScaledVector(tmpRestDirLocal, -tmpRestPoleLocal.dot(tmpRestDirLocal)).normalize() + tmpRestYLocal.copy(tmpRestPoleLocal).cross(tmpRestDirLocal).normalize() + + tmpTargetPoleWorld.set(target.pole!.x, target.pole!.y, target.pole!.z ?? 0) + if (tmpTargetPoleWorld.lengthSq() <= 1e-12) + return + tmpTargetPoleWorld.normalize() + const poleOrtho = orthonormalizePole(tmpTargetDirWorld, tmpTargetPoleWorld) + if (!poleOrtho) + return + tmpTargetPoleWorld.copy(poleOrtho) + tmpTargetYWorld.copy(tmpTargetPoleWorld).cross(tmpTargetDirWorld).normalize() + + // Reject near-180° pole flips based on previous target pole. + const prevPole = lastTargetPoleWorld[key] + if (prevPole && prevPole.dot(tmpTargetPoleWorld) < minPoleDotBeforeReject) + return + + tmpRestM.makeBasis(tmpRestDirLocal, tmpRestYLocal, tmpRestPoleLocal) + tmpTargetM.makeBasis(tmpTargetDirWorld, tmpTargetYWorld, tmpTargetPoleWorld) + + tmpRotM.copy(tmpTargetM).multiply(tmpRestM.clone().invert()) + tmpNewWorldQ.setFromRotationMatrix(tmpRotM) + tmpNewLocalQ.copy(tmpParentWorldQ).invert().multiply(tmpNewWorldQ) + } + else { + tmpCurrentDirWorld.copy(rest).applyQuaternion(tmpBoneWorldQ).normalize() + tmpDeltaQ.setFromUnitVectors(tmpCurrentDirWorld, tmpTargetDirWorld) + tmpNewWorldQ.copy(tmpDeltaQ).multiply(tmpBoneWorldQ) + tmpNewLocalQ.copy(tmpParentWorldQ).invert().multiply(tmpNewWorldQ) + } + + if (alpha >= 1) + bone.quaternion.copy(tmpNewLocalQ) + else + bone.quaternion.slerp(tmpNewLocalQ, alpha) + + // Update last targets only after a successful apply. + lastTargetDirWorld[key] = tmpTargetDirWorld.clone() + if (usePole) + lastTargetPoleWorld[key] = tmpTargetPoleWorld.clone() + else + delete lastTargetPoleWorld[key] + } + + function applyPoseDirectionsToVrm(vrm: VRM, directions: VrmPoseDirections) { + if (!vrm.humanoid) + return + + (Object.keys(CHAINS) as BoneKey[]).forEach((key) => { + const d = directions[key] + if (!isFiniteVec3(d)) + return + applyOne(vrm, key, { dir: d }) + }) + } + + function applyPoseTargetsToVrm(vrm: VRM, targets: VrmPoseTargets) { + if (!vrm.humanoid) + return + + (Object.keys(CHAINS) as BoneKey[]).forEach((key) => { + const t = targets[key] + if (!t || !isFiniteVec3(t.dir)) + return + if (t.pole && !isFiniteVec3(t.pole)) + return + applyOne(vrm, key, t) + }) + } + + return { applyPoseDirectionsToVrm, applyPoseTargetsToVrm } +} diff --git a/packages/model-driver-mediapipe/src/three/index.ts b/packages/model-driver-mediapipe/src/three/index.ts new file mode 100644 index 000000000..4708c3cbb --- /dev/null +++ b/packages/model-driver-mediapipe/src/three/index.ts @@ -0,0 +1,2 @@ +export * from './apply-pose-to-vrm' +export * from './pose-to-vrm' diff --git a/packages/model-driver-mediapipe/src/three/pose-to-vrm.ts b/packages/model-driver-mediapipe/src/three/pose-to-vrm.ts new file mode 100644 index 000000000..758c93c46 --- /dev/null +++ b/packages/model-driver-mediapipe/src/three/pose-to-vrm.ts @@ -0,0 +1,347 @@ +import type { Vector3Like } from 'three' + +import type { PoseState } from '../types' + +export interface PoseToVrmOptions { + /** + * Axis remap from MediaPipe world to three/VRM-ish space. + * This is a pragmatic default; adjust if you see mirrored or inverted motion. + */ + axis?: { + x: 1 | -1 + y: 1 | -1 + z: 1 | -1 + } + + /** + * Confidence gating based on MediaPipe pose landmark `visibility` / `presence`. + * When a landmark is not confident (e.g. off-screen), we skip emitting targets that depend on it. + */ + confidence?: { + /** + * Min `visibility` in [0..1]. Only enforced when the field exists on the landmark. + */ + minVisibility?: number + /** + * Min `presence` in [0..1]. Only enforced when the field exists on the landmark. + */ + minPresence?: number + } + + /** + * Use previous frame information to avoid sudden 180° flips caused by ambiguous poles. + */ + stabilize?: { + previousTargets?: VrmPoseTargets + previousForward?: Vector3Like + } +} + +export type VrmPoseDirections = Partial> + +export interface VrmPoseTarget { + dir: Vector3Like + pole?: Vector3Like +} + +export type VrmPoseTargets = Partial> + +const DEFAULT_AXIS = { x: 1 as const, y: 1 as const, z: 1 as const } +const DEFAULT_MIN_VISIBILITY = 0.5 +const DEFAULT_MIN_PRESENCE = 0 + +// TODO: Consider consolidating these vector helpers into a shared math utility if more drivers need them. +function vSub(a: Vector3Like, b: Vector3Like): Vector3Like { + return { x: a.x - b.x, y: a.y - b.y, z: (a.z ?? 0) - (b.z ?? 0) } +} + +function vAdd(a: Vector3Like, b: Vector3Like): Vector3Like { + return { x: a.x + b.x, y: a.y + b.y, z: (a.z ?? 0) + (b.z ?? 0) } +} + +function vScale(v: Vector3Like, s: number): Vector3Like { + return { x: v.x * s, y: v.y * s, z: (v.z ?? 0) * s } +} + +function vLen(v: Vector3Like): number { + return Math.hypot(v.x, v.y, v.z ?? 0) +} + +function vNormalize(v: Vector3Like): Vector3Like | null { + const len = vLen(v) + if (!Number.isFinite(len) || len <= 1e-6) + return null + return vScale(v, 1 / len) +} + +function vRemapAxis(v: Vector3Like, axis: { x: 1 | -1, y: 1 | -1, z: 1 | -1 }): Vector3Like { + return { x: v.x * axis.x, y: v.y * axis.y, z: (v.z ?? 0) * axis.z } +} + +function vCross(a: Vector3Like, b: Vector3Like): Vector3Like { + const az = a.z ?? 0 + const bz = b.z ?? 0 + return { + x: a.y * bz - az * b.y, + y: az * b.x - a.x * bz, + z: a.x * b.y - a.y * b.x, + } +} + +function vDot(a: Vector3Like, b: Vector3Like): number { + return a.x * b.x + a.y * b.y + (a.z ?? 0) * (b.z ?? 0) +} + +function vNeg(v: Vector3Like): Vector3Like { + return { x: -v.x, y: -v.y, z: -(v.z ?? 0) } +} + +function safePole(dir: Vector3Like, pole: Vector3Like, threshold = 0.85): Vector3Like | null { + const d = Math.abs(vDot(dir, pole)) + if (!Number.isFinite(d)) + return null + return d > threshold ? null : pole +} + +function get(points: Vector3Like[], index: number): Vector3Like | null { + const p = points[index] + if (!p) + return null + if (!Number.isFinite(p.x) || !Number.isFinite(p.y)) + return null + return { x: p.x, y: p.y, z: p.z ?? 0 } +} + +// NOTICE: mediapipe doesn't provide this type correctly, so we define it here. +interface LandmarkWithPresence { presence?: number } +interface LandmarkWithVisibility { visibility?: number } + +function getOptionalPresence(landmark: unknown): number | undefined { + if (landmark && typeof landmark === 'object' && 'presence' in landmark) + return (landmark as LandmarkWithPresence).presence + return undefined +} + +function getOptionalVisibility(landmark: unknown): number | undefined { + if (landmark && typeof landmark === 'object' && 'visibility' in landmark) + return (landmark as LandmarkWithVisibility).visibility + return undefined +} + +function isConfident(pose: PoseState, index: number, thresholds: { minVisibility: number, minPresence: number }): boolean { + // User requirement: do not output anything when `visibility` is missing. + // Prefer 2D landmarks for visibility/presence (they are more consistently populated), + // but still allow fallback to world landmarks when needed. + const lm2d = pose.landmarks2d?.[index] + const lm3d = pose.worldLandmarks?.[index] + if (!lm2d && !lm3d) + return false + + const visibility = getOptionalVisibility(lm2d) ?? getOptionalVisibility(lm3d) + if (visibility == null || !Number.isFinite(visibility)) + return false + if (visibility < thresholds.minVisibility) + return false + + if (thresholds.minPresence > 0) { + const presence = getOptionalPresence(lm2d) ?? getOptionalPresence(lm3d) + if (presence != null && Number.isFinite(presence) && presence < thresholds.minPresence) + return false + } + return true +} + +function mid(a: Vector3Like, b: Vector3Like): Vector3Like { + return vScale(vAdd(a, b), 0.5) +} + +export function poseToVrmTargets(pose: PoseState, options?: PoseToVrmOptions): VrmPoseTargets { + // Prefer world landmarks when available; fallback to normalized landmarks to keep the pipeline usable. + const points = pose.worldLandmarks + if (!points?.length) + return {} + + const axis = options?.axis ?? DEFAULT_AXIS + const thresholds = { + minVisibility: options?.confidence?.minVisibility ?? DEFAULT_MIN_VISIBILITY, + minPresence: options?.confidence?.minPresence ?? DEFAULT_MIN_PRESENCE, + } + + const getC = (index: number) => (isConfident(pose, index, thresholds) ? get(points, index) : null) + + const leftShoulder = getC(11) + const rightShoulder = getC(12) + const leftElbow = getC(13) + const rightElbow = getC(14) + const leftWrist = getC(15) + const rightWrist = getC(16) + const leftHip = getC(23) + const rightHip = getC(24) + const leftKnee = getC(25) + const rightKnee = getC(26) + const leftAnkle = getC(27) + const rightAnkle = getC(28) + + const out: VrmPoseTargets = {} + + const shoulderCenter = leftShoulder && rightShoulder ? mid(leftShoulder, rightShoulder) : null + const hipCenter = leftHip && rightHip ? mid(leftHip, rightHip) : null + + const prevTargets = options?.stabilize?.previousTargets + const stabilizePole = (key: keyof VrmPoseTargets, pole: Vector3Like): Vector3Like => { + const prev = prevTargets?.[key]?.pole + if (prev && vDot(prev, pole) < 0) + return vNeg(pole) + return pole + } + + // Torso basis: + // - up: hipCenter -> shoulderCenter (dir) + // - right: leftShoulder -> rightShoulder + // - forward: cross(right, up) + // + // NOTICE: In apply-pose-to-vrm.ts, pole is used as the Z axis in `makeBasis(X=dir, Y=..., Z=pole)`, + // so torso pole must be "forward-like" (body facing), not "right-like" (shoulder line). + let torsoForward: Vector3Like | null = null + if (hipCenter && shoulderCenter && leftShoulder && rightShoulder) { + const rightRaw = vSub(rightShoulder, leftShoulder) + const upRaw = vSub(shoulderCenter, hipCenter) + const right = vNormalize(vRemapAxis(rightRaw, axis)) + const up = vNormalize(vRemapAxis(upRaw, axis)) + if (right && up) { + // Base forward from right×up, then fix sign so that (up×forward) matches observed shoulder-right. + let fw = vNormalize(vCross(right, up)) + if (fw) { + const rightFromBasis = vNormalize(vCross(fw, up)) + if (rightFromBasis && vDot(rightFromBasis, right) < 0) + fw = vNeg(fw) + + const prevForward = options?.stabilize?.previousForward ?? prevTargets?.hips?.pole ?? prevTargets?.spine?.pole ?? prevTargets?.chest?.pole + torsoForward = prevForward && vDot(prevForward, fw) < 0 ? vNeg(fw) : fw + } + } + } + + // Hips/spine/chest: drive "up" with forward pole so the avatar can turn with you. + if (hipCenter && shoulderCenter) { + const up = vNormalize(vRemapAxis(vSub(shoulderCenter, hipCenter), axis)) + if (up) { + if (torsoForward) + out.hips = { dir: up, pole: stabilizePole('hips', torsoForward) } + else + out.hips = { dir: up } + + if (torsoForward) { + out.spine = { dir: up, pole: stabilizePole('spine', torsoForward) } + out.chest = { dir: up, pole: stabilizePole('chest', torsoForward) } + } + else { + out.spine = { dir: up } + out.chest = { dir: up } + } + } + } + + // Shoulder (clavicle-ish): shoulder center -> shoulder + if (shoulderCenter && leftShoulder) { + const d = vNormalize(vRemapAxis(vSub(leftShoulder, shoulderCenter), axis)) + if (d) + out.leftShoulder = { dir: d } + } + if (shoulderCenter && rightShoulder) { + const d = vNormalize(vRemapAxis(vSub(rightShoulder, shoulderCenter), axis)) + if (d) + out.rightShoulder = { dir: d } + } + + // Arms (with pole from elbow bend plane) + if (leftShoulder && leftElbow) { + const upper = vNormalize(vRemapAxis(vSub(leftElbow, leftShoulder), axis)) + if (upper) { + const poleRaw = leftWrist ? vCross(vSub(leftElbow, leftShoulder), vSub(leftWrist, leftElbow)) : null + const pole = poleRaw ? vNormalize(vRemapAxis(poleRaw, axis)) : null + out.leftUpperArm = pole ? { dir: upper, pole } : { dir: upper } + } + } + if (leftElbow && leftWrist) { + const lower = vNormalize(vRemapAxis(vSub(leftWrist, leftElbow), axis)) + if (lower) + out.leftLowerArm = { dir: lower } + } + if (rightShoulder && rightElbow) { + const upper = vNormalize(vRemapAxis(vSub(rightElbow, rightShoulder), axis)) + if (upper) { + const poleRaw = rightWrist ? vCross(vSub(rightElbow, rightShoulder), vSub(rightWrist, rightElbow)) : null + const pole = poleRaw ? vNormalize(vRemapAxis(poleRaw, axis)) : null + out.rightUpperArm = pole ? { dir: upper, pole } : { dir: upper } + } + } + if (rightElbow && rightWrist) { + const lower = vNormalize(vRemapAxis(vSub(rightWrist, rightElbow), axis)) + if (lower) + out.rightLowerArm = { dir: lower } + } + + // Legs: require ankle to reduce hallucinated flips when lower body is off-screen + if (leftHip && leftKnee && leftAnkle) { + const upper = vNormalize(vRemapAxis(vSub(leftKnee, leftHip), axis)) + if (upper) { + const poleRaw = vCross(vSub(leftKnee, leftHip), vSub(leftAnkle, leftKnee)) + let pole = vNormalize(vRemapAxis(poleRaw, axis)) + if (pole) + pole = stabilizePole('leftUpperLeg', pole) + if (pole) + pole = safePole(upper, pole) + out.leftUpperLeg = pole ? { dir: upper, pole } : { dir: upper } + } + + const lower = vNormalize(vRemapAxis(vSub(leftAnkle, leftKnee), axis)) + if (lower) + out.leftLowerLeg = { dir: lower } + } + + if (rightHip && rightKnee && rightAnkle) { + const upper = vNormalize(vRemapAxis(vSub(rightKnee, rightHip), axis)) + if (upper) { + const poleRaw = vCross(vSub(rightKnee, rightHip), vSub(rightAnkle, rightKnee)) + let pole = vNormalize(vRemapAxis(poleRaw, axis)) + if (pole) + pole = stabilizePole('rightUpperLeg', pole) + if (pole) + pole = safePole(upper, pole) + out.rightUpperLeg = pole ? { dir: upper, pole } : { dir: upper } + } + + const lower = vNormalize(vRemapAxis(vSub(rightAnkle, rightKnee), axis)) + if (lower) + out.rightLowerLeg = { dir: lower } + } + + return out +} + +export function poseToVrmDirections(pose: PoseState, options?: PoseToVrmOptions): VrmPoseDirections { + const targets = poseToVrmTargets(pose, options) + const out: VrmPoseDirections = {} + ;(Object.keys(targets) as (keyof VrmPoseDirections)[]).forEach((k) => { + const t = targets[k] + if (t) + out[k] = t.dir + }) + return out +} diff --git a/packages/model-driver-mediapipe/src/types.ts b/packages/model-driver-mediapipe/src/types.ts new file mode 100644 index 000000000..f87840b68 --- /dev/null +++ b/packages/model-driver-mediapipe/src/types.ts @@ -0,0 +1,74 @@ +import type { FilesetResolver, Landmark, NormalizedLandmark } from '@mediapipe/tasks-vision' + +export type VisionTaskModule = typeof import('@mediapipe/tasks-vision') + +// Indirect export from @mediapipe/tasks-vision +export type VisionTaskWasmFileset = Awaited> + +export type Landmark2D = NormalizedLandmark +export type Landmark3D = Landmark + +export interface PoseState { + landmarks2d?: Landmark2D[] + worldLandmarks?: Landmark3D[] +} + +export interface HandState { + handedness: 'Left' | 'Right' + landmarks2d: Landmark2D[] + score?: number +} + +export interface FaceState { + hasFace?: boolean + landmarks2d?: Landmark2D[] +} + +export interface PerceptionQuality { + fps: number + latencyMs?: number + droppedFrames?: number + backend: 'mediapipe' + mode: 'split-tasks' +} + +export interface PerceptionPartial { + pose?: PoseState + hands?: HandState[] + face?: FaceState +} + +export interface PerceptionState extends PerceptionPartial { + t: number + quality: PerceptionQuality +} + +export type MocapJob = 'pose' | 'hands' | 'face' + +export interface MocapConfig { + enabled: Record + hz: Record + maxPeople: 1 +} + +export interface FrameSource { + getFrame: () => TexImageSource +} + +export interface MocapBackend { + init: (config: MocapConfig) => Promise + isBusy: () => boolean + run: (frame: TexImageSource, jobs: MocapJob[], nowMs: number) => Promise +} + +export interface MocapEngine { + init: () => Promise + start: ( + source: FrameSource, + onState: (state: PerceptionState) => void, + options?: { onError?: (error: unknown) => void }, + ) => void + stop: () => void + updateConfig: (config: MocapConfig) => void + resetState: () => void +} diff --git a/packages/model-driver-mediapipe/src/utils/overlay.ts b/packages/model-driver-mediapipe/src/utils/overlay.ts new file mode 100644 index 000000000..d1f19c909 --- /dev/null +++ b/packages/model-driver-mediapipe/src/utils/overlay.ts @@ -0,0 +1,93 @@ +import type { FaceState, HandState, Landmark2D, PerceptionState, PoseState } from '../types' + +import { DrawingUtils, HandLandmarker, PoseLandmarker } from '@mediapipe/tasks-vision' + +const POSE_CONNECTIONS: Readonly<{ start: number, end: number }[]> = PoseLandmarker.POSE_CONNECTIONS +const HAND_CONNECTIONS: Readonly<{ start: number, end: number }[]> = HandLandmarker.HAND_CONNECTIONS + +// NOTICE: Palette inspired by https://github.com/proj-airi/webai-examples (see review link in PR). +const OVERLAY_PALETTE = [ + { point: 'rgba(80, 200, 255, 0.95)', connector: 'rgba(80, 200, 255, 0.55)' }, + { point: 'rgba(120, 255, 140, 0.95)', connector: 'rgba(120, 255, 140, 0.55)' }, + { point: 'rgba(255, 180, 80, 0.95)', connector: 'rgba(255, 180, 80, 0.55)' }, + { point: 'rgba(180, 120, 255, 0.55)', connector: 'rgba(180, 120, 255, 0.55)' }, +] + +// NOTICE: Tuned for devtools readability; see https://ai.google.dev/edge/api/mediapipe/js/tasks-vision.drawingutils. +const OVERLAY_STYLES = { + connectorLineWidth: 2.5, + pointRadius: 6, + facePointRadius: 2, + pointLineWidth: 1.5, +} + +const paletteFor = (index: number) => OVERLAY_PALETTE[index % OVERLAY_PALETTE.length] + +function drawFace(drawing: DrawingUtils, face: FaceState) { + if (!face.landmarks2d?.length) + return + + // Face has 468 points; keep them small to reduce clutter. + const colors = paletteFor(3) + drawing.drawLandmarks(face.landmarks2d as Landmark2D[], { + color: colors.point, + radius: OVERLAY_STYLES.facePointRadius, + lineWidth: OVERLAY_STYLES.pointLineWidth, + }) +} + +function drawPose(drawing: DrawingUtils, pose: PoseState) { + if (!pose.landmarks2d?.length) + return + + const colors = paletteFor(0) + drawing.drawConnectors(pose.landmarks2d as Landmark2D[], POSE_CONNECTIONS.map(({ start, end }) => ({ start, end })), { + color: colors.connector, + lineWidth: OVERLAY_STYLES.connectorLineWidth, + }) + drawing.drawLandmarks(pose.landmarks2d as Landmark2D[], { + color: colors.point, + radius: OVERLAY_STYLES.pointRadius, + lineWidth: OVERLAY_STYLES.pointLineWidth, + }) +} + +function drawHands(drawing: DrawingUtils, hands: HandState[]) { + hands.forEach((hand) => { + const handIndex = hand.handedness === 'Left' ? 1 : 2 + const colors = paletteFor(handIndex) + drawing.drawConnectors(hand.landmarks2d as Landmark2D[], HAND_CONNECTIONS.map(({ start, end }) => ({ start, end })), { + color: colors.connector, + lineWidth: OVERLAY_STYLES.connectorLineWidth, + }) + drawing.drawLandmarks(hand.landmarks2d as Landmark2D[], { + color: colors.point, + radius: OVERLAY_STYLES.pointRadius, + lineWidth: OVERLAY_STYLES.pointLineWidth, + }) + }) +} + +export function drawOverlay( + ctx: CanvasRenderingContext2D, + state: PerceptionState, + enabled?: Partial>, +) { + const { width, height } = ctx.canvas + ctx.clearRect(0, 0, width, height) + + const drawing = new DrawingUtils(ctx) + + const showPose = enabled?.pose ?? true + const showHands = enabled?.hands ?? true + const showFace = enabled?.face ?? true + + if (state.face && showFace) + drawFace(drawing, state.face) + + if (state.pose && showPose) + drawPose(drawing, state.pose) + + if (state.hands?.length && showHands) + drawHands(drawing, state.hands) +} diff --git a/packages/model-driver-mediapipe/tasks/prepare-tasks.ts b/packages/model-driver-mediapipe/tasks/prepare-tasks.ts new file mode 100644 index 000000000..ce86479c2 --- /dev/null +++ b/packages/model-driver-mediapipe/tasks/prepare-tasks.ts @@ -0,0 +1,58 @@ +/* eslint-disable antfu/no-top-level-await */ +/* eslint-disable no-console */ + +import type { VisionTaskAssets } from './tasks' + +import fs from 'node:fs/promises' + +import { Buffer } from 'node:buffer' +import { fileURLToPath } from 'node:url' + +import { ofetch } from 'ofetch' + +import { visionTaskAssets } from './tasks' + +const taskSources: Record = { + pose: 'https://storage.googleapis.com/mediapipe-models/pose_landmarker/pose_landmarker_lite/float16/1/pose_landmarker_lite.task', + hands: 'https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/1/hand_landmarker.task', + face: 'https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task', +} + +await fs.mkdir(fileURLToPath(new URL('./assets', import.meta.url)), { recursive: true }) + +await Promise.all(Object.entries(taskSources).map( + async ([key, url]) => { + console.log(`Downloading MediaPipe vision task asset for ${key} from ${url}...`) + const res = await ofetch(url, { responseType: 'arrayBuffer' }) + const outputPath = fileURLToPath(visionTaskAssets[key as keyof VisionTaskAssets]) + await fs.writeFile(outputPath, Buffer.from(res)) + console.log(`MediaPipe vision task asset for ${key} saved to ${outputPath}`) + }, +)) + +const wasmSourceDir = fileURLToPath(new URL('../node_modules/@mediapipe/tasks-vision/wasm', import.meta.url)) +const wasmOutputDir = fileURLToPath(new URL('./assets/wasm', import.meta.url)) +await fs.mkdir(wasmOutputDir, { recursive: true }) +await fs.cp(wasmSourceDir, wasmOutputDir, { recursive: true, force: true }) + +await Promise.all(Object.entries(visionTaskAssets).map( + async ([key, url]) => { + const path = fileURLToPath(url) + try { + await fs.access(path, fs.constants.R_OK) + } + catch (err) { + throw new Error(`Failed to ensure MediaPipe vision task asset for ${key}: ${err}`) + } + const stat = await fs.stat(path) + if (!stat.isFile()) { + throw new Error(`Failed to ensure MediaPipe vision task asset for ${key}: not a file: ${path}`) + } + }, +)) + +const wasmEntries = await fs.readdir(wasmOutputDir) +if (!wasmEntries.length) + throw new Error(`Failed to ensure MediaPipe WASM assets: ${wasmOutputDir} is empty`) + +console.log('All MediaPipe vision task assets are prepared.') diff --git a/packages/model-driver-mediapipe/tasks/tasks.ts b/packages/model-driver-mediapipe/tasks/tasks.ts new file mode 100644 index 000000000..ec5eea5f5 --- /dev/null +++ b/packages/model-driver-mediapipe/tasks/tasks.ts @@ -0,0 +1,13 @@ +export interface VisionTaskAssets { + pose: string + hands: string + face: string +} + +export const visionTaskAssets: VisionTaskAssets = { + pose: new URL('./assets/pose_landmarker_lite.task', import.meta.url).href, + hands: new URL('./assets/hand_landmarker.task', import.meta.url).href, + face: new URL('./assets/face_landmarker.task', import.meta.url).href, +} + +export const visionTaskWasmRoot = new URL('./assets/wasm', import.meta.url).href diff --git a/packages/model-driver-mediapipe/tsconfig.json b/packages/model-driver-mediapipe/tsconfig.json new file mode 100644 index 000000000..cc53a4905 --- /dev/null +++ b/packages/model-driver-mediapipe/tsconfig.json @@ -0,0 +1,30 @@ +{ + "compilerOptions": { + "target": "ESNext", + "lib": [ + "DOM", + "ESNext" + ], + "module": "ESNext", + "moduleResolution": "Bundler", + "resolveJsonModule": true, + "strict": true, + "strictNullChecks": true, + "noUnusedLocals": true, + "noEmit": true, + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true, + "isolatedModules": true, + "verbatimModuleSyntax": true, + "skipLibCheck": true + }, + "include": [ + "src/**/*.ts", + "src/**/*.d.ts", + "tasks/**/*.ts" + ], + "exclude": [ + "dist", + "node_modules" + ] +} diff --git a/packages/stage-ui-three/src/components/Model/VRMModel.vue b/packages/stage-ui-three/src/components/Model/VRMModel.vue index 57fc6d0ed..788238122 100644 --- a/packages/stage-ui-three/src/components/Model/VRMModel.vue +++ b/packages/stage-ui-three/src/components/Model/VRMModel.vue @@ -159,6 +159,9 @@ let stopCameraWatch: WatchStopHandle | undefined // Animation related ref const vrmAnimationMixer = ref() const { onBeforeRender, stop, start } = useLoop() + +type VrmFrameHook = (vrm: VRM, delta: number) => void +const vrmFrameHook = shallowRef() let disposeBeforeRenderLoop: (() => void | undefined) // Expressions @@ -417,13 +420,23 @@ async function loadModel() { // Clean up & animation setting disposeBeforeRenderLoop = onBeforeRender(({ delta }) => { vrmAnimationMixer.value?.update(delta) - vrm.value?.update(delta) - vrm.value?.lookAt?.update?.(delta) - blink.update(vrm.value, delta) - idleEyeSaccades.update(vrm.value, lookAtTarget, delta) + const activeVrm = vrm.value + if (activeVrm && vrmFrameHook.value) { + try { + vrmFrameHook.value(activeVrm, delta) + } + catch (err) { + console.error(err) + emit('error', err) + } + } + activeVrm?.humanoid.update() + activeVrm?.lookAt?.update?.(delta) + blink.update(activeVrm, delta) + idleEyeSaccades.update(activeVrm, lookAtTarget, delta) vrmEmote.value?.update(delta) - vrmLipSync.update(vrm.value, delta) - vrm.value?.springBoneManager?.update(delta) + vrmLipSync.update(activeVrm, delta) + activeVrm?.springBoneManager?.update(delta) }).off // update the 'last model src' @@ -562,6 +575,9 @@ defineExpose({ setExpression(expression: string) { vrmEmote.value?.setEmotionWithResetAfter(expression, 1000) }, + setVrmFrameHook(hook?: VrmFrameHook) { + vrmFrameHook.value = hook + }, scene: computed(() => vrm.value?.scene), lookAtUpdate(target: Vec3) { idleEyeSaccades.instantUpdate(vrm.value, target) diff --git a/packages/stage-ui-three/src/components/ThreeScene.vue b/packages/stage-ui-three/src/components/ThreeScene.vue index 52661d278..831e2532c 100644 --- a/packages/stage-ui-three/src/components/ThreeScene.vue +++ b/packages/stage-ui-three/src/components/ThreeScene.vue @@ -7,6 +7,7 @@ * - Src of model is obtained from stage-ui via props, which is NOT a part of stage-ui-three package */ +import type { VRM } from '@pixiv/three-vrm' import type { TresContext } from '@tresjs/core' import type { DirectionalLight, SphericalHarmonics3, Texture, WebGLRenderTarget } from 'three' @@ -198,6 +199,12 @@ const effectProps = { blendFunction: BlendFunction.SRC, } +const vrmFrameHook = shallowRef<((vrm: VRM, delta: number) => void) | undefined>(undefined) +function applyVrmFrameHook() { + modelRef.value?.setVrmFrameHook(vrmFrameHook.value) +} +watch(modelRef, () => applyVrmFrameHook(), { immediate: true }) + // === Directional Light === // TODO: wrap to integrate all the below code const sceneReady = ref(false) @@ -270,6 +277,10 @@ defineExpose({ setExpression: (expression: string) => { modelRef.value?.setExpression(expression) }, + setVrmFrameHook: (hook?: (vrm: VRM, delta: number) => void) => { + vrmFrameHook.value = hook + applyVrmFrameHook() + }, canvasElement: () => { return tresCanvasRef.value?.renderer.instance.domElement }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c7ab3034f..8859ef006 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -72,9 +72,15 @@ catalogs: nanoid: specifier: ^5.1.6 version: 5.1.6 + ofetch: + specifier: ^1.5.1 + version: 1.5.1 posthog-js: specifier: 1.306.1 version: 1.306.1 + tsx: + specifier: ^4.21.0 + version: 4.21.0 uncrypto: specifier: ^0.1.3 version: 0.1.3 @@ -739,6 +745,9 @@ importers: '@proj-airi/i18n': specifier: workspace:^ version: link:../../packages/i18n + '@proj-airi/model-driver-mediapipe': + specifier: workspace:^ + version: link:../../packages/model-driver-mediapipe '@proj-airi/server-sdk': specifier: workspace:^ version: link:../../packages/server-sdk @@ -1273,6 +1282,31 @@ importers: specifier: ^1.3.0 version: 1.3.0 + packages/model-driver-mediapipe: + dependencies: + '@mediapipe/tasks-vision': + specifier: ^0.10.0 + version: 0.10.21 + '@pixiv/three-vrm': + specifier: ^3.4.4 + version: 3.4.4(three@0.182.0) + es-toolkit: + specifier: 'catalog:' + version: 1.43.0 + three: + specifier: ^0.182.0 + version: 0.182.0 + devDependencies: + '@types/three': + specifier: ^0.182.0 + version: 0.182.0 + ofetch: + specifier: 'catalog:' + version: 1.5.1 + tsx: + specifier: 'catalog:' + version: 4.21.0 + packages/pipelines-audio: dependencies: '@moeru/std': @@ -4972,6 +5006,9 @@ packages: markdown-it: optional: true + '@mediapipe/tasks-vision@0.10.21': + resolution: {integrity: sha512-TuhKH+credq4zLksGbYrnvJ1aLIWMc5r0UHwzxzql4BHECJwIAoBR61ZrqwGOW6ZmSBIzU1t4VtKj8hbxFaKeA==} + '@modelcontextprotocol/sdk@1.25.1': resolution: {integrity: sha512-yO28oVFFC7EBoiKdAn+VqRm+plcfv4v0xp6osG/VsCB0NlPZWi87ajbCZZ8f/RvOFLEu7//rSRmuZZ7lMoe3gQ==} engines: {node: '>=18'} @@ -17749,6 +17786,8 @@ snapshots: optionalDependencies: markdown-it: 14.1.0 + '@mediapipe/tasks-vision@0.10.21': {} + '@modelcontextprotocol/sdk@1.25.1(@cfworker/json-schema@4.1.1)(hono@4.11.3)(zod@4.2.1)': dependencies: '@hono/node-server': 1.19.7(hono@4.11.3) diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 504620f4a..1ba19245b 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -1,7 +1,6 @@ catalogMode: prefer shellEmulator: true - packages: - packages/** - plugins/** @@ -20,7 +19,6 @@ overrides: safer-buffer: npm:@nolyfill/safer-buffer@^1.0.44 side-channel: npm:@nolyfill/side-channel@^1.0.44 string.prototype.matchall: npm:@nolyfill/string.prototype.matchall@^1.0.44 - catalog: '@guiiai/logg': ^1.2.11 '@moeru/eslint-config': 0.1.0-beta.14 @@ -44,7 +42,9 @@ catalog: es-toolkit: 1.43.0 injeca: ^0.1.5 nanoid: ^5.1.6 + ofetch: ^1.5.1 posthog-js: 1.306.1 + tsx: ^4.21.0 uncrypto: ^0.1.3 unplugin-info: 1.2.4 xsschema: ^0.4.0-beta.12