feat(model-driver-mediapipe): integrate MediaPipe as a motion capture tool (#828)

--------- Co-authored-by: Makito <i@maki.to>
2026-08-14 00:48:06 +00:00 · 2025-12-30 12:11:08 +08:00
parent 989c1c8994
commit ecd5eee155
25 changed files with 2020 additions and 8 deletions
@@ -110,3 +110,6 @@ twitter-session.json
 result*
 # Nix develop outputs
 outputs/
+
+# MediaPipe task assets
+packages/model-driver-mediapipe/tasks/assets/*
@@ -28,6 +28,7 @@
    "@proj-airi/font-cjkfonts-allseto": "workspace:^",
    "@proj-airi/font-xiaolai": "workspace:^",
    "@proj-airi/i18n": "workspace:^",
+    "@proj-airi/model-driver-mediapipe": "workspace:^",
    "@proj-airi/server-sdk": "workspace:^",
    "@proj-airi/stage-ui": "workspace:^",
    "@proj-airi/stage-ui-three": "workspace:^",
@@ -0,0 +1,485 @@
+<script setup lang="ts">
+import type { PerceptionState, VrmPoseTargets } from '@proj-airi/model-driver-mediapipe'
+import type { Vector3Like } from 'three'
+
+import { createMediaPipeBackend, createMocapEngine, createVrmPoseApplier, drawOverlay, poseToVrmTargets } from '@proj-airi/model-driver-mediapipe'
+import { ThreeScene } from '@proj-airi/stage-ui-three'
+import { animations } from '@proj-airi/stage-ui-three/assets/vrm'
+import { useSettings } from '@proj-airi/stage-ui/stores/settings'
+import { Checkbox } from '@proj-airi/ui'
+import { storeToRefs } from 'pinia'
+import { computed, onMounted, onUnmounted, ref, toRaw, watch } from 'vue'
+
+const status = ref<'idle' | 'starting' | 'running' | 'error'>('idle')
+const errorMessage = ref('')
+const pipelineEnabled = ref(true)
+const syncingToggleState = ref(false)
+const ignoreErrorsUntil = ref(0)
+
+const videoRef = ref<HTMLVideoElement>()
+const canvasRef = ref<HTMLCanvasElement>()
+const sceneRef = ref<InstanceType<typeof ThreeScene>>()
+let stream: MediaStream | undefined
+let engine: ReturnType<typeof createMocapEngine> | undefined
+
+// config on the page
+const config = ref({
+  enabled: {
+    pose: true,
+    hands: true,
+    face: true,
+  },
+  hz: {
+    pose: 30,
+    hands: 30,
+    face: 30,
+  },
+  maxPeople: 1 as const, // Fixed to 1 for simplicity
+})
+
+const vrmMapping = ref({
+  flipX: true,
+  flipY: true,
+  flipZ: false,
+})
+
+const poseFiltering = ref({
+  minVisibility: 0.5,
+})
+
+// MediaPipe assets config
+const latestState = ref<PerceptionState>()
+const latestPoseTargets = ref<VrmPoseTargets>()
+const prevPoseTargets = ref<VrmPoseTargets>()
+const prevPoseForward = ref<Vector3Like>()
+
+// VRM pose applier
+const vrmPoseApplier = createVrmPoseApplier({ alpha: 1 })
+function onVrmFrame(vrm: Parameters<typeof vrmPoseApplier.applyPoseDirectionsToVrm>[0]) {
+  const targets = latestPoseTargets.value
+  if (!targets)
+    return
+  vrmPoseApplier.applyPoseTargetsToVrm(vrm, targets)
+}
+const vrmFrameHook = (vrm: Parameters<typeof vrmPoseApplier.applyPoseDirectionsToVrm>[0]) => onVrmFrame(vrm)
+
+const settingsStore = useSettings()
+const { stageModelRenderer, stageModelSelected, stageModelSelectedUrl, stageViewControlsEnabled } = storeToRefs(settingsStore)
+
+// Snapshot summary of the running state
+const summary = computed(() => {
+  const enabled = Object.entries(config.value.enabled)
+    .filter(([, v]) => v)
+    .map(([k]) => k)
+    .join(', ') || 'none'
+
+  const fps = latestState.value?.quality.fps
+  const latency = latestState.value?.quality.latencyMs
+  const dropped = latestState.value?.quality.droppedFrames
+
+  return [
+    `enabled: ${enabled}`,
+    `hz: pose ${config.value.hz.pose}, hands ${config.value.hz.hands}, face ${config.value.hz.face}`,
+    fps != null ? `fps ${fps.toFixed(1)}` : null,
+    latency != null ? `latency ${latency.toFixed(1)}ms` : null,
+    dropped != null ? `dropped ${dropped}` : null,
+  ].filter(Boolean).join(' | ')
+})
+
+const poseVisibilityDebug = computed(() => {
+  const pose = latestState.value?.pose
+  const lm = pose?.landmarks2d
+  const world = pose?.worldLandmarks
+  if (!lm?.length)
+    return 'pose: (no landmarks)'
+
+  const pick = (i: number) => {
+    const v = lm[i]?.visibility
+    return v == null || !Number.isFinite(v) ? 'na' : v.toFixed(2)
+  }
+
+  const withVis = lm.filter(p => p.visibility != null && Number.isFinite(p.visibility)).length
+  const worldWithVis = world?.filter(p => p.visibility != null && Number.isFinite(p.visibility)).length ?? 0
+  return [
+    `pose 2d vis ${withVis}/${lm.length}`,
+    `pose 3d vis ${worldWithVis}/${world?.length ?? 0}`,
+    `LS ${pick(11)} RS ${pick(12)} LE ${pick(13)} RE ${pick(14)}`,
+    `LW ${pick(15)} RW ${pick(16)} LH ${pick(23)} RH ${pick(24)}`,
+  ].join(' | ')
+})
+
+// Start camera and pipeline
+async function startCamera() {
+  if (status.value === 'starting' || status.value === 'running')
+    return
+
+  status.value = 'starting'
+  errorMessage.value = ''
+
+  try {
+    stop()
+    stream = await navigator.mediaDevices.getUserMedia({ video: true, audio: false })
+    if (!videoRef.value)
+      throw new Error('video element not mounted')
+
+    videoRef.value.srcObject = stream
+    await videoRef.value.play()
+
+    status.value = 'running'
+    await startPipeline()
+  }
+  catch (err) {
+    status.value = 'error'
+    errorMessage.value = err instanceof Error ? err.message : String(err)
+    console.error('Failed to start camera or pipeline:', err)
+
+    syncingToggleState.value = true
+    pipelineEnabled.value = false
+    syncingToggleState.value = false
+  }
+}
+
+async function startPipeline() {
+  if (!videoRef.value)
+    return
+  if (engine)
+    return
+
+  const backend = createMediaPipeBackend()
+  engine = createMocapEngine(backend, toRaw(config.value))
+  await engine.init()
+
+  engine.start(
+    { getFrame: () => videoRef.value as HTMLVideoElement },
+    (state) => {
+      latestState.value = state
+      const axis = {
+        x: vrmMapping.value.flipX ? -1 : 1,
+        y: vrmMapping.value.flipY ? -1 : 1,
+        z: vrmMapping.value.flipZ ? -1 : 1,
+      } as const
+
+      const poseTargets = (config.value.enabled.pose && state.pose?.worldLandmarks?.length)
+        ? poseToVrmTargets(state.pose, {
+            axis,
+            confidence: { minVisibility: poseFiltering.value.minVisibility },
+            stabilize: {
+              previousTargets: prevPoseTargets.value,
+              previousForward: prevPoseForward.value,
+            },
+          })
+        : {}
+
+      const hasAny = Object.keys(poseTargets).length > 0
+      latestPoseTargets.value = hasAny ? poseTargets : undefined
+      if (hasAny) {
+        prevPoseTargets.value = poseTargets
+        const derivedForward = poseTargets.hips?.pole ?? poseTargets.spine?.pole
+        if (derivedForward)
+          prevPoseForward.value = derivedForward
+      }
+
+      const canvas = canvasRef.value
+      const video = videoRef.value
+      if (!canvas || !video)
+        return
+
+      const w = video.videoWidth || 640
+      const h = video.videoHeight || 480
+      if (canvas.width !== w)
+        canvas.width = w
+      if (canvas.height !== h)
+        canvas.height = h
+
+      const ctx = canvas.getContext('2d')
+      if (!ctx)
+        return
+
+      drawOverlay(ctx, state, config.value.enabled)
+    },
+    {
+      onError: (err) => {
+        if (!pipelineEnabled.value || Date.now() < ignoreErrorsUntil.value) {
+          console.warn('Ignored pipeline error during stop:', err)
+          return
+        }
+
+        errorMessage.value = err instanceof Error ? err.message : String(err)
+        // Ensure resources are released, but keep the error status visible.
+        stop()
+        status.value = 'error'
+        console.error('Pipeline error:', err)
+      },
+    },
+  )
+}
+
+function stopPipeline() {
+  engine?.stop()
+  engine = undefined
+  latestState.value = undefined
+  latestPoseTargets.value = undefined
+  prevPoseTargets.value = undefined
+  prevPoseForward.value = undefined
+}
+
+function stop() {
+  // During stop, MediaPipe may still be processing an in-flight frame; ignore transient errors.
+  ignoreErrorsUntil.value = Date.now() + 1500
+  canvasRef.value?.getContext('2d')?.clearRect(0, 0, canvasRef.value.width, canvasRef.value.height)
+  stopPipeline()
+
+  try {
+    stream?.getTracks().forEach(t => t.stop())
+  }
+  catch {}
+
+  stream = undefined
+
+  if (videoRef.value)
+    videoRef.value.srcObject = null
+
+  status.value = 'idle'
+}
+
+watch(config, (val) => {
+  engine?.updateConfig(toRaw(val))
+}, { deep: true })
+
+watch(sceneRef, (scene, prev) => {
+  prev?.setVrmFrameHook(undefined)
+  scene?.setVrmFrameHook(vrmFrameHook)
+}, { immediate: true })
+
+watch(pipelineEnabled, async (enabled) => {
+  if (syncingToggleState.value)
+    return
+
+  if (enabled)
+    await startCamera()
+  else
+    stop()
+})
+
+onMounted(() => {
+  // Ensure a VRM model is selected for the viewer (preserve existing selection if already VRM).
+  const needsFallback = !stageModelSelectedUrl.value || stageModelRenderer.value !== 'vrm'
+  if (needsFallback)
+    stageModelSelected.value = 'preset-vrm-1'
+
+  settingsStore.updateStageModel().catch((err) => {
+    console.error('Failed to init VRM model:', err)
+  })
+
+  // Autostart for convenience
+  if (pipelineEnabled.value)
+    startCamera()
+})
+
+onUnmounted(() => {
+  sceneRef.value?.setVrmFrameHook(undefined)
+  stop()
+})
+</script>
+
+<template>
+  <div :class="['p-4', 'space-y-4']">
+    <div>
+      <div :class="['text-lg', 'font-600']">
+        MediaPipe Workshop Playground
+      </div>
+    </div>
+
+    <!-- Top config -->
+    <div :class="['rounded-2xl', 'border', 'border-neutral-300/40', 'dark:border-neutral-700/40', 'p-3', 'space-y-3']">
+      <div :class="['flex', 'items-start', 'justify-between', 'gap-3', 'flex-wrap']">
+        <div :class="['space-y-1']">
+          <div :class="['font-600']">
+            Config
+          </div>
+          <div :class="['text-xs', 'text-neutral-500']">
+            {{ summary }}
+          </div>
+        </div>
+
+        <label :class="['flex', 'items-center', 'gap-3']">
+          <div :class="['text-sm', 'text-neutral-600', 'dark:text-neutral-300']">
+            {{ pipelineEnabled ? 'Running' : 'Stopped' }}
+          </div>
+          <Checkbox v-model="pipelineEnabled" />
+        </label>
+      </div>
+
+      <div :class="['grid', 'gap-3', 'lg:grid-cols-3']">
+        <div :class="['flex', 'items-center', 'justify-between', 'gap-3']">
+          <label :class="['flex', 'items-center', 'gap-2', 'text-sm']">
+            <input v-model="config.enabled.pose" type="checkbox">
+            Pose
+          </label>
+          <label :class="['flex', 'items-center', 'gap-2']">
+            <div :class="['text-xs', 'text-neutral-500']">
+              Hz
+            </div>
+            <input
+              v-model.number="config.hz.pose"
+              type="number"
+              min="1"
+              max="60"
+              :class="['w-24', 'rounded-lg', 'border', 'border-neutral-300/60', 'bg-white', 'px-2', 'py-1', 'text-sm', 'dark:bg-neutral-900/60', 'dark:border-neutral-700/60']"
+            >
+          </label>
+        </div>
+
+        <div :class="['flex', 'items-center', 'justify-between', 'gap-3']">
+          <label :class="['flex', 'items-center', 'gap-2', 'text-sm']">
+            <input v-model="config.enabled.hands" type="checkbox">
+            Hands
+          </label>
+          <label :class="['flex', 'items-center', 'gap-2']">
+            <div :class="['text-xs', 'text-neutral-500']">
+              Hz
+            </div>
+            <input
+              v-model.number="config.hz.hands"
+              type="number"
+              min="1"
+              max="60"
+              :class="['w-24', 'rounded-lg', 'border', 'border-neutral-300/60', 'bg-white', 'px-2', 'py-1', 'text-sm', 'dark:bg-neutral-900/60', 'dark:border-neutral-700/60']"
+            >
+          </label>
+        </div>
+
+        <div :class="['flex', 'items-center', 'justify-between', 'gap-3']">
+          <label :class="['flex', 'items-center', 'gap-2', 'text-sm']">
+            <input v-model="config.enabled.face" type="checkbox">
+            Face
+          </label>
+          <label :class="['flex', 'items-center', 'gap-2']">
+            <div :class="['text-xs', 'text-neutral-500']">
+              Hz
+            </div>
+            <input
+              v-model.number="config.hz.face"
+              type="number"
+              min="1"
+              max="60"
+              :class="['w-24', 'rounded-lg', 'border', 'border-neutral-300/60', 'bg-white', 'px-2', 'py-1', 'text-sm', 'dark:bg-neutral-900/60', 'dark:border-neutral-700/60']"
+            >
+          </label>
+        </div>
+      </div>
+
+      <div :class="['flex', 'items-center', 'justify-between', 'gap-4', 'flex-wrap']">
+        <div :class="['text-sm', 'text-neutral-600', 'dark:text-neutral-300']">
+          VRM Mapping
+        </div>
+        <div :class="['flex', 'items-center', 'gap-6', 'flex-wrap']">
+          <label :class="['flex', 'items-center', 'gap-3']">
+            <div :class="['text-sm', 'text-neutral-600', 'dark:text-neutral-300']">
+              Flip X
+            </div>
+            <Checkbox v-model="vrmMapping.flipX" />
+          </label>
+          <label :class="['flex', 'items-center', 'gap-3']">
+            <div :class="['text-sm', 'text-neutral-600', 'dark:text-neutral-300']">
+              Flip Y
+            </div>
+            <Checkbox v-model="vrmMapping.flipY" />
+          </label>
+          <label :class="['flex', 'items-center', 'gap-3']">
+            <div :class="['text-sm', 'text-neutral-600', 'dark:text-neutral-300']">
+              Flip Z
+            </div>
+            <Checkbox v-model="vrmMapping.flipZ" />
+          </label>
+        </div>
+      </div>
+
+      <div :class="['flex', 'items-center', 'justify-between', 'gap-4', 'flex-wrap']">
+        <div :class="['text-sm', 'text-neutral-600', 'dark:text-neutral-300']">
+          Pose Filtering
+        </div>
+        <label :class="['flex', 'items-center', 'gap-3']">
+          <div :class="['text-sm', 'text-neutral-600', 'dark:text-neutral-300']">
+            Min Visibility
+          </div>
+          <input
+            v-model.number="poseFiltering.minVisibility"
+            type="number"
+            min="0"
+            max="1"
+            step="0.05"
+            :class="['w-24', 'rounded-lg', 'border', 'border-neutral-300/60', 'bg-white', 'px-2', 'py-1', 'text-sm', 'dark:bg-neutral-900/60', 'dark:border-neutral-700/60']"
+          >
+        </label>
+      </div>
+
+      <div :class="['text-xs', 'text-neutral-500']">
+        Note: `@mediapipe/tasks-vision` runs sync and may block the main thread. This workshop drops frames when busy to keep UI responsive.
+      </div>
+      <div :class="['text-xs', 'text-neutral-500', 'break-words']">
+        {{ poseVisibilityDebug }}
+      </div>
+    </div>
+
+    <!-- Main: camera + VRM -->
+    <div :class="['grid', 'gap-4', 'lg:grid-cols-2']">
+      <div :class="['rounded-2xl', 'border', 'border-neutral-300/40', 'dark:border-neutral-700/40', 'overflow-hidden']">
+        <div :class="['relative', 'aspect-video', 'bg-black']">
+          <video
+            ref="videoRef"
+            muted
+            playsinline
+            :class="['absolute', 'inset-0', 'h-full', 'w-full', 'object-cover', 'opacity-70']"
+          />
+          <canvas
+            ref="canvasRef"
+            :class="['absolute', 'inset-0', 'h-full', 'w-full', 'object-cover', 'opacity-70']"
+          />
+
+          <div
+            :class="[
+              'absolute',
+              'left-3',
+              'top-3',
+              'rounded-lg',
+              'bg-black/50',
+              'px-2',
+              'py-1',
+              'text-xs',
+              'text-white',
+              'backdrop-blur',
+            ]"
+          >
+            <div>Status: {{ status }}</div>
+            <div v-if="status === 'error'" :class="['text-red-300']">
+              {{ errorMessage }}
+            </div>
+          </div>
+        </div>
+      </div>
+
+      <div :class="['rounded-2xl', 'border', 'border-neutral-300/40', 'dark:border-neutral-700/40', 'overflow-hidden']">
+        <div :class="['h-full', 'min-h-80']">
+          <ThreeScene
+            v-if="stageModelRenderer === 'vrm'"
+            ref="sceneRef"
+            :model-src="stageModelSelectedUrl"
+            :idle-animation="animations.idleLoop.toString()"
+            :show-axes="stageViewControlsEnabled"
+            :paused="false"
+            @error="console.error"
+          />
+          <div v-else :class="['p-4', 'text-sm', 'text-red-500']">
+            请选择 VRM 模型（当前模型类型不支持）。
+          </div>
+        </div>
+      </div>
+    </div>
+  </div>
+</template>
+
+<route lang="yaml">
+meta:
+  layout: plain
+</route>
@@ -80,6 +80,12 @@ const menu = computed(() => [
    icon: 'i-solar:sledgehammer-bold-duotone',
    to: '/devtools/performance-playground',
  },
+  {
+    title: 'MediaPipe Workshop',
+    description: 'Single-person mocap playground (MediaPipe backend) with scheduling knobs',
+    icon: 'i-solar:sledgehammer-bold-duotone',
+    to: '/devtools/model-driver-mediapipe',
+  },
 ])
 </script>

@@ -162,6 +162,7 @@ words:
  - matchall
  - mdit
  - mediabunny
+  - mediapipe
  - MeshToonMaterial
  - micvad
  - mineflayer
@@ -0,0 +1,35 @@
+# model-driver-mediapipe Agent Notes
+
+Scope: `packages/model-driver-mediapipe/**`
+
+## Intent
+
+This package is an experimental single-person mocap pipeline for stage-web devtools.
+
+`camera frame` → `@mediapipe/tasks-vision` → `PerceptionState` → `overlay`
+
+## Style / Conventions
+
+- Prefer functional programming (FP) and pure functions where practical.
+  - Use factory functions + closures for stateful modules (example: `createMocapEngine()`).
+  - Avoid classes unless extending browser APIs or required by external libraries.
+- Keep the backend boundary clean:
+  - Engine/scheduler should not import `@mediapipe/tasks-vision`.
+  - MediaPipe specifics live under `src/backends/`.
+- Keep types stable and narrow:
+  - Stage consumers depend on `src/types.ts` as the contract.
+  - New fields should be optional and backwards compatible.
+
+## Key Files
+
+- `src/types.ts`: middle-layer contract (`PerceptionState`, config types)
+- `src/engine.ts`: scheduling + dropped-frame policy + partial merge (FP)
+- `src/backends/mediapipe.ts`: MediaPipe Tasks Vision adapter (sync `detectForVideo`)
+- `src/overlay.ts`: debug overlay renderer (points + connectors)
+- `references/tasks-vision-api.md`: minimal upstream API notes
+
+## Performance Notes
+
+- `detectForVideo()` is synchronous; avoid blocking UI:
+  - Engine drops frames when backend reports `isBusy()`.
+  - Scheduler controls per-task rates (`hz`) to cap work.
@@ -0,0 +1,35 @@
+# @proj-airi/model-driver-mediapipe
+
+Single-person motion capture workshop package.
+
+**Goal**
+
+Provide a minimal closed loop that stage-web can consume:
+
+`camera frame` → `MediaPipe Tasks Vision` → `PerceptionState` → `canvas overlay`
+
+**Where to try it**
+
+- Devtools page: `apps/stage-web/src/pages/devtools/model-driver-mediapipe.vue`
+- Menu entry: Settings → System → Developer → “MediaPipe Workshop”
+
+**Key files**
+
+- `packages/model-driver-mediapipe/src/types.ts`: middle-layer contract (`PerceptionState`)
+- `packages/model-driver-mediapipe/src/engine.ts`: scheduler + dropped-frame policy + state merge
+- `packages/model-driver-mediapipe/src/backends/mediapipe.ts`: `@mediapipe/tasks-vision` integration
+- `packages/model-driver-mediapipe/src/utils/overlay.ts`: canvas overlay renderer
+
+**Backend assumptions**
+
+- Single person (`maxPeople: 1`)
+- Running mode: `VIDEO`
+- Landmarks are normalized (`x`/`y` in `[0..1]`) and drawn onto the overlay canvas
+
+**Docs**
+
+Keep upstream docs/snippets in `packages/model-driver-mediapipe/references/`.
+
+The minimal API surface this package uses is summarized in:
+
+- `packages/model-driver-mediapipe/references/tasks-vision-api.md`
@@ -0,0 +1,36 @@
+{
+  "name": "@proj-airi/model-driver-mediapipe",
+  "type": "module",
+  "private": true,
+  "description": "MediaPipe motion capture workshop (experimental) for Project AIRI",
+  "author": {
+    "name": "Moeru AI Project AIRI Team",
+    "email": "airi@moeru.ai",
+    "url": "https://github.com/moeru-ai"
+  },
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/moeru-ai/airi.git",
+    "directory": "packages/model-driver-mediapipe"
+  },
+  "exports": {
+    ".": "./src/index.ts",
+    "./types": "./src/types.ts"
+  },
+  "scripts": {
+    "typecheck": "tsc --noEmit",
+    "postinstall": "tsx ./tasks/prepare-tasks.ts"
+  },
+  "dependencies": {
+    "@mediapipe/tasks-vision": "^0.10.0",
+    "@pixiv/three-vrm": "^3.4.4",
+    "es-toolkit": "catalog:",
+    "three": "^0.182.0"
+  },
+  "devDependencies": {
+    "@types/three": "^0.182.0",
+    "ofetch": "catalog:",
+    "tsx": "catalog:"
+  }
+}
@@ -0,0 +1,3 @@
+# References
+
+Put MediaPipe Tasks Vision API docs / snippets / notes here so the implementation can follow the exact API surface you’re using.
@@ -0,0 +1,44 @@
+# MediaPipe Tasks Vision (minimal API notes)
+
+This package targets `@mediapipe/tasks-vision` (web, JS/TS).
+
+## Core init
+
+```ts
+import { FilesetResolver, PoseLandmarker } from '@mediapipe/tasks-vision'
+
+const vision = await FilesetResolver.forVisionTasks(wasmRoot)
+const pose = await PoseLandmarker.createFromOptions(vision, {
+  baseOptions: { modelAssetPath: poseModelUrl },
+  runningMode: 'VIDEO',
+  numPoses: 1,
+})
+```
+
+## Video inference
+
+MediaPipe Tasks Vision `detectForVideo()` runs synchronously and can block the main thread.
+
+```ts
+const nowMs = performance.now()
+const res = pose.detectForVideo(videoEl, nowMs)
+```
+
+## Result shapes (single-person usage)
+
+### Pose
+
+- `res.landmarks`: `NormalizedLandmark[][]` (take `[0]` for single person)
+- each landmark includes `{ x, y, z, visibility?, presence? }` with `x/y` in `[0..1]`
+
+### Hands
+
+- `res.landmarks`: `NormalizedLandmark[][]` (each entry is 21 landmarks for one hand)
+- `res.handedness`: `Category[][]` aligned with `landmarks`
+  - `handedness[i][0].categoryName` is typically `'Left' | 'Right'`
+  - `handedness[i][0].score` is confidence
+
+### Face (optional)
+
+- `res.faceLandmarks`: `NormalizedLandmark[][]` (468 landmarks, heavy)
+- For the workshop we treat this as presence-only by default.
@@ -0,0 +1,158 @@
+import type {
+  Category,
+  FaceLandmarker,
+  FaceLandmarkerResult,
+  HandLandmarker,
+  HandLandmarkerResult,
+  Landmark,
+  NormalizedLandmark,
+  PoseLandmarker,
+  PoseLandmarkerResult,
+} from '@mediapipe/tasks-vision'
+
+import type { MocapBackend, MocapConfig, MocapJob, PerceptionPartial, VisionTaskModule, VisionTaskWasmFileset } from '../types'
+
+import { Semaphore } from 'es-toolkit'
+
+import { visionTaskAssets, visionTaskWasmRoot } from '../../tasks/tasks'
+
+export function createMediaPipeBackend(): MocapBackend {
+  const semaphore = new Semaphore(1)
+  let busy = false
+  let config: MocapConfig | undefined
+  let tasksVision: VisionTaskModule | undefined
+  let vision: VisionTaskWasmFileset | undefined
+
+  let poseLandmarker: PoseLandmarker | undefined
+  let handLandmarker: HandLandmarker | undefined
+  let faceLandmarker: FaceLandmarker | undefined
+
+  async function init(nextConfig: MocapConfig) {
+    config = nextConfig
+
+    if (!tasksVision)
+      tasksVision = await import('@mediapipe/tasks-vision')
+
+    if (!vision) {
+      const { FilesetResolver } = tasksVision
+      vision = await FilesetResolver.forVisionTasks(visionTaskWasmRoot)
+    }
+  }
+
+  function isBusy() {
+    return busy
+  }
+
+  async function ensurePoseLandmarker() {
+    if (poseLandmarker)
+      return poseLandmarker
+
+    const { PoseLandmarker } = tasksVision!
+    poseLandmarker = await PoseLandmarker.createFromOptions(vision!, {
+      baseOptions: { modelAssetPath: visionTaskAssets.pose },
+      runningMode: 'VIDEO',
+      numPoses: 1,
+    })
+
+    return poseLandmarker
+  }
+
+  async function ensureHandLandmarker() {
+    if (handLandmarker)
+      return handLandmarker
+
+    const { HandLandmarker } = tasksVision!
+    handLandmarker = await HandLandmarker.createFromOptions(vision!, {
+      baseOptions: { modelAssetPath: visionTaskAssets.hands },
+      runningMode: 'VIDEO',
+      numHands: 2,
+    })
+
+    return handLandmarker
+  }
+
+  async function ensureFaceLandmarker() {
+    if (faceLandmarker)
+      return faceLandmarker
+
+    const { FaceLandmarker } = tasksVision!
+    faceLandmarker = await FaceLandmarker.createFromOptions(vision!, {
+      baseOptions: { modelAssetPath: visionTaskAssets.face },
+      runningMode: 'VIDEO',
+      numFaces: 1,
+    })
+
+    return faceLandmarker
+  }
+
+  async function run(frame: TexImageSource, jobs: MocapJob[], nowMs: number): Promise<PerceptionPartial> {
+    if (!config)
+      throw new Error('MediaPipe backend not initialized (call init() first)')
+
+    await semaphore.acquire()
+    busy = true
+    try {
+      const partial: PerceptionPartial = {}
+
+      for (const job of jobs) {
+        if (!config.enabled[job])
+          continue
+
+        if (job === 'pose') {
+          const landmarker = await ensurePoseLandmarker()
+          const res: PoseLandmarkerResult = landmarker.detectForVideo(frame, nowMs)
+          const firstPose: NormalizedLandmark[] = res.landmarks[0] ?? []
+          const firstWorld: Landmark[] = res.worldLandmarks[0] ?? []
+          partial.pose = {
+            landmarks2d: firstPose,
+            worldLandmarks: firstWorld.map(p => ({
+              x: p.x,
+              y: p.y,
+              z: p.z,
+              visibility: p.visibility,
+            })),
+          }
+        }
+        else if (job === 'hands') {
+          const landmarker = await ensureHandLandmarker()
+          const res: HandLandmarkerResult = landmarker.detectForVideo(frame, nowMs)
+          const landmarks: NormalizedLandmark[][] = res.landmarks ?? []
+          const handedness: Category[][] = res.handedness ?? []
+
+          partial.hands = landmarks.map((lm, i) => {
+            const mostLikelyCategory = handedness[i]?.[0]
+            const categoryName = mostLikelyCategory?.categoryName
+            const score = mostLikelyCategory?.score
+            const handed = categoryName === 'Left' || categoryName === 'Right' ? categoryName : 'Right'
+            return {
+              handedness: handed,
+              landmarks2d: lm,
+              score,
+            }
+          })
+        }
+        else if (job === 'face') {
+          const landmarker = await ensureFaceLandmarker()
+          const res: FaceLandmarkerResult = landmarker.detectForVideo(frame, nowMs)
+          const firstFace: NormalizedLandmark[] = res.faceLandmarks?.[0] ?? []
+          partial.face = {
+            hasFace: firstFace.length > 0,
+            landmarks2d: firstFace,
+          }
+        }
+      }
+
+      return partial
+    }
+    finally {
+      busy = false
+      semaphore.release()
+    }
+  }
+
+  return {
+    init,
+    isBusy,
+    run,
+  }
+}
@@ -0,0 +1,156 @@
+import type { FrameSource, MocapBackend, MocapConfig, MocapEngine, MocapJob, PerceptionPartial, PerceptionState } from './types'
+
+export function createStats() {
+  let lastTs = 0
+  let smoothedFps = 0
+
+  function tick(nowMs: number) {
+    if (!lastTs) {
+      lastTs = nowMs
+      smoothedFps = 0
+      return 0
+    }
+
+    const dt = nowMs - lastTs
+    lastTs = nowMs
+
+    if (dt <= 0)
+      return smoothedFps
+
+    const fps = 1000 / dt
+    smoothedFps = smoothedFps ? (smoothedFps * 0.9 + fps * 0.1) : fps
+
+    return smoothedFps
+  }
+
+  return { tick }
+}
+
+export function createScheduler(initialConfig: MocapConfig) {
+  let config = initialConfig
+  const lastRun: Record<MocapJob, number> = { pose: 0, hands: 0, face: 0 }
+
+  function updateConfig(next: MocapConfig) {
+    config = next
+  }
+
+  function plan(nowMs: number): MocapJob[] {
+    const jobs: MocapJob[] = []
+
+    for (const job of ['pose', 'hands', 'face'] as const) {
+      if (!config.enabled[job])
+        continue
+
+      const hz = config.hz[job]
+      if (!hz || hz <= 0)
+        continue
+
+      if (nowMs - lastRun[job] >= (1000 / hz))
+        jobs.push(job)
+    }
+
+    for (const j of jobs)
+      lastRun[j] = nowMs
+
+    return jobs
+  }
+
+  return {
+    plan,
+    updateConfig,
+  }
+}
+
+export function createMocapEngine(backend: MocapBackend, initialConfig: MocapConfig): MocapEngine {
+  let config = initialConfig
+  const scheduler = createScheduler(config)
+  const stats = createStats()
+
+  let running = false
+  let droppedFrames = 0
+  let lastPartial: PerceptionPartial = {}
+  let rafId: number | undefined
+
+  async function init() {
+    await backend.init(config)
+  }
+
+  function updateConfig(next: MocapConfig) {
+    config = next
+    scheduler.updateConfig(next)
+  }
+
+  function resetState() {
+    lastPartial = {}
+    droppedFrames = 0
+  }
+
+  function start(
+    source: FrameSource,
+    onState: (state: PerceptionState) => void,
+    options?: { onError?: (error: unknown) => void },
+  ) {
+    running = true
+    resetState()
+
+    const tick = async () => {
+      try {
+        if (!running)
+          return
+
+        const frame = source.getFrame()
+        const now = performance.now()
+
+        // Skip this frame if the backend is still busy.
+        if (backend.isBusy()) {
+          droppedFrames++
+          rafId = requestAnimationFrame(tick)
+          return
+        }
+
+        const jobs = scheduler.plan(now)
+        const t0 = performance.now()
+        const partial = jobs.length > 0 ? await backend.run(frame, jobs, now) : {}
+        const latencyMs = performance.now() - t0
+
+        lastPartial = { ...lastPartial, ...partial }
+
+        onState({
+          t: now,
+          ...lastPartial,
+          quality: {
+            fps: stats.tick(now),
+            latencyMs,
+            droppedFrames,
+            backend: 'mediapipe',
+            mode: 'split-tasks',
+          },
+        })
+
+        rafId = requestAnimationFrame(tick)
+      }
+      catch (err) {
+        // Stop the loop to avoid spamming errors; consumers can restart.
+        stop()
+        options?.onError?.(err)
+      }
+    }
+
+    rafId = requestAnimationFrame(tick)
+  }
+
+  function stop() {
+    running = false
+    if (rafId != null)
+      cancelAnimationFrame(rafId)
+    rafId = undefined
+  }
+
+  return {
+    init,
+    start,
+    stop,
+    updateConfig,
+    resetState,
+  }
+}
@@ -0,0 +1,5 @@
+export * from './backends/mediapipe'
+export * from './engine'
+export * from './three'
+export * from './types'
+export * from './utils/overlay'
@@ -0,0 +1,361 @@
+import type { VRM } from '@pixiv/three-vrm'
+
+import type { VrmPoseDirections, VrmPoseTarget, VrmPoseTargets } from './pose-to-vrm'
+
+import { Matrix4, Quaternion, Vector3 } from 'three'
+
+export interface VrmPoseApplyOptions {
+  /**
+   * Slerp factor per frame in [0..1]. Higher = snappier, lower = smoother.
+   */
+  alpha?: number
+  /**
+   * Reject sudden flips: if the target direction is close to the opposite of the
+   * current bone direction (in world space), skip this update.
+   *
+   * `minDot` in [-1..1]. Example: `-0.2` rejects angles > ~101°.
+   */
+  minDotBeforeReject?: number
+  /**
+   * Similar to `minDotBeforeReject`, but applied to the pole vector when using
+   * pole-based orientation. Helps avoid 180° roll/yaw flips when the pole sign
+   * becomes ambiguous.
+   */
+  minPoleDotBeforeReject?: number
+}
+
+type BoneKey = keyof VrmPoseDirections
+
+interface BoneChain {
+  bone: string
+  childCandidates: readonly string[]
+}
+
+const DEFAULT_ALPHA = 0.35
+
+const CHAINS: Readonly<Record<BoneKey, BoneChain>> = {
+  hips: {
+    bone: 'hips',
+    childCandidates: ['spine'],
+  },
+  spine: {
+    bone: 'spine',
+    childCandidates: ['chest', 'upperChest', 'neck'],
+  },
+  chest: {
+    bone: 'chest',
+    childCandidates: ['upperChest', 'neck'],
+  },
+  leftShoulder: {
+    bone: 'leftShoulder',
+    childCandidates: ['leftUpperArm'],
+  },
+  rightShoulder: {
+    bone: 'rightShoulder',
+    childCandidates: ['rightUpperArm'],
+  },
+  leftUpperArm: {
+    bone: 'leftUpperArm',
+    childCandidates: ['leftLowerArm'],
+  },
+  leftLowerArm: {
+    bone: 'leftLowerArm',
+    childCandidates: ['leftHand'],
+  },
+  rightUpperArm: {
+    bone: 'rightUpperArm',
+    childCandidates: ['rightLowerArm'],
+  },
+  rightLowerArm: {
+    bone: 'rightLowerArm',
+    childCandidates: ['rightHand'],
+  },
+  leftUpperLeg: {
+    bone: 'leftUpperLeg',
+    childCandidates: ['leftLowerLeg'],
+  },
+  leftLowerLeg: {
+    bone: 'leftLowerLeg',
+    childCandidates: ['leftFoot'],
+  },
+  rightUpperLeg: {
+    bone: 'rightUpperLeg',
+    childCandidates: ['rightLowerLeg'],
+  },
+  rightLowerLeg: {
+    bone: 'rightLowerLeg',
+    childCandidates: ['rightFoot'],
+  },
+} as const
+
+const POLE_KEYS: ReadonlySet<BoneKey> = new Set(Object.keys(CHAINS) as BoneKey[])
+const LIMB_POLE_KEYS: ReadonlySet<BoneKey> = new Set([
+  'leftUpperArm',
+  'rightUpperArm',
+  'leftUpperLeg',
+  'rightUpperLeg',
+])
+
+function isFiniteVec3(v: { x: number, y: number, z?: number } | undefined): v is { x: number, y: number, z?: number } {
+  if (!v)
+    return false
+  return Number.isFinite(v.x) && Number.isFinite(v.y) && Number.isFinite(v.z ?? 0)
+}
+
+function firstExistingBone(vrm: VRM, candidates: readonly string[]) {
+  for (const name of candidates) {
+    const node = vrm.humanoid?.getNormalizedBoneNode(name as any)
+    if (node)
+      return node
+  }
+  return null
+}
+
+function grandchildCandidatesFor(key: BoneKey): readonly string[] {
+  if (key === 'leftUpperArm')
+    return ['leftHand']
+  if (key === 'rightUpperArm')
+    return ['rightHand']
+  if (key === 'leftUpperLeg')
+    return ['leftFoot']
+  if (key === 'rightUpperLeg')
+    return ['rightFoot']
+  return []
+}
+
+function orthonormalizePole(dir: Vector3, pole: Vector3): Vector3 | null {
+  const poleOrtho = pole.clone().addScaledVector(dir, -pole.dot(dir))
+  if (poleOrtho.lengthSq() <= 1e-12)
+    return null
+  return poleOrtho.normalize()
+}
+
+export function createVrmPoseApplier(options?: VrmPoseApplyOptions) {
+  const alpha = options?.alpha ?? DEFAULT_ALPHA
+  const minDotBeforeReject = options?.minDotBeforeReject ?? -0.2
+  const minPoleDotBeforeReject = options?.minPoleDotBeforeReject ?? -0.2
+
+  const restDirLocal: Partial<Record<BoneKey, Vector3>> = {}
+  const restPoleLocal: Partial<Record<BoneKey, Vector3>> = {}
+  const lastTargetDirWorld: Partial<Record<BoneKey, Vector3>> = {}
+  const lastTargetPoleWorld: Partial<Record<BoneKey, Vector3>> = {}
+
+  const tmpBoneWorldQ = new Quaternion()
+  const tmpParentWorldQ = new Quaternion()
+  const tmpDeltaQ = new Quaternion()
+  const tmpNewWorldQ = new Quaternion()
+  const tmpNewLocalQ = new Quaternion()
+
+  const tmpRestM = new Matrix4()
+  const tmpTargetM = new Matrix4()
+  const tmpRotM = new Matrix4()
+
+  const tmpBoneWorldPos = new Vector3()
+  const tmpChildWorldPos = new Vector3()
+  const tmpGrandWorldPos = new Vector3()
+  const tmpDirWorld = new Vector3()
+  const tmpDirLocal = new Vector3()
+  const tmpCurrentDirWorld = new Vector3()
+  const tmpTargetDirWorld = new Vector3()
+  const tmpTargetPoleWorld = new Vector3()
+  const tmpRestDirLocal = new Vector3()
+  const tmpRestPoleLocal = new Vector3()
+  const tmpRestYLocal = new Vector3()
+  const tmpTargetYWorld = new Vector3()
+  const tmpWorldForward = new Vector3(0, 0, -1)
+
+  function ensureRestDirection(vrm: VRM, key: BoneKey): Vector3 | null {
+    const existing = restDirLocal[key]
+    if (existing)
+      return existing
+
+    const chain = CHAINS[key]
+    const bone = vrm.humanoid?.getNormalizedBoneNode(chain.bone as any)
+    if (!bone)
+      return null
+
+    const child = firstExistingBone(vrm, chain.childCandidates)
+    if (!child)
+      return null
+
+    bone.updateMatrixWorld(true)
+    child.updateMatrixWorld(true)
+
+    bone.getWorldQuaternion(tmpBoneWorldQ)
+    bone.getWorldPosition(tmpBoneWorldPos)
+    child.getWorldPosition(tmpChildWorldPos)
+
+    tmpDirWorld.copy(tmpChildWorldPos).sub(tmpBoneWorldPos).normalize()
+    tmpDirLocal.copy(tmpDirWorld).applyQuaternion(tmpBoneWorldQ.clone().invert()).normalize()
+
+    const stored = tmpDirLocal.clone()
+    restDirLocal[key] = stored
+    return stored
+  }
+
+  function ensureRestPole(vrm: VRM, key: BoneKey): Vector3 | null {
+    if (!POLE_KEYS.has(key))
+      return null
+
+    const existing = restPoleLocal[key]
+    if (existing)
+      return existing
+
+    const chain = CHAINS[key]
+    const bone = vrm.humanoid?.getNormalizedBoneNode(chain.bone as any)
+    if (!bone)
+      return null
+
+    bone.updateMatrixWorld(true)
+    bone.getWorldQuaternion(tmpBoneWorldQ)
+
+    if (LIMB_POLE_KEYS.has(key)) {
+      const child = firstExistingBone(vrm, chain.childCandidates)
+      if (!child)
+        return null
+
+      const grandCandidates = grandchildCandidatesFor(key)
+      const grand = grandCandidates.length ? firstExistingBone(vrm, grandCandidates) : null
+      if (!grand)
+        return null
+
+      child.updateMatrixWorld(true)
+      grand.updateMatrixWorld(true)
+
+      bone.getWorldPosition(tmpBoneWorldPos)
+      child.getWorldPosition(tmpChildWorldPos)
+      grand.getWorldPosition(tmpGrandWorldPos)
+
+      const dir1 = tmpChildWorldPos.clone().sub(tmpBoneWorldPos)
+      const dir2 = tmpGrandWorldPos.clone().sub(tmpChildWorldPos)
+      tmpDirWorld.copy(dir1).cross(dir2)
+      if (tmpDirWorld.lengthSq() <= 1e-12)
+        return null
+      tmpDirWorld.normalize()
+
+      tmpDirLocal.copy(tmpDirWorld).applyQuaternion(tmpBoneWorldQ.clone().invert()).normalize()
+      const stored = tmpDirLocal.clone()
+      restPoleLocal[key] = stored
+      return stored
+    }
+
+    // For torso: use "world forward" as rest pole, transformed into bone local.
+    tmpDirLocal.copy(tmpWorldForward).applyQuaternion(tmpBoneWorldQ.clone().invert()).normalize()
+    const stored = tmpDirLocal.clone()
+    restPoleLocal[key] = stored
+    return stored
+  }
+
+  function applyOne(vrm: VRM, key: BoneKey, target: VrmPoseTarget) {
+    const chain = CHAINS[key]
+    const bone = vrm.humanoid?.getNormalizedBoneNode(chain.bone as any)
+    if (!bone)
+      return
+
+    const rest = ensureRestDirection(vrm, key)
+    if (!rest)
+      return
+
+    tmpTargetDirWorld.set(target.dir.x, target.dir.y, target.dir.z ?? 0)
+    if (tmpTargetDirWorld.lengthSq() <= 1e-12)
+      return
+    tmpTargetDirWorld.normalize()
+
+    // Reject near-180° instant flips based on previous target (not current bone pose).
+    // Using the current bone direction is unreliable when tracking reacquires or when
+    // the bind pose differs from the mocap space.
+    const prevDir = lastTargetDirWorld[key]
+    if (prevDir && prevDir.dot(tmpTargetDirWorld) < minDotBeforeReject)
+      return
+
+    bone.updateMatrixWorld(true)
+    bone.getWorldQuaternion(tmpBoneWorldQ)
+
+    const parent = bone.parent
+    if (parent) {
+      parent.updateMatrixWorld(true)
+      parent.getWorldQuaternion(tmpParentWorldQ)
+    }
+    else {
+      tmpParentWorldQ.identity()
+    }
+
+    const restPole = target.pole ? ensureRestPole(vrm, key) : null
+    const usePole = !!(target.pole && restPole)
+
+    if (usePole) {
+      tmpRestDirLocal.copy(rest).normalize()
+      tmpRestPoleLocal.copy(restPole!).normalize()
+      tmpRestPoleLocal.addScaledVector(tmpRestDirLocal, -tmpRestPoleLocal.dot(tmpRestDirLocal)).normalize()
+      tmpRestYLocal.copy(tmpRestPoleLocal).cross(tmpRestDirLocal).normalize()
+
+      tmpTargetPoleWorld.set(target.pole!.x, target.pole!.y, target.pole!.z ?? 0)
+      if (tmpTargetPoleWorld.lengthSq() <= 1e-12)
+        return
+      tmpTargetPoleWorld.normalize()
+      const poleOrtho = orthonormalizePole(tmpTargetDirWorld, tmpTargetPoleWorld)
+      if (!poleOrtho)
+        return
+      tmpTargetPoleWorld.copy(poleOrtho)
+      tmpTargetYWorld.copy(tmpTargetPoleWorld).cross(tmpTargetDirWorld).normalize()
+
+      // Reject near-180° pole flips based on previous target pole.
+      const prevPole = lastTargetPoleWorld[key]
+      if (prevPole && prevPole.dot(tmpTargetPoleWorld) < minPoleDotBeforeReject)
+        return
+
+      tmpRestM.makeBasis(tmpRestDirLocal, tmpRestYLocal, tmpRestPoleLocal)
+      tmpTargetM.makeBasis(tmpTargetDirWorld, tmpTargetYWorld, tmpTargetPoleWorld)
+
+      tmpRotM.copy(tmpTargetM).multiply(tmpRestM.clone().invert())
+      tmpNewWorldQ.setFromRotationMatrix(tmpRotM)
+      tmpNewLocalQ.copy(tmpParentWorldQ).invert().multiply(tmpNewWorldQ)
+    }
+    else {
+      tmpCurrentDirWorld.copy(rest).applyQuaternion(tmpBoneWorldQ).normalize()
+      tmpDeltaQ.setFromUnitVectors(tmpCurrentDirWorld, tmpTargetDirWorld)
+      tmpNewWorldQ.copy(tmpDeltaQ).multiply(tmpBoneWorldQ)
+      tmpNewLocalQ.copy(tmpParentWorldQ).invert().multiply(tmpNewWorldQ)
+    }
+
+    if (alpha >= 1)
+      bone.quaternion.copy(tmpNewLocalQ)
+    else
+      bone.quaternion.slerp(tmpNewLocalQ, alpha)
+
+    // Update last targets only after a successful apply.
+    lastTargetDirWorld[key] = tmpTargetDirWorld.clone()
+    if (usePole)
+      lastTargetPoleWorld[key] = tmpTargetPoleWorld.clone()
+    else
+      delete lastTargetPoleWorld[key]
+  }
+
+  function applyPoseDirectionsToVrm(vrm: VRM, directions: VrmPoseDirections) {
+    if (!vrm.humanoid)
+      return
+
+    (Object.keys(CHAINS) as BoneKey[]).forEach((key) => {
+      const d = directions[key]
+      if (!isFiniteVec3(d))
+        return
+      applyOne(vrm, key, { dir: d })
+    })
+  }
+
+  function applyPoseTargetsToVrm(vrm: VRM, targets: VrmPoseTargets) {
+    if (!vrm.humanoid)
+      return
+
+    (Object.keys(CHAINS) as BoneKey[]).forEach((key) => {
+      const t = targets[key]
+      if (!t || !isFiniteVec3(t.dir))
+        return
+      if (t.pole && !isFiniteVec3(t.pole))
+        return
+      applyOne(vrm, key, t)
+    })
+  }
+
+  return { applyPoseDirectionsToVrm, applyPoseTargetsToVrm }
+}
@@ -0,0 +1,2 @@
+export * from './apply-pose-to-vrm'
+export * from './pose-to-vrm'
@@ -0,0 +1,347 @@
+import type { Vector3Like } from 'three'
+
+import type { PoseState } from '../types'
+
+export interface PoseToVrmOptions {
+  /**
+   * Axis remap from MediaPipe world to three/VRM-ish space.
+   * This is a pragmatic default; adjust if you see mirrored or inverted motion.
+   */
+  axis?: {
+    x: 1 | -1
+    y: 1 | -1
+    z: 1 | -1
+  }
+
+  /**
+   * Confidence gating based on MediaPipe pose landmark `visibility` / `presence`.
+   * When a landmark is not confident (e.g. off-screen), we skip emitting targets that depend on it.
+   */
+  confidence?: {
+    /**
+     * Min `visibility` in [0..1]. Only enforced when the field exists on the landmark.
+     */
+    minVisibility?: number
+    /**
+     * Min `presence` in [0..1]. Only enforced when the field exists on the landmark.
+     */
+    minPresence?: number
+  }
+
+  /**
+   * Use previous frame information to avoid sudden 180° flips caused by ambiguous poles.
+   */
+  stabilize?: {
+    previousTargets?: VrmPoseTargets
+    previousForward?: Vector3Like
+  }
+}
+
+export type VrmPoseDirections = Partial<Record<
+  | 'hips'
+  | 'spine'
+  | 'chest'
+  | 'leftShoulder'
+  | 'rightShoulder'
+  | 'leftUpperArm'
+  | 'leftLowerArm'
+  | 'rightUpperArm'
+  | 'rightLowerArm'
+  | 'leftUpperLeg'
+  | 'leftLowerLeg'
+  | 'rightUpperLeg'
+  | 'rightLowerLeg',
+  Vector3Like
+>>
+
+export interface VrmPoseTarget {
+  dir: Vector3Like
+  pole?: Vector3Like
+}
+
+export type VrmPoseTargets = Partial<Record<keyof VrmPoseDirections, VrmPoseTarget>>
+
+const DEFAULT_AXIS = { x: 1 as const, y: 1 as const, z: 1 as const }
+const DEFAULT_MIN_VISIBILITY = 0.5
+const DEFAULT_MIN_PRESENCE = 0
+
+// TODO: Consider consolidating these vector helpers into a shared math utility if more drivers need them.
+function vSub(a: Vector3Like, b: Vector3Like): Vector3Like {
+  return { x: a.x - b.x, y: a.y - b.y, z: (a.z ?? 0) - (b.z ?? 0) }
+}
+
+function vAdd(a: Vector3Like, b: Vector3Like): Vector3Like {
+  return { x: a.x + b.x, y: a.y + b.y, z: (a.z ?? 0) + (b.z ?? 0) }
+}
+
+function vScale(v: Vector3Like, s: number): Vector3Like {
+  return { x: v.x * s, y: v.y * s, z: (v.z ?? 0) * s }
+}
+
+function vLen(v: Vector3Like): number {
+  return Math.hypot(v.x, v.y, v.z ?? 0)
+}
+
+function vNormalize(v: Vector3Like): Vector3Like | null {
+  const len = vLen(v)
+  if (!Number.isFinite(len) || len <= 1e-6)
+    return null
+  return vScale(v, 1 / len)
+}
+
+function vRemapAxis(v: Vector3Like, axis: { x: 1 | -1, y: 1 | -1, z: 1 | -1 }): Vector3Like {
+  return { x: v.x * axis.x, y: v.y * axis.y, z: (v.z ?? 0) * axis.z }
+}
+
+function vCross(a: Vector3Like, b: Vector3Like): Vector3Like {
+  const az = a.z ?? 0
+  const bz = b.z ?? 0
+  return {
+    x: a.y * bz - az * b.y,
+    y: az * b.x - a.x * bz,
+    z: a.x * b.y - a.y * b.x,
+  }
+}
+
+function vDot(a: Vector3Like, b: Vector3Like): number {
+  return a.x * b.x + a.y * b.y + (a.z ?? 0) * (b.z ?? 0)
+}
+
+function vNeg(v: Vector3Like): Vector3Like {
+  return { x: -v.x, y: -v.y, z: -(v.z ?? 0) }
+}
+
+function safePole(dir: Vector3Like, pole: Vector3Like, threshold = 0.85): Vector3Like | null {
+  const d = Math.abs(vDot(dir, pole))
+  if (!Number.isFinite(d))
+    return null
+  return d > threshold ? null : pole
+}
+
+function get(points: Vector3Like[], index: number): Vector3Like | null {
+  const p = points[index]
+  if (!p)
+    return null
+  if (!Number.isFinite(p.x) || !Number.isFinite(p.y))
+    return null
+  return { x: p.x, y: p.y, z: p.z ?? 0 }
+}
+
+// NOTICE: mediapipe doesn't provide this type correctly, so we define it here.
+interface LandmarkWithPresence { presence?: number }
+interface LandmarkWithVisibility { visibility?: number }
+
+function getOptionalPresence(landmark: unknown): number | undefined {
+  if (landmark && typeof landmark === 'object' && 'presence' in landmark)
+    return (landmark as LandmarkWithPresence).presence
+  return undefined
+}
+
+function getOptionalVisibility(landmark: unknown): number | undefined {
+  if (landmark && typeof landmark === 'object' && 'visibility' in landmark)
+    return (landmark as LandmarkWithVisibility).visibility
+  return undefined
+}
+
+function isConfident(pose: PoseState, index: number, thresholds: { minVisibility: number, minPresence: number }): boolean {
+  // User requirement: do not output anything when `visibility` is missing.
+  // Prefer 2D landmarks for visibility/presence (they are more consistently populated),
+  // but still allow fallback to world landmarks when needed.
+  const lm2d = pose.landmarks2d?.[index]
+  const lm3d = pose.worldLandmarks?.[index]
+  if (!lm2d && !lm3d)
+    return false
+
+  const visibility = getOptionalVisibility(lm2d) ?? getOptionalVisibility(lm3d)
+  if (visibility == null || !Number.isFinite(visibility))
+    return false
+  if (visibility < thresholds.minVisibility)
+    return false
+
+  if (thresholds.minPresence > 0) {
+    const presence = getOptionalPresence(lm2d) ?? getOptionalPresence(lm3d)
+    if (presence != null && Number.isFinite(presence) && presence < thresholds.minPresence)
+      return false
+  }
+  return true
+}
+
+function mid(a: Vector3Like, b: Vector3Like): Vector3Like {
+  return vScale(vAdd(a, b), 0.5)
+}
+
+export function poseToVrmTargets(pose: PoseState, options?: PoseToVrmOptions): VrmPoseTargets {
+  // Prefer world landmarks when available; fallback to normalized landmarks to keep the pipeline usable.
+  const points = pose.worldLandmarks
+  if (!points?.length)
+    return {}
+
+  const axis = options?.axis ?? DEFAULT_AXIS
+  const thresholds = {
+    minVisibility: options?.confidence?.minVisibility ?? DEFAULT_MIN_VISIBILITY,
+    minPresence: options?.confidence?.minPresence ?? DEFAULT_MIN_PRESENCE,
+  }
+
+  const getC = (index: number) => (isConfident(pose, index, thresholds) ? get(points, index) : null)
+
+  const leftShoulder = getC(11)
+  const rightShoulder = getC(12)
+  const leftElbow = getC(13)
+  const rightElbow = getC(14)
+  const leftWrist = getC(15)
+  const rightWrist = getC(16)
+  const leftHip = getC(23)
+  const rightHip = getC(24)
+  const leftKnee = getC(25)
+  const rightKnee = getC(26)
+  const leftAnkle = getC(27)
+  const rightAnkle = getC(28)
+
+  const out: VrmPoseTargets = {}
+
+  const shoulderCenter = leftShoulder && rightShoulder ? mid(leftShoulder, rightShoulder) : null
+  const hipCenter = leftHip && rightHip ? mid(leftHip, rightHip) : null
+
+  const prevTargets = options?.stabilize?.previousTargets
+  const stabilizePole = (key: keyof VrmPoseTargets, pole: Vector3Like): Vector3Like => {
+    const prev = prevTargets?.[key]?.pole
+    if (prev && vDot(prev, pole) < 0)
+      return vNeg(pole)
+    return pole
+  }
+
+  // Torso basis:
+  // - up: hipCenter -> shoulderCenter (dir)
+  // - right: leftShoulder -> rightShoulder
+  // - forward: cross(right, up)
+  //
+  // NOTICE: In apply-pose-to-vrm.ts, pole is used as the Z axis in `makeBasis(X=dir, Y=..., Z=pole)`,
+  // so torso pole must be "forward-like" (body facing), not "right-like" (shoulder line).
+  let torsoForward: Vector3Like | null = null
+  if (hipCenter && shoulderCenter && leftShoulder && rightShoulder) {
+    const rightRaw = vSub(rightShoulder, leftShoulder)
+    const upRaw = vSub(shoulderCenter, hipCenter)
+    const right = vNormalize(vRemapAxis(rightRaw, axis))
+    const up = vNormalize(vRemapAxis(upRaw, axis))
+    if (right && up) {
+      // Base forward from right×up, then fix sign so that (up×forward) matches observed shoulder-right.
+      let fw = vNormalize(vCross(right, up))
+      if (fw) {
+        const rightFromBasis = vNormalize(vCross(fw, up))
+        if (rightFromBasis && vDot(rightFromBasis, right) < 0)
+          fw = vNeg(fw)
+
+        const prevForward = options?.stabilize?.previousForward ?? prevTargets?.hips?.pole ?? prevTargets?.spine?.pole ?? prevTargets?.chest?.pole
+        torsoForward = prevForward && vDot(prevForward, fw) < 0 ? vNeg(fw) : fw
+      }
+    }
+  }
+
+  // Hips/spine/chest: drive "up" with forward pole so the avatar can turn with you.
+  if (hipCenter && shoulderCenter) {
+    const up = vNormalize(vRemapAxis(vSub(shoulderCenter, hipCenter), axis))
+    if (up) {
+      if (torsoForward)
+        out.hips = { dir: up, pole: stabilizePole('hips', torsoForward) }
+      else
+        out.hips = { dir: up }
+
+      if (torsoForward) {
+        out.spine = { dir: up, pole: stabilizePole('spine', torsoForward) }
+        out.chest = { dir: up, pole: stabilizePole('chest', torsoForward) }
+      }
+      else {
+        out.spine = { dir: up }
+        out.chest = { dir: up }
+      }
+    }
+  }
+
+  // Shoulder (clavicle-ish): shoulder center -> shoulder
+  if (shoulderCenter && leftShoulder) {
+    const d = vNormalize(vRemapAxis(vSub(leftShoulder, shoulderCenter), axis))
+    if (d)
+      out.leftShoulder = { dir: d }
+  }
+  if (shoulderCenter && rightShoulder) {
+    const d = vNormalize(vRemapAxis(vSub(rightShoulder, shoulderCenter), axis))
+    if (d)
+      out.rightShoulder = { dir: d }
+  }
+
+  // Arms (with pole from elbow bend plane)
+  if (leftShoulder && leftElbow) {
+    const upper = vNormalize(vRemapAxis(vSub(leftElbow, leftShoulder), axis))
+    if (upper) {
+      const poleRaw = leftWrist ? vCross(vSub(leftElbow, leftShoulder), vSub(leftWrist, leftElbow)) : null
+      const pole = poleRaw ? vNormalize(vRemapAxis(poleRaw, axis)) : null
+      out.leftUpperArm = pole ? { dir: upper, pole } : { dir: upper }
+    }
+  }
+  if (leftElbow && leftWrist) {
+    const lower = vNormalize(vRemapAxis(vSub(leftWrist, leftElbow), axis))
+    if (lower)
+      out.leftLowerArm = { dir: lower }
+  }
+  if (rightShoulder && rightElbow) {
+    const upper = vNormalize(vRemapAxis(vSub(rightElbow, rightShoulder), axis))
+    if (upper) {
+      const poleRaw = rightWrist ? vCross(vSub(rightElbow, rightShoulder), vSub(rightWrist, rightElbow)) : null
+      const pole = poleRaw ? vNormalize(vRemapAxis(poleRaw, axis)) : null
+      out.rightUpperArm = pole ? { dir: upper, pole } : { dir: upper }
+    }
+  }
+  if (rightElbow && rightWrist) {
+    const lower = vNormalize(vRemapAxis(vSub(rightWrist, rightElbow), axis))
+    if (lower)
+      out.rightLowerArm = { dir: lower }
+  }
+
+  // Legs: require ankle to reduce hallucinated flips when lower body is off-screen
+  if (leftHip && leftKnee && leftAnkle) {
+    const upper = vNormalize(vRemapAxis(vSub(leftKnee, leftHip), axis))
+    if (upper) {
+      const poleRaw = vCross(vSub(leftKnee, leftHip), vSub(leftAnkle, leftKnee))
+      let pole = vNormalize(vRemapAxis(poleRaw, axis))
+      if (pole)
+        pole = stabilizePole('leftUpperLeg', pole)
+      if (pole)
+        pole = safePole(upper, pole)
+      out.leftUpperLeg = pole ? { dir: upper, pole } : { dir: upper }
+    }
+
+    const lower = vNormalize(vRemapAxis(vSub(leftAnkle, leftKnee), axis))
+    if (lower)
+      out.leftLowerLeg = { dir: lower }
+  }
+
+  if (rightHip && rightKnee && rightAnkle) {
+    const upper = vNormalize(vRemapAxis(vSub(rightKnee, rightHip), axis))
+    if (upper) {
+      const poleRaw = vCross(vSub(rightKnee, rightHip), vSub(rightAnkle, rightKnee))
+      let pole = vNormalize(vRemapAxis(poleRaw, axis))
+      if (pole)
+        pole = stabilizePole('rightUpperLeg', pole)
+      if (pole)
+        pole = safePole(upper, pole)
+      out.rightUpperLeg = pole ? { dir: upper, pole } : { dir: upper }
+    }
+
+    const lower = vNormalize(vRemapAxis(vSub(rightAnkle, rightKnee), axis))
+    if (lower)
+      out.rightLowerLeg = { dir: lower }
+  }
+
+  return out
+}
+
+export function poseToVrmDirections(pose: PoseState, options?: PoseToVrmOptions): VrmPoseDirections {
+  const targets = poseToVrmTargets(pose, options)
+  const out: VrmPoseDirections = {}
+  ;(Object.keys(targets) as (keyof VrmPoseDirections)[]).forEach((k) => {
+    const t = targets[k]
+    if (t)
+      out[k] = t.dir
+  })
+  return out
+}
@@ -0,0 +1,74 @@
+import type { FilesetResolver, Landmark, NormalizedLandmark } from '@mediapipe/tasks-vision'
+
+export type VisionTaskModule = typeof import('@mediapipe/tasks-vision')
+
+// Indirect export from @mediapipe/tasks-vision
+export type VisionTaskWasmFileset = Awaited<ReturnType<typeof FilesetResolver.forVisionTasks>>
+
+export type Landmark2D = NormalizedLandmark
+export type Landmark3D = Landmark
+
+export interface PoseState {
+  landmarks2d?: Landmark2D[]
+  worldLandmarks?: Landmark3D[]
+}
+
+export interface HandState {
+  handedness: 'Left' | 'Right'
+  landmarks2d: Landmark2D[]
+  score?: number
+}
+
+export interface FaceState {
+  hasFace?: boolean
+  landmarks2d?: Landmark2D[]
+}
+
+export interface PerceptionQuality {
+  fps: number
+  latencyMs?: number
+  droppedFrames?: number
+  backend: 'mediapipe'
+  mode: 'split-tasks'
+}
+
+export interface PerceptionPartial {
+  pose?: PoseState
+  hands?: HandState[]
+  face?: FaceState
+}
+
+export interface PerceptionState extends PerceptionPartial {
+  t: number
+  quality: PerceptionQuality
+}
+
+export type MocapJob = 'pose' | 'hands' | 'face'
+
+export interface MocapConfig {
+  enabled: Record<MocapJob, boolean>
+  hz: Record<MocapJob, number>
+  maxPeople: 1
+}
+
+export interface FrameSource {
+  getFrame: () => TexImageSource
+}
+
+export interface MocapBackend {
+  init: (config: MocapConfig) => Promise<void>
+  isBusy: () => boolean
+  run: (frame: TexImageSource, jobs: MocapJob[], nowMs: number) => Promise<PerceptionPartial>
+}
+
+export interface MocapEngine {
+  init: () => Promise<void>
+  start: (
+    source: FrameSource,
+    onState: (state: PerceptionState) => void,
+    options?: { onError?: (error: unknown) => void },
+  ) => void
+  stop: () => void
+  updateConfig: (config: MocapConfig) => void
+  resetState: () => void
+}
@@ -0,0 +1,93 @@
+import type { FaceState, HandState, Landmark2D, PerceptionState, PoseState } from '../types'
+
+import { DrawingUtils, HandLandmarker, PoseLandmarker } from '@mediapipe/tasks-vision'
+
+const POSE_CONNECTIONS: Readonly<{ start: number, end: number }[]> = PoseLandmarker.POSE_CONNECTIONS
+const HAND_CONNECTIONS: Readonly<{ start: number, end: number }[]> = HandLandmarker.HAND_CONNECTIONS
+
+// NOTICE: Palette inspired by https://github.com/proj-airi/webai-examples (see review link in PR).
+const OVERLAY_PALETTE = [
+  { point: 'rgba(80, 200, 255, 0.95)', connector: 'rgba(80, 200, 255, 0.55)' },
+  { point: 'rgba(120, 255, 140, 0.95)', connector: 'rgba(120, 255, 140, 0.55)' },
+  { point: 'rgba(255, 180, 80, 0.95)', connector: 'rgba(255, 180, 80, 0.55)' },
+  { point: 'rgba(180, 120, 255, 0.55)', connector: 'rgba(180, 120, 255, 0.55)' },
+]
+
+// NOTICE: Tuned for devtools readability; see https://ai.google.dev/edge/api/mediapipe/js/tasks-vision.drawingutils.
+const OVERLAY_STYLES = {
+  connectorLineWidth: 2.5,
+  pointRadius: 6,
+  facePointRadius: 2,
+  pointLineWidth: 1.5,
+}
+
+const paletteFor = (index: number) => OVERLAY_PALETTE[index % OVERLAY_PALETTE.length]
+
+function drawFace(drawing: DrawingUtils, face: FaceState) {
+  if (!face.landmarks2d?.length)
+    return
+
+  // Face has 468 points; keep them small to reduce clutter.
+  const colors = paletteFor(3)
+  drawing.drawLandmarks(face.landmarks2d as Landmark2D[], {
+    color: colors.point,
+    radius: OVERLAY_STYLES.facePointRadius,
+    lineWidth: OVERLAY_STYLES.pointLineWidth,
+  })
+}
+
+function drawPose(drawing: DrawingUtils, pose: PoseState) {
+  if (!pose.landmarks2d?.length)
+    return
+
+  const colors = paletteFor(0)
+  drawing.drawConnectors(pose.landmarks2d as Landmark2D[], POSE_CONNECTIONS.map(({ start, end }) => ({ start, end })), {
+    color: colors.connector,
+    lineWidth: OVERLAY_STYLES.connectorLineWidth,
+  })
+  drawing.drawLandmarks(pose.landmarks2d as Landmark2D[], {
+    color: colors.point,
+    radius: OVERLAY_STYLES.pointRadius,
+    lineWidth: OVERLAY_STYLES.pointLineWidth,
+  })
+}
+
+function drawHands(drawing: DrawingUtils, hands: HandState[]) {
+  hands.forEach((hand) => {
+    const handIndex = hand.handedness === 'Left' ? 1 : 2
+    const colors = paletteFor(handIndex)
+    drawing.drawConnectors(hand.landmarks2d as Landmark2D[], HAND_CONNECTIONS.map(({ start, end }) => ({ start, end })), {
+      color: colors.connector,
+      lineWidth: OVERLAY_STYLES.connectorLineWidth,
+    })
+    drawing.drawLandmarks(hand.landmarks2d as Landmark2D[], {
+      color: colors.point,
+      radius: OVERLAY_STYLES.pointRadius,
+      lineWidth: OVERLAY_STYLES.pointLineWidth,
+    })
+  })
+}
+
+export function drawOverlay(
+  ctx: CanvasRenderingContext2D,
+  state: PerceptionState,
+  enabled?: Partial<Record<'pose' | 'hands' | 'face', boolean>>,
+) {
+  const { width, height } = ctx.canvas
+  ctx.clearRect(0, 0, width, height)
+
+  const drawing = new DrawingUtils(ctx)
+
+  const showPose = enabled?.pose ?? true
+  const showHands = enabled?.hands ?? true
+  const showFace = enabled?.face ?? true
+
+  if (state.face && showFace)
+    drawFace(drawing, state.face)
+
+  if (state.pose && showPose)
+    drawPose(drawing, state.pose)
+
+  if (state.hands?.length && showHands)
+    drawHands(drawing, state.hands)
+}
@@ -0,0 +1,58 @@
+/* eslint-disable antfu/no-top-level-await */
+/* eslint-disable no-console */
+
+import type { VisionTaskAssets } from './tasks'
+
+import fs from 'node:fs/promises'
+
+import { Buffer } from 'node:buffer'
+import { fileURLToPath } from 'node:url'
+
+import { ofetch } from 'ofetch'
+
+import { visionTaskAssets } from './tasks'
+
+const taskSources: Record<keyof VisionTaskAssets, string> = {
+  pose: 'https://storage.googleapis.com/mediapipe-models/pose_landmarker/pose_landmarker_lite/float16/1/pose_landmarker_lite.task',
+  hands: 'https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/1/hand_landmarker.task',
+  face: 'https://storage.googleapis.com/mediapipe-models/face_landmarker/face_landmarker/float16/1/face_landmarker.task',
+}
+
+await fs.mkdir(fileURLToPath(new URL('./assets', import.meta.url)), { recursive: true })
+
+await Promise.all(Object.entries(taskSources).map(
+  async ([key, url]) => {
+    console.log(`Downloading MediaPipe vision task asset for ${key} from ${url}...`)
+    const res = await ofetch(url, { responseType: 'arrayBuffer' })
+    const outputPath = fileURLToPath(visionTaskAssets[key as keyof VisionTaskAssets])
+    await fs.writeFile(outputPath, Buffer.from(res))
+    console.log(`MediaPipe vision task asset for ${key} saved to ${outputPath}`)
+  },
+))
+
+const wasmSourceDir = fileURLToPath(new URL('../node_modules/@mediapipe/tasks-vision/wasm', import.meta.url))
+const wasmOutputDir = fileURLToPath(new URL('./assets/wasm', import.meta.url))
+await fs.mkdir(wasmOutputDir, { recursive: true })
+await fs.cp(wasmSourceDir, wasmOutputDir, { recursive: true, force: true })
+
+await Promise.all(Object.entries(visionTaskAssets).map(
+  async ([key, url]) => {
+    const path = fileURLToPath(url)
+    try {
+      await fs.access(path, fs.constants.R_OK)
+    }
+    catch (err) {
+      throw new Error(`Failed to ensure MediaPipe vision task asset for ${key}: ${err}`)
+    }
+    const stat = await fs.stat(path)
+    if (!stat.isFile()) {
+      throw new Error(`Failed to ensure MediaPipe vision task asset for ${key}: not a file: ${path}`)
+    }
+  },
+))
+
+const wasmEntries = await fs.readdir(wasmOutputDir)
+if (!wasmEntries.length)
+  throw new Error(`Failed to ensure MediaPipe WASM assets: ${wasmOutputDir} is empty`)
+
+console.log('All MediaPipe vision task assets are prepared.')
@@ -0,0 +1,13 @@
+export interface VisionTaskAssets {
+  pose: string
+  hands: string
+  face: string
+}
+
+export const visionTaskAssets: VisionTaskAssets = {
+  pose: new URL('./assets/pose_landmarker_lite.task', import.meta.url).href,
+  hands: new URL('./assets/hand_landmarker.task', import.meta.url).href,
+  face: new URL('./assets/face_landmarker.task', import.meta.url).href,
+}
+
+export const visionTaskWasmRoot = new URL('./assets/wasm', import.meta.url).href
@@ -0,0 +1,30 @@
+{
+  "compilerOptions": {
+    "target": "ESNext",
+    "lib": [
+      "DOM",
+      "ESNext"
+    ],
+    "module": "ESNext",
+    "moduleResolution": "Bundler",
+    "resolveJsonModule": true,
+    "strict": true,
+    "strictNullChecks": true,
+    "noUnusedLocals": true,
+    "noEmit": true,
+    "esModuleInterop": true,
+    "forceConsistentCasingInFileNames": true,
+    "isolatedModules": true,
+    "verbatimModuleSyntax": true,
+    "skipLibCheck": true
+  },
+  "include": [
+    "src/**/*.ts",
+    "src/**/*.d.ts",
+    "tasks/**/*.ts"
+  ],
+  "exclude": [
+    "dist",
+    "node_modules"
+  ]
+}
@@ -159,6 +159,9 @@ let stopCameraWatch: WatchStopHandle | undefined
 // Animation related ref
 const vrmAnimationMixer = ref<AnimationMixer>()
 const { onBeforeRender, stop, start } = useLoop()
+
+type VrmFrameHook = (vrm: VRM, delta: number) => void
+const vrmFrameHook = shallowRef<VrmFrameHook>()
 let disposeBeforeRenderLoop: (() => void | undefined)

 // Expressions
@@ -417,13 +420,23 @@ async function loadModel() {
      // Clean up & animation setting
      disposeBeforeRenderLoop = onBeforeRender(({ delta }) => {
        vrmAnimationMixer.value?.update(delta)
-        vrm.value?.update(delta)
-        vrm.value?.lookAt?.update?.(delta)
-        blink.update(vrm.value, delta)
-        idleEyeSaccades.update(vrm.value, lookAtTarget, delta)
+        const activeVrm = vrm.value
+        if (activeVrm && vrmFrameHook.value) {
+          try {
+            vrmFrameHook.value(activeVrm, delta)
+          }
+          catch (err) {
+            console.error(err)
+            emit('error', err)
+          }
+        }
+        activeVrm?.humanoid.update()
+        activeVrm?.lookAt?.update?.(delta)
+        blink.update(activeVrm, delta)
+        idleEyeSaccades.update(activeVrm, lookAtTarget, delta)
        vrmEmote.value?.update(delta)
-        vrmLipSync.update(vrm.value, delta)
-        vrm.value?.springBoneManager?.update(delta)
+        vrmLipSync.update(activeVrm, delta)
+        activeVrm?.springBoneManager?.update(delta)
      }).off

      // update the 'last model src'
@@ -562,6 +575,9 @@ defineExpose({
  setExpression(expression: string) {
    vrmEmote.value?.setEmotionWithResetAfter(expression, 1000)
  },
+  setVrmFrameHook(hook?: VrmFrameHook) {
+    vrmFrameHook.value = hook
+  },
  scene: computed(() => vrm.value?.scene),
  lookAtUpdate(target: Vec3) {
    idleEyeSaccades.instantUpdate(vrm.value, target)
@@ -7,6 +7,7 @@
  * - Src of model is obtained from stage-ui via props, which is NOT a part of stage-ui-three package
 */

+import type { VRM } from '@pixiv/three-vrm'
 import type { TresContext } from '@tresjs/core'
 import type { DirectionalLight, SphericalHarmonics3, Texture, WebGLRenderTarget } from 'three'

@@ -198,6 +199,12 @@ const effectProps = {
  blendFunction: BlendFunction.SRC,
 }

+const vrmFrameHook = shallowRef<((vrm: VRM, delta: number) => void) | undefined>(undefined)
+function applyVrmFrameHook() {
+  modelRef.value?.setVrmFrameHook(vrmFrameHook.value)
+}
+watch(modelRef, () => applyVrmFrameHook(), { immediate: true })
+
 // === Directional Light ===
 // TODO: wrap <TresDirectionalLight> to integrate all the below code
 const sceneReady = ref(false)
@@ -270,6 +277,10 @@ defineExpose({
  setExpression: (expression: string) => {
    modelRef.value?.setExpression(expression)
  },
+  setVrmFrameHook: (hook?: (vrm: VRM, delta: number) => void) => {
+    vrmFrameHook.value = hook
+    applyVrmFrameHook()
+  },
  canvasElement: () => {
    return tresCanvasRef.value?.renderer.instance.domElement
  },
@@ -72,9 +72,15 @@ catalogs:
    nanoid:
      specifier: ^5.1.6
      version: 5.1.6
+    ofetch:
+      specifier: ^1.5.1
+      version: 1.5.1
    posthog-js:
      specifier: 1.306.1
      version: 1.306.1
+    tsx:
+      specifier: ^4.21.0
+      version: 4.21.0
    uncrypto:
      specifier: ^0.1.3
      version: 0.1.3
@@ -739,6 +745,9 @@ importers:
      '@proj-airi/i18n':
        specifier: workspace:^
        version: link:../../packages/i18n
+      '@proj-airi/model-driver-mediapipe':
+        specifier: workspace:^
+        version: link:../../packages/model-driver-mediapipe
      '@proj-airi/server-sdk':
        specifier: workspace:^
        version: link:../../packages/server-sdk
@@ -1273,6 +1282,31 @@ importers:
        specifier: ^1.3.0
        version: 1.3.0

+  packages/model-driver-mediapipe:
+    dependencies:
+      '@mediapipe/tasks-vision':
+        specifier: ^0.10.0
+        version: 0.10.21
+      '@pixiv/three-vrm':
+        specifier: ^3.4.4
+        version: 3.4.4(three@0.182.0)
+      es-toolkit:
+        specifier: 'catalog:'
+        version: 1.43.0
+      three:
+        specifier: ^0.182.0
+        version: 0.182.0
+    devDependencies:
+      '@types/three':
+        specifier: ^0.182.0
+        version: 0.182.0
+      ofetch:
+        specifier: 'catalog:'
+        version: 1.5.1
+      tsx:
+        specifier: 'catalog:'
+        version: 4.21.0
+
  packages/pipelines-audio:
    dependencies:
      '@moeru/std':
@@ -4972,6 +5006,9 @@ packages:
      markdown-it:
        optional: true

+  '@mediapipe/tasks-vision@0.10.21':
+    resolution: {integrity: sha512-TuhKH+credq4zLksGbYrnvJ1aLIWMc5r0UHwzxzql4BHECJwIAoBR61ZrqwGOW6ZmSBIzU1t4VtKj8hbxFaKeA==}
+
  '@modelcontextprotocol/sdk@1.25.1':
    resolution: {integrity: sha512-yO28oVFFC7EBoiKdAn+VqRm+plcfv4v0xp6osG/VsCB0NlPZWi87ajbCZZ8f/RvOFLEu7//rSRmuZZ7lMoe3gQ==}
    engines: {node: '>=18'}
@@ -17749,6 +17786,8 @@ snapshots:
    optionalDependencies:
      markdown-it: 14.1.0

+  '@mediapipe/tasks-vision@0.10.21': {}
+
  '@modelcontextprotocol/sdk@1.25.1(@cfworker/json-schema@4.1.1)(hono@4.11.3)(zod@4.2.1)':
    dependencies:
      '@hono/node-server': 1.19.7(hono@4.11.3)
@@ -1,7 +1,6 @@
 catalogMode: prefer

 shellEmulator: true
-
 packages:
  - packages/**
  - plugins/**
@@ -20,7 +19,6 @@ overrides:
  safer-buffer: npm:@nolyfill/safer-buffer@^1.0.44
  side-channel: npm:@nolyfill/side-channel@^1.0.44
  string.prototype.matchall: npm:@nolyfill/string.prototype.matchall@^1.0.44
-
 catalog:
  '@guiiai/logg': ^1.2.11
  '@moeru/eslint-config': 0.1.0-beta.14
@@ -44,7 +42,9 @@ catalog:
  es-toolkit: 1.43.0
  injeca: ^0.1.5
  nanoid: ^5.1.6
+  ofetch: ^1.5.1
  posthog-js: 1.306.1
+  tsx: ^4.21.0
  uncrypto: ^0.1.3
  unplugin-info: 1.2.4
  xsschema: ^0.4.0-beta.12