feat(model-driver-lipsync): new package for model driving capabilities

apps/stage-tamagotchi/src/main/windows/widgets/index.ts

@@ -157,7 +157,7 @@ export function setupWidgetsWindowManager(): WidgetsWindowManager {
   }
 
   function toSnapshot(record: WidgetRecord): WidgetSnapshot {
-    const { timer, ...snapshot } = record
+    const { timer: _timer, ...snapshot } = record
     return snapshot
   }
 

cspell.config.yaml

@@ -141,6 +141,7 @@ words:
   - libsamplerate
   - libsodium
   - lightningcss
+  - lipsync
   - listhen
   - live2dcubismcore
   - live2dcubismframework

packages/model-driver-lipsync/package.json

@@ -0,0 +1,15 @@
+{
+  "name": "@proj-airi/model-driver-lipsync",
+  "type": "module",
+  "version": "0.0.0",
+  "private": true,
+  "exports": {
+    ".": "./src/index.ts",
+    "./live2d": "./src/live2d/index.ts",
+    "./shared/wlipsync": "./src/shared/wlipsync/index.ts",
+    "./shared/wlipsync/profile.json": "./src/shared/wlipsync/profile.json"
+  },
+  "dependencies": {
+    "wlipsync": "^1.3.0"
+  }
+}

packages/model-driver-lipsync/src/index.ts

@@ -0,0 +1 @@
+export * from './live2d'

packages/model-driver-lipsync/src/live2d/index.ts

@@ -0,0 +1,105 @@
+import type { Profile } from 'wlipsync'
+
+import { createWLipSyncNode } from 'wlipsync'
+
+const RAW_KEYS = ['A', 'E', 'I', 'O', 'U', 'S'] as const
+const RAW_TO_VOWEL: Record<typeof RAW_KEYS[number], VowelKey> = {
+  A: 'A',
+  E: 'E',
+  I: 'I',
+  O: 'O',
+  U: 'U',
+  // Treat S as silence/closed; map to a small I-like mouth to avoid a hard snap
+  S: 'I',
+}
+
+export type VowelKey = 'A' | 'E' | 'I' | 'O' | 'U'
+
+export interface Live2DLipSync {
+  /**
+   * The underlying wLipSync AudioWorkletNode. Connect your audio source to it.
+   */
+  node: Awaited<ReturnType<typeof createWLipSyncNode>>
+  /**
+   * Get per-vowel weights (already remapped from AEIOUS to AEIOU) scaled by current volume.
+   */
+  getVowelWeights: () => Record<VowelKey, number>
+  /**
+   * Get a single mouth-open value (0-1) derived from the loudest vowel weight.
+   */
+  getMouthOpen: () => number
+  /**
+   * Convenience helper to connect an audio source node to the lip sync node.
+   */
+  connectSource: (source: AudioNode) => void
+}
+
+export interface Live2DLipSyncOptions {
+  /**
+   * Overall cap for each vowel weight after scaling.
+   * Defaults to 0.7 to keep Live2D mouth movement natural.
+   */
+  cap?: number
+  /**
+   * Volume multiplier applied before exponent.
+   * Defaults to 0.9.
+   */
+  volumeScale?: number
+  /**
+   * Exponent for the volume curve to soften peaks.
+   * Defaults to 0.7.
+   */
+  volumeExponent?: number
+}
+
+/**
+ * Create a Live2D-friendly lip sync helper using the wLipSync worklet.
+ * - Compute AEIOUS weights from the worklet
+ * - Remap to AEIOU
+ * - Scale by volume to derive a mouth-open value
+ */
+export async function createLive2DLipSync(
+  audioContext: AudioContext,
+  profile: Profile,
+  options: Live2DLipSyncOptions = {},
+): Promise<Live2DLipSync> {
+  const node = await createWLipSyncNode(audioContext, profile)
+
+  const cap = options.cap ?? 0.7
+  const volumeScale = options.volumeScale ?? 0.9
+  const volumeExponent = options.volumeExponent ?? 0.7
+
+  const getVowelWeights = (): Record<VowelKey, number> => {
+    const projected: Record<VowelKey, number> = { A: 0, E: 0, I: 0, O: 0, U: 0 }
+    const amp = Math.min((node.volume ?? 0) * volumeScale, 1) ** volumeExponent
+
+    for (const raw of RAW_KEYS) {
+      const vowel = RAW_TO_VOWEL[raw]
+      const rawVal = node.weights?.[raw] ?? 0
+      projected[vowel] = Math.max(projected[vowel], Math.min(cap, rawVal * amp))
+    }
+
+    return projected
+  }
+
+  const getMouthOpen = () => {
+    const weights = Object.values(getVowelWeights())
+    return weights.length ? Math.max(...weights) : 0
+  }
+
+  const connectSource = (source: AudioNode) => {
+    try {
+      source.connect(node)
+    }
+    catch (error) {
+      console.error('[model-driver-lipsync] failed to connect source to lip sync node', error)
+    }
+  }
+
+  return {
+    node,
+    getVowelWeights,
+    getMouthOpen,
+    connectSource,
+  }
+}

packages/model-driver-lipsync/src/shared/wlipsync/index.ts

@@ -0,0 +1,2 @@
+export { default as wlipsyncProfile } from './profile.json' assert { type: 'json' }
+export type { Profile } from 'wlipsync'

packages/model-driver-lipsync/tsconfig.json

@@ -0,0 +1,14 @@
+{
+  "compilerOptions": {
+    "target": "ESNext",
+    "lib": ["ESNext", "DOM"],
+    "module": "ESNext",
+    "moduleResolution": "bundler",
+    "esModuleInterop": true,
+    "forceConsistentCasingInFileNames": true,
+    "isolatedModules": true,
+    "verbatimModuleSyntax": true,
+    "skipLibCheck": true
+  },
+  "include": ["src/**/*.ts"]
+}

packages/stage-ui/package.json

@@ -73,6 +73,7 @@
     "@proj-airi/font-cjkfonts-allseto": "workspace:^",
     "@proj-airi/font-xiaolai": "workspace:^",
     "@proj-airi/i18n": "workspace:^",
+    "@proj-airi/model-driver-lipsync": "workspace:^",
     "@proj-airi/server-sdk": "workspace:^",
     "@proj-airi/stage-shared": "workspace:^",
     "@proj-airi/stage-ui-three": "workspace:^",

packages/stage-ui/src/components/scenes/Stage.vue

@@ -1,5 +1,7 @@
 <script setup lang="ts">
 import type { DuckDBWasmDrizzleDatabase } from '@proj-airi/drizzle-duckdb-wasm'
+import type { Live2DLipSync } from '@proj-airi/model-driver-lipsync'
+import type { Profile } from '@proj-airi/model-driver-lipsync/shared/wlipsync'
 import type { SpeechProviderWithExtraOptions } from '@xsai-ext/shared-providers'
 import type { UnElevenLabsOptions } from 'unspeech'
 
@@ -9,6 +11,8 @@ import type { TTSChunkItem } from '../../utils/tts'
 
 import { drizzle } from '@proj-airi/drizzle-duckdb-wasm'
 import { getImportUrlBundles } from '@proj-airi/drizzle-duckdb-wasm/bundles/import-url-browser'
+import { createLive2DLipSync } from '@proj-airi/model-driver-lipsync'
+import { wlipsyncProfile } from '@proj-airi/model-driver-lipsync/shared/wlipsync'
 import { ThreeScene, useModelStore } from '@proj-airi/stage-ui-three'
 import { animations } from '@proj-airi/stage-ui-three/assets/vrm'
 import { useBroadcastChannel } from '@vueuse/core'
@@ -67,13 +71,13 @@ const { textSegmentationQueue } = storeToRefs(textSegmentationStore)
 clearTextSegmentationHooks()
 
 const characterSpeechPlaybackQueue = usePipelineCharacterSpeechPlaybackQueueStore()
-const { connectAudioContext, connectAudioAnalyser, clearAll, onPlaybackStarted, onPlaybackFinished } = characterSpeechPlaybackQueue
+const { connectAudioContext, connectAudioAnalyser, connectLipSyncNode, clearAll, onPlaybackStarted, onPlaybackFinished } = characterSpeechPlaybackQueue
 const { currentAudioSource, playbackQueue } = storeToRefs(characterSpeechPlaybackQueue)
 
 const settingsStore = useSettings()
 const { stageModelRenderer, stageViewControlsEnabled, live2dDisableFocus, stageModelSelectedUrl, stageModelSelected } = storeToRefs(settingsStore)
 const { mouthOpenSize } = storeToRefs(useSpeakingStore())
-const { audioContext, calculateVolume } = useAudioContext()
+const { audioContext } = useAudioContext()
 connectAudioContext(audioContext)
 
 const { onBeforeMessageComposed, onBeforeSend, onTokenLiteral, onTokenSpecial, onStreamEnd, onAssistantResponseEnd, clearHooks } = useChatStore()
@@ -121,6 +125,8 @@ vrmStore.onShouldUpdateView(async () => {
 const audioAnalyser = ref<AnalyserNode>()
 const nowSpeaking = ref(false)
 const lipSyncStarted = ref(false)
+const lipSyncLoopId = ref<number>()
+const live2dLipSync = ref<Live2DLipSync>()
 
 const speechStore = useSpeechStore()
 const { ssmlEnabled, activeSpeechProvider, activeSpeechModel, activeSpeechVoice, pitch } = storeToRefs(speechStore)
@@ -224,20 +230,39 @@ onTextSegmented((chunkItem) => {
   ttsQueue.enqueue(chunkItem)
 })
 
-function getVolumeWithMinMaxNormalizeWithFrameUpdates() {
-  requestAnimationFrame(getVolumeWithMinMaxNormalizeWithFrameUpdates)
-  if (!nowSpeaking.value)
+function startLipSyncLoop() {
+  if (lipSyncLoopId.value)
     return
 
-  mouthOpenSize.value = calculateVolume(audioAnalyser.value!, 'linear')
+  const tick = () => {
+    if (!nowSpeaking.value || !live2dLipSync.value) {
+      mouthOpenSize.value = 0
+    }
+    else {
+      mouthOpenSize.value = live2dLipSync.value.getMouthOpen()
+    }
+    lipSyncLoopId.value = requestAnimationFrame(tick)
+  }
+
+  lipSyncLoopId.value = requestAnimationFrame(tick)
 }
 
-function setupLipSync() {
-  if (!lipSyncStarted.value) {
-    getVolumeWithMinMaxNormalizeWithFrameUpdates()
-    audioContext.resume()
+async function setupLipSync() {
+  if (lipSyncStarted.value)
+    return
+
+  try {
+    const lipSync = await createLive2DLipSync(audioContext, wlipsyncProfile as Profile)
+    live2dLipSync.value = lipSync
+    connectLipSyncNode(lipSync.node)
+    await audioContext.resume()
+    startLipSyncLoop()
     lipSyncStarted.value = true
   }
+  catch (error) {
+    lipSyncStarted.value = false
+    console.error('Failed to setup Live2D lip sync', error)
+  }
 }
 
 function setupAnalyser() {
@@ -250,7 +275,7 @@ function setupAnalyser() {
 onBeforeMessageComposed(async () => {
   clearAll()
   setupAnalyser()
-  setupLipSync()
+  await setupLipSync()
   // Reset assistant caption for a new message
   assistantCaption.value = ''
   postCaption({ type: 'caption-assistant', text: '' })
@@ -303,11 +328,24 @@ function canvasElement() {
     return vrmViewerRef.value?.canvasElement()
 }
 
+onUnmounted(() => {
+  if (lipSyncLoopId.value) {
+    cancelAnimationFrame(lipSyncLoopId.value)
+    lipSyncLoopId.value = undefined
+  }
+})
+
 defineExpose({
   canvasElement,
 })
 
+onPlaybackFinished(() => {
+  nowSpeaking.value = false
+  mouthOpenSize.value = 0
+})
+
 onPlaybackStarted(({ text }) => {
+  nowSpeaking.value = true
   // NOTICE: currently, postCaption, postPresent from useBroadcastChannel may throw error
   // once we navigate away from the page that created the BroadcastChannel,
   // as the channel gets closed on unmount, leading to "Failed to execute 'postMessage' on 'BroadcastChannel': The channel is closed."

packages/stage-ui/src/composables/queues.ts

@@ -126,6 +126,7 @@ export const usePipelineCharacterSpeechPlaybackQueueStore = defineStore('pipelin
 
   const audioContext = shallowRef<AudioContext>()
   const audioAnalyser = shallowRef<AnalyserNode>()
+  const lipSyncNode = shallowRef<AudioNode>()
 
   function connectAudioContext(context: AudioContext) {
     audioContext.value = context
@@ -135,6 +136,10 @@ export const usePipelineCharacterSpeechPlaybackQueueStore = defineStore('pipelin
     audioAnalyser.value = analyser
   }
 
+  function connectLipSyncNode(node: AudioNode) {
+    lipSyncNode.value = node
+  }
+
   function clearPlaying() {
     if (currentAudioSource) {
       try {
@@ -172,6 +177,9 @@ export const usePipelineCharacterSpeechPlaybackQueueStore = defineStore('pipelin
             source.connect(audioContext.value.destination)
             // Connect the source to the analyzer
             source.connect(audioAnalyser.value!)
+            // Connect to lip sync tap if provided
+            if (lipSyncNode.value)
+              source.connect(lipSyncNode.value)
 
             // Start playing the audio
             for (const hook of onPlaybackStartedHooks.value) {
@@ -192,15 +200,13 @@ export const usePipelineCharacterSpeechPlaybackQueueStore = defineStore('pipelin
             currentAudioSource.value = source
             source.start(0)
             source.onended = () => {
-              // Play special token: delay or emotion
-              if (ctx.data.special) {
-                for (const hook of onPlaybackFinishedHooks.value) {
-                  try {
-                    hook({ special: ctx.data.special })
-                  }
-                  catch (err) {
-                    console.error('Error in onPlaybackFinished hook:', err)
-                  }
+              // Notify hooks regardless; consumers can decide how to use the special token (if any).
+              for (const hook of onPlaybackFinishedHooks.value) {
+                try {
+                  hook({ special: ctx.data.special ?? '' })
+                }
+                catch (err) {
+                  console.error('Error in onPlaybackFinished hook:', err)
                 }
               }
 
@@ -230,6 +236,7 @@ export const usePipelineCharacterSpeechPlaybackQueueStore = defineStore('pipelin
 
     connectAudioContext,
     connectAudioAnalyser,
+    connectLipSyncNode,
     clearPlaying,
     clearQueue,
     clearAll,

packages/stage-ui/src/stores/providers/aliyun/stream-transcription.ts

@@ -80,7 +80,7 @@ function eventListenerOf(type: string, listener: EventListenerOrEventListenerObj
   }
 }
 
-async function startRealtimeSession(options: InternalRealtimeOptions): Promise<void> {
+async function startRealtimeSession(options: InternalRealtimeOptions): Promise<AliyunStreamTranscriptionHandle> {
   const {
     accessKeyId,
     accessKeySecret,
@@ -103,11 +103,13 @@ async function startRealtimeSession(options: InternalRealtimeOptions): Promise<v
   const websocket = new WebSocket(url)
   websocket.binaryType = 'arraybuffer'
 
-  const abortHandler = eventListenerOf('abort', () => cleanup(abortSignal?.reason ?? new DOMException('Aborted', 'AbortError')), abortSignal)
-  abortSignal && abortHandler.on()
+  const abortHandler = abortSignal
+    ? eventListenerOf('abort', () => cleanup(abortSignal?.reason ?? new DOMException('Aborted', 'AbortError')), abortSignal)
+    : undefined
+  abortHandler?.on()
 
   async function cleanup(error?: unknown) {
-    abortHandler && abortSignal && abortHandler.off()
+    abortHandler?.off()
     mayThrow(async () => await reader.cancel())
 
     if (websocket) {
@@ -123,6 +125,10 @@ async function startRealtimeSession(options: InternalRealtimeOptions): Promise<v
     await onSessionTerminated?.(error)
   }
 
+  const handle: AliyunStreamTranscriptionHandle = {
+    close: async () => await cleanup(new DOMException('Closed', 'AbortError')),
+  }
+
   async function onTranscriptionStarted() {
     try {
       while (true) {
@@ -188,6 +194,8 @@ async function startRealtimeSession(options: InternalRealtimeOptions): Promise<v
 
   if (abortSignal?.aborted)
     throw abortSignal.reason ?? new DOMException('Aborted', 'AbortError')
+
+  return handle
 }
 
 export function createAliyunNLSProvider(
@@ -240,6 +248,16 @@ export function createAliyunNLSProvider(
                     controller.enqueue(encodeSSE({ delta: text, type: 'transcript.text.delta' }))
                   controller.enqueue(encodeSSE({ delta: '', type: 'transcript.text.done' }))
                 },
+              }).then((handle) => {
+                sessionHandle = handle
+              }).catch(async (error) => {
+                controllerClosed = true
+                try {
+                  await extraOptions?.onSessionTerminated?.(error)
+                }
+                finally {
+                  controller.error(error instanceof Error ? error : new Error(String(error)))
+                }
               })
             },
             cancel: async () => {

plugins/airi-plugin-vscode/scripts/publish.ts

@@ -34,7 +34,11 @@ async function publish() {
     {
       // eslint-disable-next-line no-console
       console.log('\nPublish to VSCE...\n')
-      const execPublish = exec('pnpx', ['@vscode/vsce', 'publish', '--no-dependencies', '-p', process.env.VSCE_TOKEN!, ...[(isPreview ? '--pre-release' : '')]], { nodeOptions: { cwd: root } })
+      const vsceArgs = ['@vscode/vsce', 'publish', '--no-dependencies', '-p', process.env.VSCE_TOKEN!]
+      if (isPreview)
+        vsceArgs.push('--pre-release')
+
+      const execPublish = exec('pnpx', vsceArgs, { nodeOptions: { cwd: root } })
       for await (const line of execPublish) {
       // eslint-disable-next-line no-console
         console.log(line)
@@ -43,7 +47,11 @@ async function publish() {
     {
       // eslint-disable-next-line no-console
       console.log('\nPublish to OVSE...\n')
-      const execPublish = exec('pnpx', ['ovsx', 'publish', '--no-dependencies', '-p', process.env.OVSX_TOKEN!, ...[(isPreview ? '--pre-release' : '')]], { nodeOptions: { cwd: root } })
+      const ovseArgs = ['ovsx', 'publish', '--no-dependencies', '-p', process.env.OVSX_TOKEN!]
+      if (isPreview)
+        ovseArgs.push('--pre-release')
+
+      const execPublish = exec('pnpx', ovseArgs, { nodeOptions: { cwd: root } })
       for await (const line of execPublish) {
       // eslint-disable-next-line no-console
         console.log(line)

pnpm-lock.yaml

@@ -1183,6 +1183,12 @@ importers:
         specifier: ^7.2.7
         version: 7.2.7(@types/node@24.10.4)(jiti@2.6.1)(less@4.4.2)(lightningcss@1.30.2)(terser@5.44.1)(tsx@4.21.0)(yaml@2.8.2)
 
+  packages/model-driver-lipsync:
+    dependencies:
+      wlipsync:
+        specifier: ^1.3.0
+        version: 1.3.0
+
   packages/ccc:
     dependencies:
       meta-png:
@@ -1439,6 +1445,9 @@ importers:
       '@proj-airi/audio':
         specifier: workspace:^
         version: link:../audio
+      '@proj-airi/model-driver-lipsync':
+        specifier: workspace:^
+        version: link:../model-driver-lipsync
       '@proj-airi/ccc':
         specifier: workspace:^
         version: link:../ccc