From 66d7ef207e8affa14f338ed89b170e989e6b31d6 Mon Sep 17 00:00:00 2001 From: Neko Ayaka Date: Sun, 9 Aug 2026 18:22:29 +0800 Subject: [PATCH] refactor(stage-web,stage-pages,stage-ui): better structure for asr & vad --- apps/stage-web/src/pages/index.vue | 40 +- ...ders-transcription-realtime-aliyun-nls.vue | 9 +- .../aliyun-nls-transcription.vue | 2 +- .../transcription/browser-web-speech-api.vue | 2 +- packages/stage-shared/src/perf/io-trace.ts | 4 + packages/stage-ui/package.json | 4 +- .../libs/audio/vad-streaming-session.test.ts | 50 ++ .../src/libs/audio/vad-streaming-session.ts | 107 ++++ packages/stage-ui/src/libs/audio/vad.ts | 2 + .../providers/providers/aliyun-nls/index.ts | 8 +- .../providers/aliyun-nls/provider.ts} | 227 ++------- .../providers/aliyun-nls/session.ts} | 0 .../providers/aliyun-nls}/token.test.ts | 0 .../providers/providers/aliyun-nls}/token.ts | 0 .../providers/providers/aliyun-nls}/utils.ts | 0 .../providers/browser-web-speech-api/index.ts | 5 +- .../browser-web-speech-api/provider.ts} | 0 .../providers/providers/elevenlabs/index.ts | 2 +- .../providers/elevenlabs/list-models.ts | 0 .../stream-transcription/index.test.ts | 34 ++ .../providers/stream-transcription/index.ts | 153 ++++++ .../src/libs/providers/validators/run.test.ts | 46 ++ .../src/libs/providers/validators/run.ts | 39 +- .../stage-ui/src/stores/ai/models/vad.test.ts | 93 +++- packages/stage-ui/src/stores/ai/models/vad.ts | 27 +- .../stage-ui/src/stores/modules/hearing.ts | 464 ++++++++++-------- packages/stage-ui/src/workers/vad/vad.ts | 21 + .../audio-transcription-stream/session.ts | 2 +- 28 files changed, 913 insertions(+), 428 deletions(-) create mode 100644 packages/stage-ui/src/libs/audio/vad-streaming-session.test.ts create mode 100644 packages/stage-ui/src/libs/audio/vad-streaming-session.ts rename packages/stage-ui/src/{stores/providers/aliyun/stream-transcription.ts => libs/providers/providers/aliyun-nls/provider.ts} (62%) rename packages/stage-ui/src/{stores/providers/aliyun/index.ts => libs/providers/providers/aliyun-nls/session.ts} (100%) rename packages/stage-ui/src/{stores/providers/aliyun => libs/providers/providers/aliyun-nls}/token.test.ts (100%) rename packages/stage-ui/src/{stores/providers/aliyun => libs/providers/providers/aliyun-nls}/token.ts (100%) rename packages/stage-ui/src/{stores/providers/aliyun => libs/providers/providers/aliyun-nls}/utils.ts (100%) rename packages/stage-ui/src/{stores/providers/web-speech-api/index.ts => libs/providers/providers/browser-web-speech-api/provider.ts} (100%) rename packages/stage-ui/src/{stores => libs/providers}/providers/elevenlabs/list-models.ts (100%) create mode 100644 packages/stage-ui/src/libs/providers/stream-transcription/index.test.ts create mode 100644 packages/stage-ui/src/libs/providers/stream-transcription/index.ts create mode 100644 packages/stage-ui/src/libs/providers/validators/run.test.ts diff --git a/apps/stage-web/src/pages/index.vue b/apps/stage-web/src/pages/index.vue index 9668fbe02..1fa2221fc 100644 --- a/apps/stage-web/src/pages/index.vue +++ b/apps/stage-web/src/pages/index.vue @@ -45,7 +45,7 @@ const settingsAudioDeviceStore = useSettingsAudioDevice() const { stream, enabled } = storeToRefs(settingsAudioDeviceStore) const { startRecord, stopRecord, onStopRecord } = useAudioRecorder(stream) const hearingPipeline = useHearingSpeechInputPipeline() -const { transcribeForRecording } = hearingPipeline +const { stopStreamingTranscription, transcribeForMediaStream, transcribeForRecording } = hearingPipeline const { supportsStreamInput } = storeToRefs(hearingPipeline) const providersStore = useProviderStore() const consciousnessStore = useConsciousnessStore() @@ -67,28 +67,39 @@ const { let stopOnStopRecord: (() => void) | undefined +async function sendVoiceInputTextToChat(text: string | undefined) { + if (!text?.trim()) + return + + try { + const provider = await providersStore.getProviderInstance(activeChatProvider.value) + if (!provider || !activeChatModel.value) + return + + await chatStore.ingest(text, { model: activeChatModel.value, chatProvider: provider as ChatProvider }) + } + catch (error) { + console.error('Failed to send chat from voice:', error) + } +} + async function startAudioInteraction() { try { await initVAD() if (stream.value) await startVAD(stream.value) + if (shouldUseStreamInput.value && stream.value) { + await transcribeForMediaStream(stream.value, { + onSentenceEnd: text => void sendVoiceInputTextToChat(text), + }) + return + } + // Hook once stopOnStopRecord = onStopRecord(async (recording) => { const text = await transcribeForRecording(recording) - if (!text || !text.trim()) - return - - try { - const provider = await providersStore.getProviderInstance(activeChatProvider.value) - if (!provider || !activeChatModel.value) - return - - await chatStore.ingest(text, { model: activeChatModel.value, chatProvider: provider as ChatProvider }) - } - catch (err) { - console.error('Failed to send chat from voice:', err) - } + await sendVoiceInputTextToChat(text) }) } catch (e) { @@ -119,6 +130,7 @@ function stopAudioInteraction() { try { stopOnStopRecord?.() stopOnStopRecord = undefined + void stopStreamingTranscription(true) disposeVAD() } catch {} diff --git a/packages/stage-pages/src/pages/devtools/providers-transcription-realtime-aliyun-nls.vue b/packages/stage-pages/src/pages/devtools/providers-transcription-realtime-aliyun-nls.vue index bc19e0255..b7c159c71 100644 --- a/packages/stage-pages/src/pages/devtools/providers-transcription-realtime-aliyun-nls.vue +++ b/packages/stage-pages/src/pages/devtools/providers-transcription-realtime-aliyun-nls.vue @@ -1,10 +1,11 @@