From 01eac03af1190096091104cfaf69fe94d634300a Mon Sep 17 00:00:00 2001 From: Neko Date: Wed, 12 Aug 2026 04:16:53 +0800 Subject: [PATCH] fix(stage-ui): correct VAD segment processing (#2258) --- apps/stage-pocket/src/pages/index.vue | 8 +- apps/stage-web/src/pages/index.vue | 8 +- .../src/pages/settings/modules/hearing.vue | 6 +- .../composables/audio/audio-recorder.test.ts | 26 ++++ .../src/composables/audio/audio-recorder.ts | 24 +++- .../audio/voice-input-session.test.ts | 25 ++++ .../composables/audio/voice-input-session.ts | 9 ++ .../libs/audio/vad-streaming-session.test.ts | 16 +++ packages/stage-ui/src/libs/audio/vad.ts | 2 + .../stage-ui/src/stores/ai/models/vad.test.ts | 14 +++ packages/stage-ui/src/stores/ai/models/vad.ts | 7 ++ .../stage-ui/src/stores/modules/hearing.ts | 3 + packages/stage-ui/src/workers/vad/vad.test.ts | 114 ++++++++++++++++++ packages/stage-ui/src/workers/vad/vad.ts | 38 ++++-- 14 files changed, 280 insertions(+), 20 deletions(-) create mode 100644 packages/stage-ui/src/workers/vad/vad.test.ts diff --git a/apps/stage-pocket/src/pages/index.vue b/apps/stage-pocket/src/pages/index.vue index cc15465c6..4992e720c 100644 --- a/apps/stage-pocket/src/pages/index.vue +++ b/apps/stage-pocket/src/pages/index.vue @@ -46,7 +46,7 @@ onMounted(() => syncBackgroundTheme()) // Audio + transcription pipeline (mirrors stage-tamagotchi) const settingsAudioDeviceStore = useSettingsAudioDevice() const { stream, enabled } = storeToRefs(settingsAudioDeviceStore) -const { startRecord, stopRecord, onStopRecord } = useAudioRecorder(stream) +const { discardRecord, startRecord, stopRecord, onStopRecord } = useAudioRecorder(stream) const hearingPipeline = useHearingSpeechInputPipeline() const { removeStreamingTranscriptionConsumer, transcribeForRecording, transcribeForMediaStream, stopStreamingTranscription } = hearingPipeline const { supportsStreamInput } = storeToRefs(hearingPipeline) @@ -69,6 +69,7 @@ const { threshold: ref(0.6), onSpeechStart: () => handleSpeechStart(), onSpeechEnd: () => handleSpeechEnd(), + onSpeechCancel: () => handleSpeechCancel(), }) let stopOnStopRecord: (() => void) | undefined @@ -143,6 +144,11 @@ async function handleSpeechEnd() { stopRecord() } +async function handleSpeechCancel() { + if (!shouldUseStreamInput.value) + await discardRecord() +} + function stopAudioInteraction() { try { removeStreamingTranscriptionConsumer(transcriptionConsumerId) diff --git a/apps/stage-web/src/pages/index.vue b/apps/stage-web/src/pages/index.vue index e7cb94b7b..258b33355 100644 --- a/apps/stage-web/src/pages/index.vue +++ b/apps/stage-web/src/pages/index.vue @@ -43,7 +43,7 @@ onMounted(() => syncBackgroundTheme()) // Audio + transcription pipeline (mirrors stage-tamagotchi) const settingsAudioDeviceStore = useSettingsAudioDevice() const { stream, enabled } = storeToRefs(settingsAudioDeviceStore) -const { startRecord, stopRecord, onStopRecord } = useAudioRecorder(stream) +const { discardRecord, startRecord, stopRecord, onStopRecord } = useAudioRecorder(stream) const hearingPipeline = useHearingSpeechInputPipeline() const { removeStreamingTranscriptionConsumer, stopStreamingTranscription, transcribeForMediaStream, transcribeForRecording } = hearingPipeline const { supportsStreamInput } = storeToRefs(hearingPipeline) @@ -66,6 +66,7 @@ const { threshold: ref(0.6), onSpeechStart: () => handleSpeechStart(), onSpeechEnd: () => handleSpeechEnd(), + onSpeechCancel: () => handleSpeechCancel(), }) let stopOnStopRecord: (() => void) | undefined @@ -130,6 +131,11 @@ async function handleSpeechEnd() { stopRecord() } +async function handleSpeechCancel() { + if (!shouldUseStreamInput.value) + await discardRecord() +} + function stopAudioInteraction() { try { removeStreamingTranscriptionConsumer(transcriptionConsumerId) diff --git a/packages/stage-pages/src/pages/settings/modules/hearing.vue b/packages/stage-pages/src/pages/settings/modules/hearing.vue index e22b65faf..21d9dad84 100644 --- a/packages/stage-pages/src/pages/settings/modules/hearing.vue +++ b/packages/stage-pages/src/pages/settings/modules/hearing.vue @@ -41,7 +41,7 @@ const { trackProviderClick } = useAnalytics() const settingsAudioDeviceStore = useSettingsAudioDevice() const { askPermission, stopStream, startStream } = settingsAudioDeviceStore const { audioInputOptions, selectedAudioInput, stream } = storeToRefs(settingsAudioDeviceStore) -const { startRecord, stopRecord, onStopRecord } = useAudioRecorder(stream) +const { discardRecord, startRecord, stopRecord, onStopRecord } = useAudioRecorder(stream) const { startAnalyzer, stopAnalyzer, onAnalyzerUpdate, volumeLevel } = useAudioAnalyzer() const { audioContext } = storeToRefs(useAudioContext()) const hearingSpeechInputPipeline = useHearingSpeechInputPipeline() @@ -200,6 +200,10 @@ const { onSpeechEnd: () => { void handleSpeechEnd() }, + onSpeechCancel: () => { + if (!isTestingSTT.value && !shouldUseStreamInput.value) + void discardRecord() + }, }) const isSpeechVolume = ref(false) // Volume-based speaking detection diff --git a/packages/stage-ui/src/composables/audio/audio-recorder.test.ts b/packages/stage-ui/src/composables/audio/audio-recorder.test.ts index 483083ca9..67b4a82eb 100644 --- a/packages/stage-ui/src/composables/audio/audio-recorder.test.ts +++ b/packages/stage-ui/src/composables/audio/audio-recorder.test.ts @@ -130,6 +130,32 @@ describe('useAudioRecorder', () => { expect(activeSecondOutput?.finalized).toBe(true) }) + // https://github.com/moeru-ai/airi/pull/2258#discussion_r3759566513 + it('finalizes a canceled recording without running transcription hooks', async () => { + // ROOT CAUSE: + // + // Recorder consumers routed VAD cancellation through stopRecord. That + // method ran the normal stop hooks, so rejected noise reached ASR and could + // create a user message. + // + // We finalize canceled audio through a separate discard operation that + // does not create a recording blob or run stop hooks. + const { useAudioRecorder } = await import('./audio-recorder') + const stream = shallowRef(createMediaStream()) + const recorder = useAudioRecorder(stream) + const onStopRecord = vi.fn(async () => {}) + recorder.onStopRecord(onStopRecord) + + await recorder.startRecord() + const activeOutput = mediabunnyMock.outputs.at(-1) + + await recorder.discardRecord() + + expect(activeOutput?.finalized).toBe(true) + expect(recorder.isRecording.value).toBe(false) + expect(onStopRecord).not.toHaveBeenCalled() + }) + it('resets recorder state after startup fails so recording can be retried', async () => { const { useAudioRecorder } = await import('./audio-recorder') const stream = shallowRef(createMediaStream()) diff --git a/packages/stage-ui/src/composables/audio/audio-recorder.ts b/packages/stage-ui/src/composables/audio/audio-recorder.ts index 0f0406869..633524cc5 100644 --- a/packages/stage-ui/src/composables/audio/audio-recorder.ts +++ b/packages/stage-ui/src/composables/audio/audio-recorder.ts @@ -72,15 +72,11 @@ export function useAudioRecorder( } } - /** - * Finalizes the active recording and runs stop hooks without blocking the next recording. - */ - async function stopRecord() { + async function finalizeRecord(notifyStopHooks: boolean) { const activeOutput = mediaOutput.value const activeFormat = mediaFormat.value - if (!activeOutput) { + if (!activeOutput) return - } // Clear the active output before running transcription hooks so VAD can start the next utterance // while the previous blob is still being sent to the ASR provider. @@ -88,6 +84,9 @@ export function useAudioRecorder( mediaFormat.value = undefined await activeOutput.finalize() + if (!notifyStopHooks) + return + const bufferTarget = activeOutput.target as BufferTarget | undefined const buffer = bufferTarget?.buffer const audioBlob = buffer ? new Blob([buffer], { type: activeFormat }) : undefined @@ -107,9 +106,22 @@ export function useAudioRecorder( return audioBlob } + /** + * Finalizes the active recording and runs stop hooks without blocking the next recording. + */ + async function stopRecord() { + return await finalizeRecord(true) + } + + /** Finalizes the active recording without creating a blob or running stop hooks. */ + async function discardRecord() { + await finalizeRecord(false) + } + return { startRecord, stopRecord, + discardRecord, onStopRecord, isRecording, diff --git a/packages/stage-ui/src/composables/audio/voice-input-session.test.ts b/packages/stage-ui/src/composables/audio/voice-input-session.test.ts index b999e9037..62998b0e7 100644 --- a/packages/stage-ui/src/composables/audio/voice-input-session.test.ts +++ b/packages/stage-ui/src/composables/audio/voice-input-session.test.ts @@ -12,6 +12,7 @@ const vadMock = vi.hoisted(() => ({ options: undefined as { onSpeechStart?: () => void onSpeechEnd?: () => void + onSpeechCancel?: () => void onSpeechReady?: (event: { buffer: Float32Array, duration: number }) => void minSilenceDurationMs?: number } | undefined, @@ -40,6 +41,7 @@ vi.mock('../../stores/ai/models/vad', async () => { isSpeechProb: vue.ref(0), isSpeechHistory: vue.ref([]), inferenceError: vue.ref(), + loading: vue.ref(false), minSilenceDurationMs: vue.toRef(options?.minSilenceDurationMs ?? 1200), } }, @@ -159,6 +161,29 @@ describe('useVoiceInputSession', () => { expect(hearingPipelineMock.transcribeForRecording).toHaveBeenCalledWith(recorderRecording) }) + it('discards a VAD recording when the detected speech is shorter than the minimum duration', async () => { + const { useVoiceInputSession } = await import('./voice-input-session') + + audioRecorderMock.startRecord.mockImplementation(async () => { + audioRecorderMock.isRecording.value = true + }) + audioRecorderMock.stopRecord.mockImplementation(async () => { + audioRecorderMock.isRecording.value = false + await audioRecorderMock.onStopRecordHook?.(new Blob(['noise'], { type: 'audio/wav' })) + }) + + const session = useVoiceInputSession(shallowRef(createMediaStream()), { + volumeFallback: { enabled: false }, + }) + + await expect(session.startSegment('vad')).resolves.toBe(true) + vadMock.options?.onSpeechCancel?.() + + await vi.waitFor(() => expect(session.activeRecordingTrigger.value).toBeUndefined()) + expect(audioRecorderMock.stopRecord).toHaveBeenCalledOnce() + expect(hearingPipelineMock.transcribeForRecording).not.toHaveBeenCalled() + }) + it('clears the active recorder segment when discarding fails during stop', async () => { const { useVoiceInputSession } = await import('./voice-input-session') diff --git a/packages/stage-ui/src/composables/audio/voice-input-session.ts b/packages/stage-ui/src/composables/audio/voice-input-session.ts index 4538da6cd..0117f2324 100644 --- a/packages/stage-ui/src/composables/audio/voice-input-session.ts +++ b/packages/stage-ui/src/composables/audio/voice-input-session.ts @@ -134,6 +134,7 @@ export function useVoiceInputSession( isSpeechHistory, inferenceError: vadError, minSilenceDurationMs: vadMinSilenceDurationMs, + loading: vadLoading, } = useVAD(workletUrl, { threshold: options.vad?.threshold, minSilenceDurationMs: options.vad?.minSilenceDurationMs, @@ -145,6 +146,13 @@ export function useVoiceInputSession( onSpeechEnd: () => { void stopSegment('vad') }, + onSpeechCancel: () => { + const segment = activeRecordingSegment.value + if (!segment || segment.trigger !== 'vad') + return + + void discardActiveRecorderSegment(segment) + }, onSpeechReady: ({ buffer }) => { const segment = activeRecordingSegment.value if (!segment || segment.trigger !== 'vad') @@ -604,6 +612,7 @@ export function useVoiceInputSession( isSpeechProb, isSpeechHistory, vadLoaded, + vadLoading, vadError, startSegment, diff --git a/packages/stage-ui/src/libs/audio/vad-streaming-session.test.ts b/packages/stage-ui/src/libs/audio/vad-streaming-session.test.ts index 9e29d0aec..37c17e285 100644 --- a/packages/stage-ui/src/libs/audio/vad-streaming-session.test.ts +++ b/packages/stage-ui/src/libs/audio/vad-streaming-session.test.ts @@ -36,6 +36,22 @@ describe('createVadStreamingSession', () => { expect(stop).toHaveBeenCalledTimes(1) }) + it('starts a fresh provider session for speech detected after the first segment stops', async () => { + const start = vi.fn(async () => {}) + const stop = vi.fn(async () => {}) + const session = createVadStreamingSession({ start, stop }) + + session.onSpeechStart() + session.onSpeechEnd() + await vi.waitFor(() => expect(stop).toHaveBeenCalledTimes(1)) + + session.onSpeechStart() + session.onSpeechEnd() + await vi.waitFor(() => expect(stop).toHaveBeenCalledTimes(2)) + + expect(start).toHaveBeenCalledTimes(2) + }) + it('does not start another session after disposal', async () => { const start = vi.fn(async () => {}) const stop = vi.fn(async () => {}) diff --git a/packages/stage-ui/src/libs/audio/vad.ts b/packages/stage-ui/src/libs/audio/vad.ts index 4a50d9e4b..3795d2805 100644 --- a/packages/stage-ui/src/libs/audio/vad.ts +++ b/packages/stage-ui/src/libs/audio/vad.ts @@ -24,6 +24,8 @@ export interface VADEvents { 'speech-audio': { buffer: Float32Array } // Emitted when speech has ended 'speech-end': void + // Emitted when detected speech is too short to produce a segment + 'speech-cancel': void // Emitted when a complete speech segment is ready for transcription 'speech-ready': { buffer: Float32Array, duration: number } // Emitted for status updates and errors diff --git a/packages/stage-ui/src/stores/ai/models/vad.test.ts b/packages/stage-ui/src/stores/ai/models/vad.test.ts index 8c7cb449f..5dd6a0ba6 100644 --- a/packages/stage-ui/src/stores/ai/models/vad.test.ts +++ b/packages/stage-ui/src/stores/ai/models/vad.test.ts @@ -113,4 +113,18 @@ describe('useVAD', () => { expect(onSpeechAudio).toHaveBeenCalledWith({ buffer }) }) + + it('forwards canceled speech only to the dedicated cancel handler', async () => { + const onSpeechCancel = vi.fn() + const onSpeechEnd = vi.fn() + const { useVAD } = await import('./vad') + const vad = useVAD('vad-worker-url', { onSpeechCancel, onSpeechEnd }) + + await vad.init() + vadMocks.handlers.get('speech-start')?.() + vadMocks.handlers.get('speech-cancel')?.() + + expect(onSpeechCancel).toHaveBeenCalledOnce() + expect(onSpeechEnd).not.toHaveBeenCalled() + }) }) diff --git a/packages/stage-ui/src/stores/ai/models/vad.ts b/packages/stage-ui/src/stores/ai/models/vad.ts index 47438d75e..1fef44052 100644 --- a/packages/stage-ui/src/stores/ai/models/vad.ts +++ b/packages/stage-ui/src/stores/ai/models/vad.ts @@ -19,6 +19,7 @@ interface UseVADOptions { onSpeechStart?: () => void onSpeechAudio?: (event: { buffer: Float32Array }) => void onSpeechEnd?: () => void + onSpeechCancel?: () => void onSpeechReady?: (event: { buffer: Float32Array, duration: number }) => void } @@ -121,6 +122,12 @@ export function useVAD(workerUrl: string, options?: UseVADOptions) { options?.onSpeechEnd?.() }) + vad.value.on('speech-cancel', () => { + finishActiveSpan(true) + isSpeech.value = false + options?.onSpeechCancel?.() + }) + vad.value.on('speech-ready', (event) => { activeSpan?.setAttribute(IOAttributes.VADAudioDurationMs, event.duration) finishActiveSpan(false) diff --git a/packages/stage-ui/src/stores/modules/hearing.ts b/packages/stage-ui/src/stores/modules/hearing.ts index f30abaf98..a604c0460 100644 --- a/packages/stage-ui/src/stores/modules/hearing.ts +++ b/packages/stage-ui/src/stores/modules/hearing.ts @@ -954,6 +954,9 @@ export const useHearingSpeechInputPipeline = defineStore('modules:hearing:speech onSpeechEnd: () => { vadSession.lifecycle.onSpeechEnd() }, + onSpeechCancel: () => { + vadSession.lifecycle.onSpeechEnd() + }, }) const lifecycle = createVadStreamingSession({ start: async () => await startVadRealtimeTranscription(providerId, options, vadSession), diff --git a/packages/stage-ui/src/workers/vad/vad.test.ts b/packages/stage-ui/src/workers/vad/vad.test.ts new file mode 100644 index 000000000..5aba9f446 --- /dev/null +++ b/packages/stage-ui/src/workers/vad/vad.test.ts @@ -0,0 +1,114 @@ +import type { PreTrainedModel } from '@huggingface/transformers' + +import { AutoModel } from '@huggingface/transformers' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +import { VAD } from './vad' + +vi.mock('@huggingface/transformers', async (importOriginal) => { + const actual = await importOriginal() + + return { + ...actual, + AutoModel: { + from_pretrained: vi.fn(), + }, + } +}) + +function createProbabilityModel(probabilities: number[]) { + return vi.fn(async (input: { state: unknown }) => ({ + stateN: input.state, + output: { + data: new Float32Array([probabilities.shift() ?? 0]), + }, + })) as unknown as PreTrainedModel +} + +describe('vad speech duration', () => { + beforeEach(() => { + vi.mocked(AutoModel.from_pretrained).mockReset() + }) + + it('rejects a noise pulse that is shorter than the minimum speech duration', async () => { + // ROOT CAUSE: + // + // The minimum speech check used the full segment buffer. That buffer also + // contained the silence that closed the segment. An 800 ms silence period + // therefore made a short noise pulse pass a 300 ms speech requirement. + // + // We count detected speech samples separately from post-speech silence. + const probabilities = [0.9, ...Array.from({ length: 80 }).fill(0)] + vi.mocked(AutoModel.from_pretrained).mockResolvedValue(createProbabilityModel(probabilities)) + const vad = new VAD({ + sampleRate: 16000, + newBufferSize: 160, + minSilenceDurationMs: 800, + minSpeechDurationMs: 300, + }) + const onSpeechReady = vi.fn() + const onSpeechCancel = vi.fn() + vad.on('speech-ready', onSpeechReady) + vad.on('speech-cancel', onSpeechCancel) + + await vad.initialize() + for (let index = 0; index < 81; index++) + await vad.processAudio(new Float32Array(160)) + + expect(onSpeechReady).not.toHaveBeenCalled() + expect(onSpeechCancel).toHaveBeenCalledOnce() + }) + + it('keeps speech that reaches the minimum speech duration', async () => { + const probabilities = [ + ...Array.from({ length: 30 }).fill(0.9), + ...Array.from({ length: 80 }).fill(0), + ] + vi.mocked(AutoModel.from_pretrained).mockResolvedValue(createProbabilityModel(probabilities)) + const vad = new VAD({ + sampleRate: 16000, + newBufferSize: 160, + minSilenceDurationMs: 800, + minSpeechDurationMs: 300, + }) + const onSpeechReady = vi.fn() + vad.on('speech-ready', onSpeechReady) + + await vad.initialize() + for (let index = 0; index < 110; index++) + await vad.processAudio(new Float32Array(160)) + + expect(onSpeechReady).toHaveBeenCalledOnce() + }) + + it('serializes overlapping worklet callbacks across consecutive speech segments', async () => { + // ROOT CAUSE: + // + // AudioWorklet message handlers do not wait for an async callback to finish. + // Concurrent processAudio calls therefore captured the same stale recording + // state even though model inference itself was queued. The first segment + // could complete while the second segment never reached a valid transition. + // + // We serialize the complete detection and state transition for every chunk. + const probabilities = [ + ...Array.from({ length: 30 }).fill(0.9), + ...Array.from({ length: 80 }).fill(0), + ...Array.from({ length: 30 }).fill(0.9), + ...Array.from({ length: 80 }).fill(0), + ] + vi.mocked(AutoModel.from_pretrained).mockResolvedValue(createProbabilityModel(probabilities)) + const vad = new VAD({ + sampleRate: 16000, + newBufferSize: 160, + minSilenceDurationMs: 800, + minSpeechDurationMs: 300, + }) + const onSpeechReady = vi.fn() + vad.on('speech-ready', onSpeechReady) + + await vad.initialize() + await Promise.all(Array.from({ length: 220 }, () => vad.processAudio(new Float32Array(160)))) + + expect(onSpeechReady).toHaveBeenCalledTimes(2) + }) +}) diff --git a/packages/stage-ui/src/workers/vad/vad.ts b/packages/stage-ui/src/workers/vad/vad.ts index eeb1aa387..924fbbf00 100644 --- a/packages/stage-ui/src/workers/vad/vad.ts +++ b/packages/stage-ui/src/workers/vad/vad.ts @@ -15,9 +15,10 @@ export class VAD implements BaseVAD { private buffer: Float32Array private bufferPointer: number = 0 private isRecording: boolean = false + private speechSamples: number = 0 private postSpeechSamples: number = 0 private prevBuffers: Float32Array[] = [] - private inferenceChain: Promise = Promise.resolve() + private processingChain: Promise = Promise.resolve() private eventListeners: Partial[]>> = {} private isReady: boolean = false @@ -93,7 +94,16 @@ export class VAD implements BaseVAD { /** * Process audio buffer for speech detection */ - public async processAudio(inputBuffer: Float32Array): Promise { + public processAudio(inputBuffer: Float32Array): Promise { + // AudioWorklet dispatch does not await async message handlers. Queue the + // complete state transition so each chunk observes the previous result. + const queuedBuffer = inputBuffer.slice() + const processing = this.processingChain.then(async () => await this.processAudioChunk(queuedBuffer)) + this.processingChain = processing.catch(() => undefined) + return processing + } + + private async processAudioChunk(inputBuffer: Float32Array): Promise { if (!this.isReady) { throw new Error('VAD model is not initialized. Call initialize() first.') } @@ -103,6 +113,9 @@ export class VAD implements BaseVAD { // Perform VAD on the input buffer const isSpeech = await this.detectSpeech(inputBuffer) + if (isSpeech) + this.speechSamples += inputBuffer.length + // Calculate derived constants const sampleRateMs = this.config.sampleRate / 1000 const minSilenceDurationSamples = this.config.minSilenceDurationMs * sampleRateMs @@ -167,9 +180,9 @@ export class VAD implements BaseVAD { // Check if silence is long enough to consider speech ended if (this.postSpeechSamples >= minSilenceDurationSamples) { // Check if the speech segment is long enough to process - if (this.bufferPointer < minSpeechDurationSamples) { + if (this.speechSamples < minSpeechDurationSamples) { // Too short, reset without processing - this.emit('speech-end', undefined) + this.emit('speech-cancel', undefined) this.reset() return @@ -201,13 +214,15 @@ export class VAD implements BaseVAD { private async detectSpeech(buffer: Float32Array): Promise { const input = new Tensor('float32', buffer, [1, buffer.length]) - const { stateN, output } = await (this.inferenceChain = this.inferenceChain.then(() => - this.model?.({ - input, - sr: this.sampleRateTensor, - state: this.state, - }), - )) + const model = this.model + if (!model) + throw new Error('VAD model is not initialized. Call initialize() first.') + + const { stateN, output } = await model({ + input, + sr: this.sampleRateTensor, + state: this.state, + }) // Update the state this.state = stateN @@ -269,6 +284,7 @@ export class VAD implements BaseVAD { this.buffer.fill(0, offset) this.bufferPointer = offset this.isRecording = false + this.speechSamples = 0 this.postSpeechSamples = 0 this.prevBuffers = [] }