fix(stage-ui): correct VAD segment processing (#2258)

apps/stage-pocket/src/pages/index.vue

@@ -46,7 +46,7 @@ onMounted(() => syncBackgroundTheme())
 // Audio + transcription pipeline (mirrors stage-tamagotchi)
 const settingsAudioDeviceStore = useSettingsAudioDevice()
 const { stream, enabled } = storeToRefs(settingsAudioDeviceStore)
-const { startRecord, stopRecord, onStopRecord } = useAudioRecorder(stream)
+const { discardRecord, startRecord, stopRecord, onStopRecord } = useAudioRecorder(stream)
 const hearingPipeline = useHearingSpeechInputPipeline()
 const { removeStreamingTranscriptionConsumer, transcribeForRecording, transcribeForMediaStream, stopStreamingTranscription } = hearingPipeline
 const { supportsStreamInput } = storeToRefs(hearingPipeline)
@@ -69,6 +69,7 @@ const {
   threshold: ref(0.6),
   onSpeechStart: () => handleSpeechStart(),
   onSpeechEnd: () => handleSpeechEnd(),
+  onSpeechCancel: () => handleSpeechCancel(),
 })
 
 let stopOnStopRecord: (() => void) | undefined
@@ -143,6 +144,11 @@ async function handleSpeechEnd() {
   stopRecord()
 }
 
+async function handleSpeechCancel() {
+  if (!shouldUseStreamInput.value)
+    await discardRecord()
+}
+
 function stopAudioInteraction() {
   try {
     removeStreamingTranscriptionConsumer(transcriptionConsumerId)

apps/stage-web/src/pages/index.vue

@@ -43,7 +43,7 @@ onMounted(() => syncBackgroundTheme())
 // Audio + transcription pipeline (mirrors stage-tamagotchi)
 const settingsAudioDeviceStore = useSettingsAudioDevice()
 const { stream, enabled } = storeToRefs(settingsAudioDeviceStore)
-const { startRecord, stopRecord, onStopRecord } = useAudioRecorder(stream)
+const { discardRecord, startRecord, stopRecord, onStopRecord } = useAudioRecorder(stream)
 const hearingPipeline = useHearingSpeechInputPipeline()
 const { removeStreamingTranscriptionConsumer, stopStreamingTranscription, transcribeForMediaStream, transcribeForRecording } = hearingPipeline
 const { supportsStreamInput } = storeToRefs(hearingPipeline)
@@ -66,6 +66,7 @@ const {
   threshold: ref(0.6),
   onSpeechStart: () => handleSpeechStart(),
   onSpeechEnd: () => handleSpeechEnd(),
+  onSpeechCancel: () => handleSpeechCancel(),
 })
 
 let stopOnStopRecord: (() => void) | undefined
@@ -130,6 +131,11 @@ async function handleSpeechEnd() {
   stopRecord()
 }
 
+async function handleSpeechCancel() {
+  if (!shouldUseStreamInput.value)
+    await discardRecord()
+}
+
 function stopAudioInteraction() {
   try {
     removeStreamingTranscriptionConsumer(transcriptionConsumerId)

packages/stage-pages/src/pages/settings/modules/hearing.vue

@@ -41,7 +41,7 @@ const { trackProviderClick } = useAnalytics()
 const settingsAudioDeviceStore = useSettingsAudioDevice()
 const { askPermission, stopStream, startStream } = settingsAudioDeviceStore
 const { audioInputOptions, selectedAudioInput, stream } = storeToRefs(settingsAudioDeviceStore)
-const { startRecord, stopRecord, onStopRecord } = useAudioRecorder(stream)
+const { discardRecord, startRecord, stopRecord, onStopRecord } = useAudioRecorder(stream)
 const { startAnalyzer, stopAnalyzer, onAnalyzerUpdate, volumeLevel } = useAudioAnalyzer()
 const { audioContext } = storeToRefs(useAudioContext())
 const hearingSpeechInputPipeline = useHearingSpeechInputPipeline()
@@ -200,6 +200,10 @@ const {
   onSpeechEnd: () => {
     void handleSpeechEnd()
   },
+  onSpeechCancel: () => {
+    if (!isTestingSTT.value && !shouldUseStreamInput.value)
+      void discardRecord()
+  },
 })
 
 const isSpeechVolume = ref(false) // Volume-based speaking detection

packages/stage-ui/src/composables/audio/audio-recorder.test.ts

@@ -130,6 +130,32 @@ describe('useAudioRecorder', () => {
     expect(activeSecondOutput?.finalized).toBe(true)
   })
 
+  // https://github.com/moeru-ai/airi/pull/2258#discussion_r3759566513
+  it('finalizes a canceled recording without running transcription hooks', async () => {
+    // ROOT CAUSE:
+    //
+    // Recorder consumers routed VAD cancellation through stopRecord. That
+    // method ran the normal stop hooks, so rejected noise reached ASR and could
+    // create a user message.
+    //
+    // We finalize canceled audio through a separate discard operation that
+    // does not create a recording blob or run stop hooks.
+    const { useAudioRecorder } = await import('./audio-recorder')
+    const stream = shallowRef(createMediaStream())
+    const recorder = useAudioRecorder(stream)
+    const onStopRecord = vi.fn(async () => {})
+    recorder.onStopRecord(onStopRecord)
+
+    await recorder.startRecord()
+    const activeOutput = mediabunnyMock.outputs.at(-1)
+
+    await recorder.discardRecord()
+
+    expect(activeOutput?.finalized).toBe(true)
+    expect(recorder.isRecording.value).toBe(false)
+    expect(onStopRecord).not.toHaveBeenCalled()
+  })
+
   it('resets recorder state after startup fails so recording can be retried', async () => {
     const { useAudioRecorder } = await import('./audio-recorder')
     const stream = shallowRef(createMediaStream())

packages/stage-ui/src/composables/audio/audio-recorder.ts

@@ -72,15 +72,11 @@ export function useAudioRecorder(
     }
   }
 
-  /**
-   * Finalizes the active recording and runs stop hooks without blocking the next recording.
-   */
-  async function stopRecord() {
+  async function finalizeRecord(notifyStopHooks: boolean) {
     const activeOutput = mediaOutput.value
     const activeFormat = mediaFormat.value
-    if (!activeOutput) {
+    if (!activeOutput)
       return
-    }
 
     // Clear the active output before running transcription hooks so VAD can start the next utterance
     // while the previous blob is still being sent to the ASR provider.
@@ -88,6 +84,9 @@ export function useAudioRecorder(
     mediaFormat.value = undefined
 
     await activeOutput.finalize()
+    if (!notifyStopHooks)
+      return
+
     const bufferTarget = activeOutput.target as BufferTarget | undefined
     const buffer = bufferTarget?.buffer
     const audioBlob = buffer ? new Blob([buffer], { type: activeFormat }) : undefined
@@ -107,9 +106,22 @@ export function useAudioRecorder(
     return audioBlob
   }
 
+  /**
+   * Finalizes the active recording and runs stop hooks without blocking the next recording.
+   */
+  async function stopRecord() {
+    return await finalizeRecord(true)
+  }
+
+  /** Finalizes the active recording without creating a blob or running stop hooks. */
+  async function discardRecord() {
+    await finalizeRecord(false)
+  }
+
   return {
     startRecord,
     stopRecord,
+    discardRecord,
     onStopRecord,
 
     isRecording,

packages/stage-ui/src/composables/audio/voice-input-session.test.ts

@@ -12,6 +12,7 @@ const vadMock = vi.hoisted(() => ({
   options: undefined as {
     onSpeechStart?: () => void
     onSpeechEnd?: () => void
+    onSpeechCancel?: () => void
     onSpeechReady?: (event: { buffer: Float32Array, duration: number }) => void
     minSilenceDurationMs?: number
   } | undefined,
@@ -40,6 +41,7 @@ vi.mock('../../stores/ai/models/vad', async () => {
         isSpeechProb: vue.ref(0),
         isSpeechHistory: vue.ref([]),
         inferenceError: vue.ref(),
+        loading: vue.ref(false),
         minSilenceDurationMs: vue.toRef(options?.minSilenceDurationMs ?? 1200),
       }
     },
@@ -159,6 +161,29 @@ describe('useVoiceInputSession', () => {
     expect(hearingPipelineMock.transcribeForRecording).toHaveBeenCalledWith(recorderRecording)
   })
 
+  it('discards a VAD recording when the detected speech is shorter than the minimum duration', async () => {
+    const { useVoiceInputSession } = await import('./voice-input-session')
+
+    audioRecorderMock.startRecord.mockImplementation(async () => {
+      audioRecorderMock.isRecording.value = true
+    })
+    audioRecorderMock.stopRecord.mockImplementation(async () => {
+      audioRecorderMock.isRecording.value = false
+      await audioRecorderMock.onStopRecordHook?.(new Blob(['noise'], { type: 'audio/wav' }))
+    })
+
+    const session = useVoiceInputSession(shallowRef(createMediaStream()), {
+      volumeFallback: { enabled: false },
+    })
+
+    await expect(session.startSegment('vad')).resolves.toBe(true)
+    vadMock.options?.onSpeechCancel?.()
+
+    await vi.waitFor(() => expect(session.activeRecordingTrigger.value).toBeUndefined())
+    expect(audioRecorderMock.stopRecord).toHaveBeenCalledOnce()
+    expect(hearingPipelineMock.transcribeForRecording).not.toHaveBeenCalled()
+  })
+
   it('clears the active recorder segment when discarding fails during stop', async () => {
     const { useVoiceInputSession } = await import('./voice-input-session')
 

packages/stage-ui/src/composables/audio/voice-input-session.ts

@@ -134,6 +134,7 @@ export function useVoiceInputSession(
     isSpeechHistory,
     inferenceError: vadError,
     minSilenceDurationMs: vadMinSilenceDurationMs,
+    loading: vadLoading,
   } = useVAD(workletUrl, {
     threshold: options.vad?.threshold,
     minSilenceDurationMs: options.vad?.minSilenceDurationMs,
@@ -145,6 +146,13 @@ export function useVoiceInputSession(
     onSpeechEnd: () => {
       void stopSegment('vad')
     },
+    onSpeechCancel: () => {
+      const segment = activeRecordingSegment.value
+      if (!segment || segment.trigger !== 'vad')
+        return
+
+      void discardActiveRecorderSegment(segment)
+    },
     onSpeechReady: ({ buffer }) => {
       const segment = activeRecordingSegment.value
       if (!segment || segment.trigger !== 'vad')
@@ -604,6 +612,7 @@ export function useVoiceInputSession(
     isSpeechProb,
     isSpeechHistory,
     vadLoaded,
+    vadLoading,
     vadError,
 
     startSegment,

packages/stage-ui/src/libs/audio/vad-streaming-session.test.ts

@@ -36,6 +36,22 @@ describe('createVadStreamingSession', () => {
     expect(stop).toHaveBeenCalledTimes(1)
   })
 
+  it('starts a fresh provider session for speech detected after the first segment stops', async () => {
+    const start = vi.fn(async () => {})
+    const stop = vi.fn(async () => {})
+    const session = createVadStreamingSession({ start, stop })
+
+    session.onSpeechStart()
+    session.onSpeechEnd()
+    await vi.waitFor(() => expect(stop).toHaveBeenCalledTimes(1))
+
+    session.onSpeechStart()
+    session.onSpeechEnd()
+    await vi.waitFor(() => expect(stop).toHaveBeenCalledTimes(2))
+
+    expect(start).toHaveBeenCalledTimes(2)
+  })
+
   it('does not start another session after disposal', async () => {
     const start = vi.fn(async () => {})
     const stop = vi.fn(async () => {})

packages/stage-ui/src/libs/audio/vad.ts

@@ -24,6 +24,8 @@ export interface VADEvents {
   'speech-audio': { buffer: Float32Array }
   // Emitted when speech has ended
   'speech-end': void
+  // Emitted when detected speech is too short to produce a segment
+  'speech-cancel': void
   // Emitted when a complete speech segment is ready for transcription
   'speech-ready': { buffer: Float32Array, duration: number }
   // Emitted for status updates and errors

packages/stage-ui/src/stores/ai/models/vad.test.ts

@@ -113,4 +113,18 @@ describe('useVAD', () => {
 
     expect(onSpeechAudio).toHaveBeenCalledWith({ buffer })
   })
+
+  it('forwards canceled speech only to the dedicated cancel handler', async () => {
+    const onSpeechCancel = vi.fn()
+    const onSpeechEnd = vi.fn()
+    const { useVAD } = await import('./vad')
+    const vad = useVAD('vad-worker-url', { onSpeechCancel, onSpeechEnd })
+
+    await vad.init()
+    vadMocks.handlers.get('speech-start')?.()
+    vadMocks.handlers.get('speech-cancel')?.()
+
+    expect(onSpeechCancel).toHaveBeenCalledOnce()
+    expect(onSpeechEnd).not.toHaveBeenCalled()
+  })
 })

packages/stage-ui/src/stores/ai/models/vad.ts

@@ -19,6 +19,7 @@ interface UseVADOptions {
   onSpeechStart?: () => void
   onSpeechAudio?: (event: { buffer: Float32Array }) => void
   onSpeechEnd?: () => void
+  onSpeechCancel?: () => void
   onSpeechReady?: (event: { buffer: Float32Array, duration: number }) => void
 }
 
@@ -121,6 +122,12 @@ export function useVAD(workerUrl: string, options?: UseVADOptions) {
         options?.onSpeechEnd?.()
       })
 
+      vad.value.on('speech-cancel', () => {
+        finishActiveSpan(true)
+        isSpeech.value = false
+        options?.onSpeechCancel?.()
+      })
+
       vad.value.on('speech-ready', (event) => {
         activeSpan?.setAttribute(IOAttributes.VADAudioDurationMs, event.duration)
         finishActiveSpan(false)

packages/stage-ui/src/stores/modules/hearing.ts

@@ -954,6 +954,9 @@ export const useHearingSpeechInputPipeline = defineStore('modules:hearing:speech
       onSpeechEnd: () => {
         vadSession.lifecycle.onSpeechEnd()
       },
+      onSpeechCancel: () => {
+        vadSession.lifecycle.onSpeechEnd()
+      },
     })
     const lifecycle = createVadStreamingSession({
       start: async () => await startVadRealtimeTranscription(providerId, options, vadSession),

packages/stage-ui/src/workers/vad/vad.test.ts

@@ -0,0 +1,114 @@
+import type { PreTrainedModel } from '@huggingface/transformers'
+
+import { AutoModel } from '@huggingface/transformers'
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+import { VAD } from './vad'
+
+vi.mock('@huggingface/transformers', async (importOriginal) => {
+  const actual = await importOriginal<typeof import('@huggingface/transformers')>()
+
+  return {
+    ...actual,
+    AutoModel: {
+      from_pretrained: vi.fn(),
+    },
+  }
+})
+
+function createProbabilityModel(probabilities: number[]) {
+  return vi.fn(async (input: { state: unknown }) => ({
+    stateN: input.state,
+    output: {
+      data: new Float32Array([probabilities.shift() ?? 0]),
+    },
+  })) as unknown as PreTrainedModel
+}
+
+describe('vad speech duration', () => {
+  beforeEach(() => {
+    vi.mocked(AutoModel.from_pretrained).mockReset()
+  })
+
+  it('rejects a noise pulse that is shorter than the minimum speech duration', async () => {
+    // ROOT CAUSE:
+    //
+    // The minimum speech check used the full segment buffer. That buffer also
+    // contained the silence that closed the segment. An 800 ms silence period
+    // therefore made a short noise pulse pass a 300 ms speech requirement.
+    //
+    // We count detected speech samples separately from post-speech silence.
+    const probabilities = [0.9, ...Array.from<number>({ length: 80 }).fill(0)]
+    vi.mocked(AutoModel.from_pretrained).mockResolvedValue(createProbabilityModel(probabilities))
+    const vad = new VAD({
+      sampleRate: 16000,
+      newBufferSize: 160,
+      minSilenceDurationMs: 800,
+      minSpeechDurationMs: 300,
+    })
+    const onSpeechReady = vi.fn()
+    const onSpeechCancel = vi.fn()
+    vad.on('speech-ready', onSpeechReady)
+    vad.on('speech-cancel', onSpeechCancel)
+
+    await vad.initialize()
+    for (let index = 0; index < 81; index++)
+      await vad.processAudio(new Float32Array(160))
+
+    expect(onSpeechReady).not.toHaveBeenCalled()
+    expect(onSpeechCancel).toHaveBeenCalledOnce()
+  })
+
+  it('keeps speech that reaches the minimum speech duration', async () => {
+    const probabilities = [
+      ...Array.from<number>({ length: 30 }).fill(0.9),
+      ...Array.from<number>({ length: 80 }).fill(0),
+    ]
+    vi.mocked(AutoModel.from_pretrained).mockResolvedValue(createProbabilityModel(probabilities))
+    const vad = new VAD({
+      sampleRate: 16000,
+      newBufferSize: 160,
+      minSilenceDurationMs: 800,
+      minSpeechDurationMs: 300,
+    })
+    const onSpeechReady = vi.fn()
+    vad.on('speech-ready', onSpeechReady)
+
+    await vad.initialize()
+    for (let index = 0; index < 110; index++)
+      await vad.processAudio(new Float32Array(160))
+
+    expect(onSpeechReady).toHaveBeenCalledOnce()
+  })
+
+  it('serializes overlapping worklet callbacks across consecutive speech segments', async () => {
+    // ROOT CAUSE:
+    //
+    // AudioWorklet message handlers do not wait for an async callback to finish.
+    // Concurrent processAudio calls therefore captured the same stale recording
+    // state even though model inference itself was queued. The first segment
+    // could complete while the second segment never reached a valid transition.
+    //
+    // We serialize the complete detection and state transition for every chunk.
+    const probabilities = [
+      ...Array.from<number>({ length: 30 }).fill(0.9),
+      ...Array.from<number>({ length: 80 }).fill(0),
+      ...Array.from<number>({ length: 30 }).fill(0.9),
+      ...Array.from<number>({ length: 80 }).fill(0),
+    ]
+    vi.mocked(AutoModel.from_pretrained).mockResolvedValue(createProbabilityModel(probabilities))
+    const vad = new VAD({
+      sampleRate: 16000,
+      newBufferSize: 160,
+      minSilenceDurationMs: 800,
+      minSpeechDurationMs: 300,
+    })
+    const onSpeechReady = vi.fn()
+    vad.on('speech-ready', onSpeechReady)
+
+    await vad.initialize()
+    await Promise.all(Array.from({ length: 220 }, () => vad.processAudio(new Float32Array(160))))
+
+    expect(onSpeechReady).toHaveBeenCalledTimes(2)
+  })
+})

packages/stage-ui/src/workers/vad/vad.ts

@@ -15,9 +15,10 @@ export class VAD implements BaseVAD {
   private buffer: Float32Array
   private bufferPointer: number = 0
   private isRecording: boolean = false
+  private speechSamples: number = 0
   private postSpeechSamples: number = 0
   private prevBuffers: Float32Array[] = []
-  private inferenceChain: Promise<any> = Promise.resolve()
+  private processingChain: Promise<void> = Promise.resolve()
   private eventListeners: Partial<Record<keyof VADEvents, VADEventCallback<any>[]>> = {}
   private isReady: boolean = false
 
@@ -93,7 +94,16 @@ export class VAD implements BaseVAD {
   /**
    * Process audio buffer for speech detection
    */
-  public async processAudio(inputBuffer: Float32Array): Promise<void> {
+  public processAudio(inputBuffer: Float32Array): Promise<void> {
+    // AudioWorklet dispatch does not await async message handlers. Queue the
+    // complete state transition so each chunk observes the previous result.
+    const queuedBuffer = inputBuffer.slice()
+    const processing = this.processingChain.then(async () => await this.processAudioChunk(queuedBuffer))
+    this.processingChain = processing.catch(() => undefined)
+    return processing
+  }
+
+  private async processAudioChunk(inputBuffer: Float32Array): Promise<void> {
     if (!this.isReady) {
       throw new Error('VAD model is not initialized. Call initialize() first.')
     }
@@ -103,6 +113,9 @@ export class VAD implements BaseVAD {
     // Perform VAD on the input buffer
     const isSpeech = await this.detectSpeech(inputBuffer)
 
+    if (isSpeech)
+      this.speechSamples += inputBuffer.length
+
     // Calculate derived constants
     const sampleRateMs = this.config.sampleRate / 1000
     const minSilenceDurationSamples = this.config.minSilenceDurationMs * sampleRateMs
@@ -167,9 +180,9 @@ export class VAD implements BaseVAD {
     // Check if silence is long enough to consider speech ended
     if (this.postSpeechSamples >= minSilenceDurationSamples) {
       // Check if the speech segment is long enough to process
-      if (this.bufferPointer < minSpeechDurationSamples) {
+      if (this.speechSamples < minSpeechDurationSamples) {
         // Too short, reset without processing
-        this.emit('speech-end', undefined)
+        this.emit('speech-cancel', undefined)
         this.reset()
 
         return
@@ -201,13 +214,15 @@ export class VAD implements BaseVAD {
   private async detectSpeech(buffer: Float32Array): Promise<boolean> {
     const input = new Tensor('float32', buffer, [1, buffer.length])
 
-    const { stateN, output } = await (this.inferenceChain = this.inferenceChain.then(() =>
-      this.model?.({
-        input,
-        sr: this.sampleRateTensor,
-        state: this.state,
-      }),
-    ))
+    const model = this.model
+    if (!model)
+      throw new Error('VAD model is not initialized. Call initialize() first.')
+
+    const { stateN, output } = await model({
+      input,
+      sr: this.sampleRateTensor,
+      state: this.state,
+    })
 
     // Update the state
     this.state = stateN
@@ -269,6 +284,7 @@ export class VAD implements BaseVAD {
     this.buffer.fill(0, offset)
     this.bufferPointer = offset
     this.isRecording = false
+    this.speechSamples = 0
     this.postSpeechSamples = 0
     this.prevBuffers = []
   }