From 57397889a58776ca1b11066eb711d54d1c015e10 Mon Sep 17 00:00:00 2001 From: Neko Date: Wed, 12 Aug 2026 05:27:59 +0800 Subject: [PATCH] feat(stage-pages): add Hearing playground (#2265) --- packages/i18n/src/locales/en/settings.yaml | 13 + .../i18n/src/locales/zh-Hans/settings.yaml | 13 + .../components/hearing-playground-segment.vue | 59 ++ .../hearing-playground-transcripts.vue | 78 +++ .../src/pages/settings/modules/hearing.vue | 641 +++++------------- packages/stage-ui/src/composables/index.ts | 1 + .../use-hearing-playground-segments.test.ts | 71 ++ .../use-hearing-playground-segments.ts | 106 +++ 8 files changed, 501 insertions(+), 481 deletions(-) create mode 100644 packages/stage-pages/src/pages/settings/modules/components/hearing-playground-segment.vue create mode 100644 packages/stage-pages/src/pages/settings/modules/components/hearing-playground-transcripts.vue create mode 100644 packages/stage-ui/src/composables/use-hearing-playground-segments.test.ts create mode 100644 packages/stage-ui/src/composables/use-hearing-playground-segments.ts diff --git a/packages/i18n/src/locales/en/settings.yaml b/packages/i18n/src/locales/en/settings.yaml index 3a550042f..4577cac51 100644 --- a/packages/i18n/src/locales/en/settings.yaml +++ b/packages/i18n/src/locales/en/settings.yaml @@ -686,6 +686,19 @@ pages: section: provider-selection: description: Select the suitable speech recognition provider + playground: + current: Current transcript + description: Listen through the active Hearing pipeline and keep each completed speech segment below. + empty: Start monitoring and speak into the selected microphone. + error-title: Transcription error + listening: Listening for speech… + no-transcription: No speech was recognized in this audio. + segment: 'Speech segment {number}' + start: Start monitoring + stop: Stop monitoring + title: Hearing playground + transcribing: Transcribing… + transcription-failed: Transcription failed. confidence-threshold: title: Confidence Threshold description: >- diff --git a/packages/i18n/src/locales/zh-Hans/settings.yaml b/packages/i18n/src/locales/zh-Hans/settings.yaml index 1d718a5c1..680321bd8 100644 --- a/packages/i18n/src/locales/zh-Hans/settings.yaml +++ b/packages/i18n/src/locales/zh-Hans/settings.yaml @@ -657,6 +657,19 @@ pages: section: provider-selection: description: 选择合适的语音转文本的服务来源 + playground: + current: 当前转写 + description: 通过当前的听觉管线监听，并在下方保留每一段完成的语音转写。 + empty: 开始监听，然后对着所选麦克风说话。 + error-title: 转写错误 + listening: 正在等待语音…… + no-transcription: 此段音频中未识别到语音。 + segment: '语音片段 {number}' + start: 开始监听 + stop: 停止监听 + title: 听觉测试面板 + transcribing: 正在转写…… + transcription-failed: 转写失败。 confidence-threshold: title: 置信度阈值 description: >- diff --git a/packages/stage-pages/src/pages/settings/modules/components/hearing-playground-segment.vue b/packages/stage-pages/src/pages/settings/modules/components/hearing-playground-segment.vue new file mode 100644 index 000000000..fb2c5ab20 --- /dev/null +++ b/packages/stage-pages/src/pages/settings/modules/components/hearing-playground-segment.vue @@ -0,0 +1,59 @@ + + + diff --git a/packages/stage-pages/src/pages/settings/modules/components/hearing-playground-transcripts.vue b/packages/stage-pages/src/pages/settings/modules/components/hearing-playground-transcripts.vue new file mode 100644 index 000000000..c0b55e183 --- /dev/null +++ b/packages/stage-pages/src/pages/settings/modules/components/hearing-playground-transcripts.vue @@ -0,0 +1,78 @@ + + + diff --git a/packages/stage-pages/src/pages/settings/modules/hearing.vue b/packages/stage-pages/src/pages/settings/modules/hearing.vue index 21d9dad84..da76cdfaf 100644 --- a/packages/stage-pages/src/pages/settings/modules/hearing.vue +++ b/packages/stage-pages/src/pages/settings/modules/hearing.vue @@ -1,21 +1,19 @@ @@ -709,7 +480,7 @@ onUnmounted(() => { {
-

+

- {{ t('settings.pages.providers.provider.elevenlabs.playground.title') }} + {{ t('settings.pages.modules.hearing.sections.section.playground.title') }}

- +

+ {{ t('settings.pages.modules.hearing.sections.section.playground.description') }} +

- -
-
-
-
+
@@ -854,7 +638,7 @@ onUnmounted(() => {
{
- - -
-

- Speech-to-Text Test -

-
- Test your transcription provider with the selected audio device. This will help verify that STT is working correctly. -
- -
-
-
- Please select a transcription provider above to test -
-
- -
-
-
- Please select an audio input device to test -
-
- -
-
- -
- - - -
-
-
-
- {{ testStatusMessage }} -
-
- -
-
-
- Streaming mode: Transcription will appear in real-time as you speak -
-
- -
-
- -
-
-
- Current transcription (streaming): -
-
- {{ testStreamingText }} -
-
-
-
- Final transcription: -
-
- {{ testTranscriptionText }} -
-
-
-
- No transcription yet. Click "Start Speech-to-Text Test" and speak into your microphone. -
-
- -
-
Provider: {{ configuredTranscriptionProvidersMetadata.find(p => p.id === activeTranscriptionProvider)?.localizedName || activeTranscriptionProvider }}
-
- Model: {{ activeTranscriptionModel }} -
-
Mode: {{ shouldUseStreamInput ? 'Streaming (real-time)' : 'Recording (file-based)' }}
-
-
-
-
diff --git a/packages/stage-ui/src/composables/index.ts b/packages/stage-ui/src/composables/index.ts index 82e27d8d9..da8539c27 100644 --- a/packages/stage-ui/src/composables/index.ts +++ b/packages/stage-ui/src/composables/index.ts @@ -9,6 +9,7 @@ export * from './use-async-state' export * from './use-breakpoints' export * from './use-build-info' export * from './use-chat-session/summary' +export * from './use-hearing-playground-segments' export * from './use-inference-preload' export * from './use-inference-status' export * from './use-lamp-flicker-animation' diff --git a/packages/stage-ui/src/composables/use-hearing-playground-segments.test.ts b/packages/stage-ui/src/composables/use-hearing-playground-segments.test.ts new file mode 100644 index 000000000..c04509689 --- /dev/null +++ b/packages/stage-ui/src/composables/use-hearing-playground-segments.test.ts @@ -0,0 +1,71 @@ +import { describe, expect, it } from 'vitest' + +import { useHearingPlaygroundSegments } from './use-hearing-playground-segments' + +describe('hearing playground segments', () => { + it('keeps empty audio and the next transcript in separate segments', () => { + const playground = useHearingPlaygroundSegments() + const emptyRecording = new Blob(['noise'], { type: 'audio/wav' }) + const speechRecording = new Blob(['speech'], { type: 'audio/wav' }) + + const emptyMetadata = playground.startRecording(emptyRecording) + playground.finishEmpty(emptyMetadata) + + const speechMetadata = playground.startRecording(speechRecording) + playground.finishRecording(speechMetadata, 'Second sentence.') + + expect(playground.segments.value).toHaveLength(2) + expect(playground.segments.value[0]).toMatchObject({ + recording: emptyRecording, + status: 'empty', + text: '', + }) + expect(playground.segments.value[1]).toMatchObject({ + recording: speechRecording, + status: 'complete', + text: 'Second sentence.', + }) + }) + + it('updates each segment by identity when results finish out of order', () => { + const playground = useHearingPlaygroundSegments() + const firstMetadata = playground.startRecording(new Blob(['first'])) + const secondMetadata = playground.startRecording(new Blob(['second'])) + + playground.finishRecording(secondMetadata, 'Second sentence.') + playground.finishRecording(firstMetadata, 'First sentence.') + + expect(playground.segments.value.map(segment => segment.text)).toEqual([ + 'First sentence.', + 'Second sentence.', + ]) + }) + + it('replaces a volatile streaming transcript when the provider corrects it', () => { + // ROOT CAUSE: + // + // Apple Speech sends complete volatile snapshots that can revise earlier characters. + // Appending each snapshot kept both the incorrect text and its correction. + // + // Before: "今天天气很号 今天天气很好" + // After: "今天天气很好" + const playground = useHearingPlaygroundSegments() + + playground.replaceStreamingText('今天天气很号') + playground.replaceStreamingText('今天天气很好') + + expect(playground.current.value).toBe('今天天气很好') + }) + + it('clears streaming and completed transcripts when the playground resets', () => { + const playground = useHearingPlaygroundSegments() + const metadata = playground.startRecording(new Blob(['speech'])) + + playground.finishRecording(metadata, 'First provider result.') + playground.replaceStreamingText('Second provider partial result') + playground.clear() + + expect(playground.current.value).toBe('') + expect(playground.segments.value).toEqual([]) + }) +}) diff --git a/packages/stage-ui/src/composables/use-hearing-playground-segments.ts b/packages/stage-ui/src/composables/use-hearing-playground-segments.ts new file mode 100644 index 000000000..d7daa34e6 --- /dev/null +++ b/packages/stage-ui/src/composables/use-hearing-playground-segments.ts @@ -0,0 +1,106 @@ +import { readonly, ref, shallowRef } from 'vue' + +/** Lifecycle state of one recorded Hearing playground segment. */ +export type HearingPlaygroundSegmentStatus = 'transcribing' | 'complete' | 'empty' | 'error' + +/** A recorded playground utterance and the transcription state that belongs to it. */ +export interface HearingPlaygroundSegment { + /** Stable identity used to correlate asynchronous provider results. */ + id: number + /** Audio sent to the provider. VAD-triggered recordings include retained speech padding. */ + recording?: Blob + /** Final provider text. Empty while the request is pending or produced no text. */ + text: string + /** User-facing failure details when transcription throws. */ + error?: string + /** Current transcription state for this segment. */ + status: HearingPlaygroundSegmentStatus +} + +interface HearingPlaygroundSegmentMetadata extends Record { + playgroundSegmentId: number +} + +function segmentIdFrom(metadata: Record | undefined): number | undefined { + const id = metadata?.playgroundSegmentId + return typeof id === 'number' ? id : undefined +} + +/** + * Keeps playground recordings and asynchronous transcription results correlated. + * Empty and failed results remain visible so later text cannot shift onto earlier audio. + */ +export function useHearingPlaygroundSegments() { + const current = shallowRef('') + const segments = ref([]) + let nextSegmentId = 0 + + function updateSegment( + metadata: Record | undefined, + update: (segment: HearingPlaygroundSegment) => HearingPlaygroundSegment, + ) { + const segmentId = segmentIdFrom(metadata) + if (segmentId === undefined) + return + + segments.value = segments.value.map(segment => segment.id === segmentId ? update(segment) : segment) + } + + function startRecording(recording: Blob): HearingPlaygroundSegmentMetadata { + const id = ++nextSegmentId + segments.value = [ + ...segments.value, + { id, recording, text: '', status: 'transcribing' }, + ] + return { playgroundSegmentId: id } + } + + function finishRecording(metadata: Record | undefined, text: string) { + const finalText = text.trim() + if (!finalText) { + finishEmpty(metadata) + return + } + + updateSegment(metadata, segment => ({ ...segment, text: finalText, status: 'complete' })) + } + + function finishEmpty(metadata: Record | undefined) { + updateSegment(metadata, segment => ({ ...segment, text: '', status: 'empty' })) + } + + function finishError(metadata: Record | undefined, error: string) { + updateSegment(metadata, segment => ({ ...segment, error, status: 'error' })) + } + + function replaceStreamingText(text: string) { + current.value = text.trim() + } + + function finishStreaming(text: string) { + const finalText = text.trim() || current.value.trim() + current.value = '' + if (!finalText) + return + + const id = ++nextSegmentId + segments.value = [...segments.value, { id, text: finalText, status: 'complete' }] + } + + function clear() { + current.value = '' + segments.value = [] + } + + return { + current: readonly(current), + segments: readonly(segments), + startRecording, + finishRecording, + finishEmpty, + finishError, + replaceStreamingText, + finishStreaming, + clear, + } +}