mirror of
https://github.com/moeru-ai/airi.git
synced 2026-08-14 00:48:06 +00:00
feat(stage-pages): add Hearing playground (#2265)
This commit is contained in:
Neko
@@ -686,6 +686,19 @@ pages:
|
||||
section:
|
||||
provider-selection:
|
||||
description: Select the suitable speech recognition provider
|
||||
playground:
|
||||
current: Current transcript
|
||||
description: Listen through the active Hearing pipeline and keep each completed speech segment below.
|
||||
empty: Start monitoring and speak into the selected microphone.
|
||||
error-title: Transcription error
|
||||
listening: Listening for speech…
|
||||
no-transcription: No speech was recognized in this audio.
|
||||
segment: 'Speech segment {number}'
|
||||
start: Start monitoring
|
||||
stop: Stop monitoring
|
||||
title: Hearing playground
|
||||
transcribing: Transcribing…
|
||||
transcription-failed: Transcription failed.
|
||||
confidence-threshold:
|
||||
title: Confidence Threshold
|
||||
description: >-
|
||||
|
||||
@@ -657,6 +657,19 @@ pages:
|
||||
section:
|
||||
provider-selection:
|
||||
description: 选择合适的语音转文本的服务来源
|
||||
playground:
|
||||
current: 当前转写
|
||||
description: 通过当前的听觉管线监听，并在下方保留每一段完成的语音转写。
|
||||
empty: 开始监听，然后对着所选麦克风说话。
|
||||
error-title: 转写错误
|
||||
listening: 正在等待语音……
|
||||
no-transcription: 此段音频中未识别到语音。
|
||||
segment: '语音片段 {number}'
|
||||
start: 开始监听
|
||||
stop: 停止监听
|
||||
title: 听觉测试面板
|
||||
transcribing: 正在转写……
|
||||
transcription-failed: 转写失败。
|
||||
confidence-threshold:
|
||||
title: 置信度阈值
|
||||
description: >-
|
||||
|
||||
+59
@@ -0,0 +1,59 @@
|
||||
<script setup lang="ts">
|
||||
import type { HearingPlaygroundSegment } from '@proj-airi/stage-ui/composables'
|
||||
|
||||
import { useObjectUrl } from '@vueuse/core'
|
||||
import { computed } from 'vue'
|
||||
import { useI18n } from 'vue-i18n'
|
||||
|
||||
const props = defineProps<{
|
||||
number: number
|
||||
segment: HearingPlaygroundSegment
|
||||
}>()
|
||||
|
||||
const { t } = useI18n()
|
||||
const recording = computed(() => props.segment.recording)
|
||||
const audioUrl = useObjectUrl(recording)
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<li
|
||||
:class="[
|
||||
'flex flex-col gap-3',
|
||||
'px-3 py-2',
|
||||
]"
|
||||
data-testid="hearing-playground-segment"
|
||||
:data-status="segment.status"
|
||||
>
|
||||
<audio
|
||||
v-if="audioUrl"
|
||||
:src="audioUrl"
|
||||
:aria-label="t('settings.pages.modules.hearing.sections.section.playground.segment', { number })"
|
||||
controls
|
||||
:class="['mb-2', 'w-full']"
|
||||
/>
|
||||
<div :class="['flex items-center gap-2', 'text-xs text-neutral-400 dark:text-neutral-500']">
|
||||
<span>{{ t('settings.pages.modules.hearing.sections.section.playground.segment', { number }) }}</span>
|
||||
<span
|
||||
v-if="segment.status === 'transcribing'"
|
||||
:class="['flex items-center gap-1', 'text-primary-500 dark:text-primary-400']"
|
||||
>
|
||||
<span class="animate-spin" i-solar:spinner-line-duotone />
|
||||
{{ t('settings.pages.modules.hearing.sections.section.playground.transcribing') }}
|
||||
</span>
|
||||
</div>
|
||||
|
||||
<p
|
||||
v-if="segment.status === 'complete'"
|
||||
:class="['whitespace-pre-wrap', 'text-sm text-neutral-700 dark:text-neutral-200']"
|
||||
data-testid="hearing-playground-transcript"
|
||||
>
|
||||
{{ segment.text }}
|
||||
</p>
|
||||
<p v-else-if="segment.status === 'empty'" :class="['text-sm text-neutral-400 italic', 'dark:text-neutral-500']">
|
||||
{{ t('settings.pages.modules.hearing.sections.section.playground.no-transcription') }}
|
||||
</p>
|
||||
<p v-else-if="segment.status === 'error'" :class="['text-sm text-red-500', 'dark:text-red-400']">
|
||||
{{ segment.error || t('settings.pages.modules.hearing.sections.section.playground.transcription-failed') }}
|
||||
</p>
|
||||
</li>
|
||||
</template>
|
||||
+78
@@ -0,0 +1,78 @@
|
||||
<script setup lang="ts">
|
||||
import type { HearingPlaygroundSegment } from '@proj-airi/stage-ui/composables'
|
||||
|
||||
import { computed } from 'vue'
|
||||
import { useI18n } from 'vue-i18n'
|
||||
|
||||
import HearingPlaygroundSegmentItem from './hearing-playground-segment.vue'
|
||||
|
||||
const props = defineProps<{
|
||||
current: string
|
||||
isMonitoring: boolean
|
||||
segments: readonly HearingPlaygroundSegment[]
|
||||
}>()
|
||||
|
||||
const { t } = useI18n()
|
||||
const reversedSegments = computed(() => props.segments.toReversed())
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<div
|
||||
:class="['mb-4 min-h-28', 'rounded-xl']"
|
||||
data-testid="hearing-playground-transcripts"
|
||||
aria-live="polite"
|
||||
>
|
||||
<ol
|
||||
v-if="current || segments.length"
|
||||
v-auto-animate
|
||||
:class="[
|
||||
'max-h-xs flex flex-col overflow-y-auto',
|
||||
'rounded-xl bg-neutral-100 dark:bg-neutral-900',
|
||||
]"
|
||||
>
|
||||
<li
|
||||
v-if="current"
|
||||
:class="[
|
||||
'rounded-lg px-3 py-2',
|
||||
'bg-primary-50 dark:bg-primary-900/20',
|
||||
]"
|
||||
data-testid="hearing-playground-current"
|
||||
>
|
||||
<div :class="['mb-1', 'text-xs text-primary-600 font-medium dark:text-primary-400']">
|
||||
{{ t('settings.pages.modules.hearing.sections.section.playground.current') }}
|
||||
</div>
|
||||
<p :class="['whitespace-pre-wrap', 'text-sm text-neutral-700 dark:text-neutral-200']">
|
||||
{{ current }}
|
||||
</p>
|
||||
</li>
|
||||
|
||||
<HearingPlaygroundSegmentItem
|
||||
v-for="(segment, index) in reversedSegments"
|
||||
:key="segment.id"
|
||||
:number="segments.length - index"
|
||||
:segment="segment"
|
||||
/>
|
||||
</ol>
|
||||
|
||||
<div
|
||||
v-else-if="isMonitoring"
|
||||
:class="[
|
||||
'min-h-20 flex items-center justify-center gap-2',
|
||||
'text-sm text-neutral-400 dark:text-neutral-500',
|
||||
]"
|
||||
>
|
||||
<div class="animate-pulse" i-solar:microphone-3-line-duotone />
|
||||
{{ t('settings.pages.modules.hearing.sections.section.playground.listening') }}
|
||||
</div>
|
||||
|
||||
<div
|
||||
v-else
|
||||
:class="[
|
||||
'min-h-20 flex items-center justify-center text-center',
|
||||
'text-sm text-neutral-400 dark:text-neutral-500',
|
||||
]"
|
||||
>
|
||||
{{ t('settings.pages.modules.hearing.sections.section.playground.empty') }}
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
@@ -1,21 +1,19 @@
|
||||
<script setup lang="ts">
|
||||
import workletUrl from '@proj-airi/stage-ui/workers/vad/process.worklet?worker&url'
|
||||
|
||||
import { errorMessageFromValue } from '@proj-airi/stage-shared'
|
||||
import { errorMessageFrom } from '@moeru/std'
|
||||
import { Alert, ErrorContainer, LevelMeter, RadioCardManySelect, RadioCardSimple, TestDummyMarker, ThresholdMeter, TimeSeriesChart } from '@proj-airi/stage-ui/components'
|
||||
import { useAnalytics, useAudioAnalyzer, useAudioRecorder, useVoiceInputSession } from '@proj-airi/stage-ui/composables'
|
||||
import { useVAD } from '@proj-airi/stage-ui/stores/ai/models/vad'
|
||||
import { useAnalytics, useAudioAnalyzer, useHearingPlaygroundSegments, useVoiceInputSession } from '@proj-airi/stage-ui/composables'
|
||||
import { useAudioContext } from '@proj-airi/stage-ui/stores/audio'
|
||||
import { CONFIDENCE_THRESHOLD_DISABLED, useHearingSpeechInputPipeline, useHearingStore } from '@proj-airi/stage-ui/stores/modules/hearing'
|
||||
import { useProviderConfigStore } from '@proj-airi/stage-ui/stores/providers/config'
|
||||
import { useProviderStore } from '@proj-airi/stage-ui/stores/providers/provider'
|
||||
import { useSettingsAudioDevice } from '@proj-airi/stage-ui/stores/settings'
|
||||
import { Button, FieldCheckbox, FieldCombobox, FieldInput, FieldRange } from '@proj-airi/ui'
|
||||
import { until } from '@vueuse/core'
|
||||
import { storeToRefs } from 'pinia'
|
||||
import { computed, onMounted, onUnmounted, ref, watch } from 'vue'
|
||||
import { computed, onMounted, onUnmounted, shallowRef, watch } from 'vue'
|
||||
import { useI18n } from 'vue-i18n'
|
||||
|
||||
import HearingPlaygroundTranscripts from './components/hearing-playground-transcripts.vue'
|
||||
|
||||
const { t } = useI18n()
|
||||
|
||||
const hearingStore = useHearingStore()
|
||||
@@ -41,172 +39,99 @@ const { trackProviderClick } = useAnalytics()
|
||||
const settingsAudioDeviceStore = useSettingsAudioDevice()
|
||||
const { askPermission, stopStream, startStream } = settingsAudioDeviceStore
|
||||
const { audioInputOptions, selectedAudioInput, stream } = storeToRefs(settingsAudioDeviceStore)
|
||||
const { discardRecord, startRecord, stopRecord, onStopRecord } = useAudioRecorder(stream)
|
||||
const { startAnalyzer, stopAnalyzer, onAnalyzerUpdate, volumeLevel } = useAudioAnalyzer()
|
||||
const { audioContext } = storeToRefs(useAudioContext())
|
||||
const hearingSpeechInputPipeline = useHearingSpeechInputPipeline()
|
||||
const {
|
||||
removeStreamingTranscriptionConsumer,
|
||||
transcribeForRecording,
|
||||
transcribeForMediaStream,
|
||||
removeStreamingTranscriptionConsumer,
|
||||
stopStreamingTranscription,
|
||||
} = hearingSpeechInputPipeline
|
||||
const {
|
||||
supportsStreamInput,
|
||||
error: transcriptionPipelineError,
|
||||
} = storeToRefs(hearingSpeechInputPipeline)
|
||||
const hearingPlaygroundTranscriptionConsumerId = 'hearing-playground'
|
||||
|
||||
/** Identifies monitoring callbacks in the shared streaming transcription session. */
|
||||
const monitoringTranscriptionConsumerId = 'hearing-settings:monitoring'
|
||||
/** Identifies test callbacks in the shared streaming transcription session. */
|
||||
const testTranscriptionConsumerId = 'hearing-settings:test'
|
||||
// This page owns one monitoring session. Setup can restart it after device or Provider changes,
|
||||
// and stop releases the recorder, Provider consumer, media stream, and analyzer in that order.
|
||||
let volumeSpeechEndTimer: ReturnType<typeof setTimeout> | undefined
|
||||
|
||||
const animationFrame = ref<number>()
|
||||
const error = shallowRef('')
|
||||
const isMonitoring = shallowRef(false)
|
||||
|
||||
const error = ref<string>('')
|
||||
const isMonitoring = ref(false)
|
||||
const {
|
||||
current: currentTranscription,
|
||||
segments: playgroundSegments,
|
||||
replaceStreamingText,
|
||||
finishStreaming,
|
||||
startRecording,
|
||||
finishRecording,
|
||||
finishEmpty,
|
||||
finishError,
|
||||
clear: clearPlaygroundSegments,
|
||||
} = useHearingPlaygroundSegments()
|
||||
|
||||
const transcriptions = ref<string[]>([])
|
||||
const audios = ref<Blob[]>([])
|
||||
const audioCleanups = ref<(() => void)[]>([])
|
||||
const audioURLs = computed(() => {
|
||||
return audios.value.map((blob) => {
|
||||
const url = URL.createObjectURL(blob)
|
||||
audioCleanups.value.push(() => URL.revokeObjectURL(url))
|
||||
return url
|
||||
const useVADThreshold = shallowRef(0.6) // 0.1 - 0.9
|
||||
const useVolumeThreshold = shallowRef(10) // 1 - 80
|
||||
const useVADMinSilenceDurationMs = shallowRef(800)
|
||||
const useVADModel = shallowRef(true) // Toggle between VAD and volume-based detection
|
||||
const shouldUseStreamInput = computed(() => supportsStreamInput.value && !!stream.value)
|
||||
const sortedProviderModels = computed(() => {
|
||||
return providerModels.value.toSorted((left, right) => {
|
||||
if (left.id === activeTranscriptionModel.value)
|
||||
return -1
|
||||
if (right.id === activeTranscriptionModel.value)
|
||||
return 1
|
||||
return 0
|
||||
})
|
||||
})
|
||||
|
||||
// Speech-to-Text test state
|
||||
const isTestingSTT = ref(false)
|
||||
const testTranscriptionText = ref<string>('')
|
||||
const testTranscriptionError = ref<string>('')
|
||||
const isTranscribing = ref(false)
|
||||
const testStreamingText = ref<string>('')
|
||||
const testStatusMessage = ref<string>('')
|
||||
const testStreamWasStarted = ref(false) // Track if we started the stream for testing
|
||||
|
||||
const useVADThreshold = ref(0.6) // 0.1 - 0.9
|
||||
const useVADMinSilenceDurationMs = ref(800)
|
||||
const useVADModel = ref(true) // Toggle between VAD and volume-based detection
|
||||
const shouldUseStreamInput = computed(() => supportsStreamInput.value && !!stream.value)
|
||||
let sttTestStopTimer: ReturnType<typeof setTimeout> | undefined
|
||||
|
||||
const sttTestVoiceInputSession = useVoiceInputSession(stream, {
|
||||
shouldUseStreamInput,
|
||||
// Manual Settings tests own their 3s recording window; automatic volume segmentation
|
||||
// would race that timer and make provider diagnostics harder to interpret.
|
||||
volumeFallback: {
|
||||
enabled: false,
|
||||
},
|
||||
onSegmentStart: () => {
|
||||
testStatusMessage.value = 'Recording audio for transcription... (3 seconds)'
|
||||
},
|
||||
onTranscriptionStart: () => {
|
||||
testStatusMessage.value = 'Transcribing recording...'
|
||||
isTranscribing.value = true
|
||||
},
|
||||
onTranscriptionResult: ({ text }) => {
|
||||
testTranscriptionText.value = text
|
||||
testStatusMessage.value = 'Transcription complete!'
|
||||
isTranscribing.value = false
|
||||
isTestingSTT.value = false
|
||||
console.info('STT test transcription result:', text)
|
||||
},
|
||||
onTranscriptionEmpty: () => {
|
||||
testTranscriptionError.value = transcriptionPipelineError.value || 'No transcription result returned from provider'
|
||||
testStatusMessage.value = 'Transcription failed'
|
||||
isTranscribing.value = false
|
||||
isTestingSTT.value = false
|
||||
},
|
||||
onRecordingSkipped: ({ gate }) => {
|
||||
testTranscriptionError.value = gate?.reason || transcriptionPipelineError.value || 'No recording captured from microphone'
|
||||
testStatusMessage.value = 'Transcription failed'
|
||||
isTranscribing.value = false
|
||||
isTestingSTT.value = false
|
||||
},
|
||||
onTranscriptionError: ({ error }) => {
|
||||
testTranscriptionError.value = errorMessageFromValue(error)
|
||||
testStatusMessage.value = `Error: ${testTranscriptionError.value}`
|
||||
isTranscribing.value = false
|
||||
isTestingSTT.value = false
|
||||
console.error('STT test transcription error:', error)
|
||||
},
|
||||
})
|
||||
|
||||
async function resetSttTestVoiceInputSession() {
|
||||
if (sttTestStopTimer) {
|
||||
clearTimeout(sttTestStopTimer)
|
||||
sttTestStopTimer = undefined
|
||||
}
|
||||
|
||||
await sttTestVoiceInputSession.stop({ flushActiveRecording: false })
|
||||
}
|
||||
|
||||
function formatVADThreshold(value: number) {
|
||||
return value.toFixed(2)
|
||||
}
|
||||
|
||||
async function handleSpeechStart() {
|
||||
if (isTestingSTT.value)
|
||||
return
|
||||
|
||||
if (shouldUseStreamInput.value && stream.value) {
|
||||
// Use both callbacks to support incremental updates and final transcript replacement.
|
||||
// ChatArea uses only onSentenceEnd to avoid re-adding deleted text.
|
||||
await transcribeForMediaStream(stream.value, {
|
||||
consumerId: monitoringTranscriptionConsumerId,
|
||||
onSentenceEnd: (delta) => {
|
||||
transcriptions.value.push(delta)
|
||||
},
|
||||
onSpeechEnd: (text) => {
|
||||
transcriptions.value = [text]
|
||||
},
|
||||
})
|
||||
return
|
||||
}
|
||||
|
||||
startRecord()
|
||||
}
|
||||
|
||||
async function handleSpeechEnd() {
|
||||
if (isTestingSTT.value)
|
||||
return
|
||||
|
||||
if (shouldUseStreamInput.value) {
|
||||
// For streaming providers, keep the session alive; idle timer will handle teardown.
|
||||
return
|
||||
}
|
||||
|
||||
stopRecord()
|
||||
}
|
||||
|
||||
const {
|
||||
init: initVAD,
|
||||
dispose: disposeVAD,
|
||||
isSpeech: isSpeechVAD,
|
||||
isSpeechVAD,
|
||||
isSpeechProb,
|
||||
isSpeechHistory,
|
||||
inferenceError: vadModelError,
|
||||
start: startVAD,
|
||||
loaded: loadedVAD,
|
||||
loading: loadingVAD,
|
||||
} = useVAD(workletUrl, {
|
||||
threshold: useVADThreshold,
|
||||
minSilenceDurationMs: useVADMinSilenceDurationMs,
|
||||
onSpeechStart: () => {
|
||||
void handleSpeechStart()
|
||||
vadError: vadModelError,
|
||||
vadLoaded: loadedVAD,
|
||||
vadLoading: loadingVAD,
|
||||
startSegment,
|
||||
stopSegment,
|
||||
startAutoSegmentation,
|
||||
stop: stopVoiceInputSession,
|
||||
} = useVoiceInputSession(stream, {
|
||||
shouldUseStreamInput,
|
||||
vad: {
|
||||
threshold: useVADThreshold,
|
||||
minSilenceDurationMs: useVADMinSilenceDurationMs,
|
||||
},
|
||||
onSpeechEnd: () => {
|
||||
void handleSpeechEnd()
|
||||
volumeFallback: {
|
||||
enabled: false,
|
||||
},
|
||||
onSpeechCancel: () => {
|
||||
if (!isTestingSTT.value && !shouldUseStreamInput.value)
|
||||
void discardRecord()
|
||||
onRecordingReady: ({ recording }) => {
|
||||
if (!recording)
|
||||
return
|
||||
|
||||
return startRecording(recording)
|
||||
},
|
||||
onTranscriptionResult: ({ metadata, text }) => {
|
||||
finishRecording(metadata, text)
|
||||
error.value = ''
|
||||
},
|
||||
onTranscriptionEmpty: ({ metadata }) => {
|
||||
finishEmpty(metadata)
|
||||
},
|
||||
onTranscriptionError: ({ metadata, error: cause }) => {
|
||||
const message = errorMessageFrom(cause) ?? t('settings.pages.modules.hearing.sections.section.playground.transcription-failed')
|
||||
finishError(metadata, message)
|
||||
error.value = message
|
||||
},
|
||||
})
|
||||
|
||||
const isSpeechVolume = ref(false) // Volume-based speaking detection
|
||||
const isSpeechVolume = shallowRef(false) // Volume-based speaking detection
|
||||
const isSpeech = computed(() => {
|
||||
if (useVADModel.value && loadedVAD.value) {
|
||||
return isSpeechVAD.value
|
||||
@@ -219,7 +144,7 @@ async function setupAudioMonitoring() {
|
||||
try {
|
||||
if (!selectedAudioInput.value) {
|
||||
console.warn('No audio input device selected')
|
||||
return
|
||||
return false
|
||||
}
|
||||
|
||||
await stopAudioMonitoring()
|
||||
@@ -227,7 +152,17 @@ async function setupAudioMonitoring() {
|
||||
await startStream()
|
||||
if (!stream.value) {
|
||||
console.warn('No audio stream available')
|
||||
return
|
||||
return false
|
||||
}
|
||||
|
||||
if (supportsStreamInput.value) {
|
||||
// The Hearing pipeline owns speech segmentation and the provider session.
|
||||
// The page VAD below only drives the visualization for streaming providers.
|
||||
await transcribeForMediaStream(stream.value, {
|
||||
consumerId: hearingPlaygroundTranscriptionConsumerId,
|
||||
onSpeechEnd: finishStreaming,
|
||||
onTranscriptionUpdate: replaceStreamingText,
|
||||
})
|
||||
}
|
||||
|
||||
const source = audioContext.value.createMediaStreamSource(stream.value)
|
||||
@@ -236,44 +171,46 @@ async function setupAudioMonitoring() {
|
||||
const analyzer = startAnalyzer(audioContext.value)
|
||||
onAnalyzerUpdate((volumeLevel) => {
|
||||
if (!useVADModel.value || !loadedVAD.value) {
|
||||
isSpeechVolume.value = volumeLevel > useVADThreshold.value
|
||||
isSpeechVolume.value = volumeLevel > useVolumeThreshold.value
|
||||
}
|
||||
})
|
||||
if (analyzer)
|
||||
source.connect(analyzer)
|
||||
|
||||
if (useVADModel.value) {
|
||||
await initVAD()
|
||||
await startVAD(stream.value)
|
||||
await startAutoSegmentation()
|
||||
}
|
||||
|
||||
error.value = ''
|
||||
return true
|
||||
}
|
||||
catch (error) {
|
||||
console.error('Error setting up audio monitoring:', error)
|
||||
vadModelError.value = errorMessageFromValue(error)
|
||||
catch (cause) {
|
||||
console.error('Error setting up audio monitoring:', cause)
|
||||
error.value = errorMessageFrom(cause) ?? t('settings.pages.modules.hearing.sections.section.playground.transcription-failed')
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
async function stopAudioMonitoring() {
|
||||
if (animationFrame.value) { // Stop animation frame
|
||||
cancelAnimationFrame(animationFrame.value)
|
||||
animationFrame.value = undefined
|
||||
async function stopAudioMonitoring(disposeProviderId = activeTranscriptionProvider.value) {
|
||||
if (volumeSpeechEndTimer) {
|
||||
clearTimeout(volumeSpeechEndTimer)
|
||||
volumeSpeechEndTimer = undefined
|
||||
}
|
||||
|
||||
removeStreamingTranscriptionConsumer(monitoringTranscriptionConsumerId)
|
||||
await stopStreamingTranscription(true, activeTranscriptionProvider.value)
|
||||
await stopVoiceInputSession({ flushActiveRecording: false })
|
||||
removeStreamingTranscriptionConsumer(hearingPlaygroundTranscriptionConsumerId)
|
||||
await stopStreamingTranscription(true, disposeProviderId)
|
||||
if (stream.value) { // Stop media stream
|
||||
stopStream()
|
||||
}
|
||||
|
||||
stopAnalyzer()
|
||||
disposeVAD()
|
||||
}
|
||||
|
||||
// Monitoring toggle
|
||||
async function toggleMonitoring() {
|
||||
if (!isMonitoring.value) {
|
||||
await setupAudioMonitoring()
|
||||
isMonitoring.value = true
|
||||
isMonitoring.value = await setupAudioMonitoring()
|
||||
}
|
||||
else {
|
||||
await stopAudioMonitoring()
|
||||
@@ -333,210 +270,56 @@ function syncOpenAICompatibleSettings() {
|
||||
}
|
||||
}
|
||||
|
||||
onStopRecord(async (recording) => {
|
||||
if (shouldUseStreamInput.value)
|
||||
watch([selectedAudioInput, useVADModel], async () => {
|
||||
if (!isMonitoring.value)
|
||||
return
|
||||
|
||||
if (isTestingSTT.value)
|
||||
return
|
||||
|
||||
if (!recording || recording.size === 0)
|
||||
return
|
||||
|
||||
// Normal monitoring mode - add to audios and transcribe
|
||||
audios.value.push(recording)
|
||||
|
||||
const res = await transcribeForRecording(recording)
|
||||
|
||||
if (res) {
|
||||
transcriptions.value.push(res)
|
||||
error.value = ''
|
||||
}
|
||||
else if (transcriptionPipelineError.value) {
|
||||
error.value = transcriptionPipelineError.value
|
||||
}
|
||||
isMonitoring.value = await setupAudioMonitoring()
|
||||
})
|
||||
|
||||
// Speech-to-Text test functions
|
||||
async function startSTTTest() {
|
||||
if (!activeTranscriptionProvider.value) {
|
||||
testTranscriptionError.value = 'Please select a transcription provider first'
|
||||
watch(isSpeechVolume, (speaking) => {
|
||||
if (useVADModel.value)
|
||||
return
|
||||
|
||||
if (volumeSpeechEndTimer) {
|
||||
clearTimeout(volumeSpeechEndTimer)
|
||||
volumeSpeechEndTimer = undefined
|
||||
}
|
||||
|
||||
if (speaking) {
|
||||
void startSegment('volume')
|
||||
return
|
||||
}
|
||||
|
||||
if (!selectedAudioInput.value) {
|
||||
testTranscriptionError.value = 'Please select an audio input device first'
|
||||
return
|
||||
volumeSpeechEndTimer = setTimeout(() => {
|
||||
volumeSpeechEndTimer = undefined
|
||||
if (!useVADModel.value && !isSpeechVolume.value)
|
||||
void stopSegment('volume')
|
||||
}, useVADMinSilenceDurationMs.value)
|
||||
})
|
||||
|
||||
watch(activeTranscriptionProvider, async (provider, previousProvider) => {
|
||||
const shouldRestartMonitoring = isMonitoring.value
|
||||
|
||||
if (shouldRestartMonitoring) {
|
||||
isMonitoring.value = false
|
||||
await stopAudioMonitoring(previousProvider)
|
||||
}
|
||||
|
||||
testTranscriptionError.value = ''
|
||||
testTranscriptionText.value = ''
|
||||
testStreamingText.value = ''
|
||||
testStatusMessage.value = ''
|
||||
error.value = ''
|
||||
isTestingSTT.value = true
|
||||
isTranscribing.value = true
|
||||
clearPlaygroundSegments()
|
||||
|
||||
try {
|
||||
// Ensure audio stream is available
|
||||
if (!stream.value) {
|
||||
testStatusMessage.value = 'Starting audio stream...'
|
||||
testStreamWasStarted.value = true
|
||||
await startStream()
|
||||
|
||||
// Wait for the stream to become available with a 3-second timeout.
|
||||
try {
|
||||
await until(stream).toBeTruthy({ timeout: 3000, throwOnTimeout: true })
|
||||
}
|
||||
catch {
|
||||
handleStreamStartError()
|
||||
return
|
||||
}
|
||||
|
||||
// Type guard: until guarantees stream.value is truthy, but TypeScript doesn't know this
|
||||
if (!stream.value) {
|
||||
handleStreamStartError()
|
||||
return
|
||||
}
|
||||
}
|
||||
else {
|
||||
testStreamWasStarted.value = false // Stream was already running
|
||||
}
|
||||
|
||||
// Check if provider supports streaming input
|
||||
if (shouldUseStreamInput.value && stream.value) {
|
||||
testStatusMessage.value = 'Starting streaming transcription...'
|
||||
console.info('Starting STT test with streaming input for provider:', activeTranscriptionProvider.value)
|
||||
|
||||
await transcribeForMediaStream(stream.value, {
|
||||
consumerId: testTranscriptionConsumerId,
|
||||
onSentenceEnd: (delta) => {
|
||||
if (delta && delta.trim()) {
|
||||
testStreamingText.value += `${delta} `
|
||||
testStatusMessage.value = 'Transcribing... (streaming)'
|
||||
isTranscribing.value = true
|
||||
console.info('STT test received sentence:', delta)
|
||||
}
|
||||
},
|
||||
onSpeechEnd: (text) => {
|
||||
if (text) {
|
||||
testTranscriptionText.value = text
|
||||
testStreamingText.value = ''
|
||||
testStatusMessage.value = 'Transcription complete!'
|
||||
isTranscribing.value = false
|
||||
console.info('STT test completed with text:', text)
|
||||
}
|
||||
else {
|
||||
testStatusMessage.value = 'Waiting for speech...'
|
||||
isTranscribing.value = false
|
||||
}
|
||||
},
|
||||
})
|
||||
|
||||
testStatusMessage.value = 'Listening for speech... (streaming mode active)'
|
||||
isTranscribing.value = false // Not actively transcribing yet, just listening
|
||||
}
|
||||
else {
|
||||
// Fallback to recording-based transcription
|
||||
testStatusMessage.value = 'Recording audio for transcription... (3 seconds)'
|
||||
console.info('Starting STT test with recording-based transcription for provider:', activeTranscriptionProvider.value)
|
||||
|
||||
const recordingStarted = await sttTestVoiceInputSession.startSegment('manual')
|
||||
if (!recordingStarted) {
|
||||
if (!testTranscriptionError.value)
|
||||
testStatusMessage.value = 'Recording did not start'
|
||||
isTranscribing.value = false
|
||||
isTestingSTT.value = false
|
||||
return
|
||||
}
|
||||
|
||||
// Wait a bit for recording to start, then stop it after a delay
|
||||
sttTestStopTimer = setTimeout(async () => {
|
||||
sttTestStopTimer = undefined
|
||||
testStatusMessage.value = 'Processing transcription...'
|
||||
try {
|
||||
await sttTestVoiceInputSession.stopSegment('manual')
|
||||
}
|
||||
catch (err) {
|
||||
testTranscriptionError.value = errorMessageFromValue(err)
|
||||
testStatusMessage.value = `Error: ${testTranscriptionError.value}`
|
||||
isTranscribing.value = false
|
||||
isTestingSTT.value = false
|
||||
console.error('STT test stop timer error:', err)
|
||||
}
|
||||
}, 3000) // Record for 3 seconds
|
||||
}
|
||||
}
|
||||
catch (err) {
|
||||
testTranscriptionError.value = errorMessageFromValue(err)
|
||||
testStatusMessage.value = `Error: ${testTranscriptionError.value}`
|
||||
isTranscribing.value = false
|
||||
isTestingSTT.value = false
|
||||
console.error('STT test error:', err)
|
||||
}
|
||||
}
|
||||
|
||||
async function stopSTTTest() {
|
||||
isTestingSTT.value = false
|
||||
isTranscribing.value = false
|
||||
testStatusMessage.value = 'Stopped'
|
||||
removeStreamingTranscriptionConsumer(testTranscriptionConsumerId)
|
||||
|
||||
try {
|
||||
// Stop streaming transcription if active
|
||||
if (shouldUseStreamInput.value) {
|
||||
await stopStreamingTranscription(false, activeTranscriptionProvider.value)
|
||||
}
|
||||
else {
|
||||
await resetSttTestVoiceInputSession()
|
||||
}
|
||||
}
|
||||
catch (err) {
|
||||
console.error('Error stopping STT test:', err)
|
||||
}
|
||||
|
||||
// Finalize transcription if we have streaming text
|
||||
if (testStreamingText.value.trim() && !testTranscriptionText.value) {
|
||||
testTranscriptionText.value = testStreamingText.value.trim()
|
||||
}
|
||||
|
||||
// Stop the stream if we started it for testing (and monitoring is not active)
|
||||
if (testStreamWasStarted.value && !isMonitoring.value) {
|
||||
try {
|
||||
stopStream()
|
||||
testStreamWasStarted.value = false
|
||||
}
|
||||
catch (err) {
|
||||
console.error('Error stopping test stream:', err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
watch(selectedAudioInput, async () => isMonitoring.value && await setupAudioMonitoring())
|
||||
|
||||
function handleStreamStartError() {
|
||||
testTranscriptionError.value = 'Failed to start audio stream. Please check microphone permissions.'
|
||||
testStatusMessage.value = 'Error: Failed to start audio stream'
|
||||
isTranscribing.value = false
|
||||
isTestingSTT.value = false
|
||||
testStreamWasStarted.value = false
|
||||
}
|
||||
|
||||
watch(activeTranscriptionProvider, async (provider) => {
|
||||
if (!provider)
|
||||
return
|
||||
|
||||
await hearingStore.loadModelsForProvider(provider)
|
||||
syncOpenAICompatibleSettings()
|
||||
|
||||
// Auto-select first model for Web Speech API if no model is selected
|
||||
if (provider === 'browser-web-speech-api' && !activeTranscriptionModel.value) {
|
||||
const models = providerModels.value
|
||||
if (models.length > 0) {
|
||||
activeTranscriptionModel.value = models[0].id
|
||||
console.info('Auto-selected Web Speech API model:', models[0].id)
|
||||
}
|
||||
}
|
||||
const models = providerModels.value
|
||||
if (models.length > 0 && !models.some(model => model.id === activeTranscriptionModel.value))
|
||||
activeTranscriptionModel.value = models[0].id
|
||||
|
||||
if (shouldRestartMonitoring)
|
||||
isMonitoring.value = await setupAudioMonitoring()
|
||||
}, { immediate: true })
|
||||
|
||||
onMounted(async () => {
|
||||
@@ -545,19 +328,7 @@ onMounted(async () => {
|
||||
})
|
||||
|
||||
onUnmounted(() => {
|
||||
stopSTTTest()
|
||||
stopAudioMonitoring()
|
||||
disposeVAD()
|
||||
|
||||
// Clean up any active transcription sessions when leaving the page
|
||||
// This prevents stale sessions from interfering with other pages
|
||||
if (shouldUseStreamInput.value) {
|
||||
stopStreamingTranscription(true, activeTranscriptionProvider.value).catch((err) => {
|
||||
console.warn('[Hearing Module] Error cleaning up transcription session on unmount:', err)
|
||||
})
|
||||
}
|
||||
|
||||
audioCleanups.value.forEach(cleanup => cleanup())
|
||||
void stopAudioMonitoring().catch(cause => console.warn('[Hearing Module] Failed to stop playground monitoring:', cause))
|
||||
})
|
||||
</script>
|
||||
|
||||
@@ -709,7 +480,7 @@ onUnmounted(() => {
|
||||
<RadioCardManySelect
|
||||
v-model="activeTranscriptionModel"
|
||||
v-model:search-query="transcriptionModelSearchQuery"
|
||||
:items="providerModels.sort((a, b) => a.id === activeTranscriptionModel ? -1 : b.id === activeTranscriptionModel ? 1 : 0)"
|
||||
:items="sortedProviderModels"
|
||||
:searchable="true"
|
||||
:search-placeholder="t('settings.pages.modules.consciousness.sections.section.provider-model-selection.search_placeholder')"
|
||||
:search-no-results-title="t('settings.pages.modules.consciousness.sections.section.provider-model-selection.no_search_results')"
|
||||
@@ -787,29 +558,42 @@ onUnmounted(() => {
|
||||
<div flex="~ col gap-6" class="w-full md:w-[60%]">
|
||||
<!-- Audio Monitoring Section -->
|
||||
<div w-full rounded-xl>
|
||||
<h2 class="mb-4 text-lg text-neutral-500 md:text-2xl dark:text-neutral-400" w-full>
|
||||
<h2 :class="['mb-4', 'text-lg text-neutral-500 md:text-2xl dark:text-neutral-400']" w-full>
|
||||
<div class="inline-flex items-center gap-4">
|
||||
<TestDummyMarker />
|
||||
<div>
|
||||
{{ t('settings.pages.providers.provider.elevenlabs.playground.title') }}
|
||||
{{ t('settings.pages.modules.hearing.sections.section.playground.title') }}
|
||||
</div>
|
||||
</div>
|
||||
</h2>
|
||||
|
||||
<ErrorContainer v-if="error" title="Error occurred" :error="error" mb-4 />
|
||||
<p :class="['mb-4', 'text-sm text-neutral-400 dark:text-neutral-500']">
|
||||
{{ t('settings.pages.modules.hearing.sections.section.playground.description') }}
|
||||
</p>
|
||||
|
||||
<Button class="mb-4" w-full @click="toggleMonitoring">
|
||||
{{ isMonitoring ? 'Stop Monitoring' : 'Start Monitoring' }}
|
||||
<ErrorContainer
|
||||
v-if="error || transcriptionPipelineError"
|
||||
:title="t('settings.pages.modules.hearing.sections.section.playground.error-title')"
|
||||
:error="error || transcriptionPipelineError"
|
||||
class="mb-4"
|
||||
/>
|
||||
|
||||
<Button
|
||||
:class="['mb-4', 'w-full']"
|
||||
data-testid="hearing-playground-monitor-toggle"
|
||||
:disabled="!activeTranscriptionProvider || !selectedAudioInput"
|
||||
@click="toggleMonitoring"
|
||||
>
|
||||
{{ isMonitoring
|
||||
? t('settings.pages.modules.hearing.sections.section.playground.stop')
|
||||
: t('settings.pages.modules.hearing.sections.section.playground.start') }}
|
||||
</Button>
|
||||
|
||||
<div>
|
||||
<div v-for="(transcription, index) in transcriptions" :key="index" class="mb-2">
|
||||
<audio v-if="audioURLs[index]" :src="audioURLs[index]" controls class="w-full" />
|
||||
<div v-if="transcription" class="mt-2 text-sm text-neutral-500 dark:text-neutral-400">
|
||||
{{ transcription }}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<HearingPlaygroundTranscripts
|
||||
:current="currentTranscription"
|
||||
:is-monitoring="isMonitoring"
|
||||
:segments="playgroundSegments"
|
||||
/>
|
||||
|
||||
<div flex="~ col gap-4">
|
||||
<div class="space-y-4">
|
||||
@@ -854,7 +638,7 @@ onUnmounted(() => {
|
||||
|
||||
<div v-else class="space-y-3">
|
||||
<FieldRange
|
||||
v-model="useVADThreshold"
|
||||
v-model="useVolumeThreshold"
|
||||
label="Sensitivity"
|
||||
description="Adjust the threshold for speech detection"
|
||||
:min="1"
|
||||
@@ -928,111 +712,6 @@ onUnmounted(() => {
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Speech-to-Text Test Section -->
|
||||
<div w-full rounded-xl bg="neutral-50 dark:[rgba(0,0,0,0.3)]" p-4 flex="~ col gap-4">
|
||||
<h2 class="text-lg text-neutral-500 md:text-2xl dark:text-neutral-400">
|
||||
Speech-to-Text Test
|
||||
</h2>
|
||||
<div text="sm neutral-400 dark:neutral-500" mb-2>
|
||||
Test your transcription provider with the selected audio device. This will help verify that STT is working correctly.
|
||||
</div>
|
||||
|
||||
<div v-if="!activeTranscriptionProvider" class="border border-amber-200 rounded-lg bg-amber-50 p-3 dark:border-amber-800 dark:bg-amber-900/20">
|
||||
<div class="flex items-center gap-2 text-amber-700 dark:text-amber-400">
|
||||
<div i-solar:warning-circle-line-duotone class="text-lg" />
|
||||
<span class="text-sm font-medium">Please select a transcription provider above to test</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div v-else-if="!selectedAudioInput" class="border border-amber-200 rounded-lg bg-amber-50 p-3 dark:border-amber-800 dark:bg-amber-900/20">
|
||||
<div class="flex items-center gap-2 text-amber-700 dark:text-amber-400">
|
||||
<div i-solar:warning-circle-line-duotone class="text-lg" />
|
||||
<span class="text-sm font-medium">Please select an audio input device to test</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div v-else class="flex flex-col gap-4">
|
||||
<div class="flex items-center gap-2">
|
||||
<Button
|
||||
:disabled="isTranscribing && !isTestingSTT"
|
||||
class="flex-1"
|
||||
@click="isTestingSTT ? stopSTTTest() : startSTTTest()"
|
||||
>
|
||||
<div v-if="isTranscribing" class="mr-2 animate-spin">
|
||||
<div i-solar:spinner-line-duotone text-lg />
|
||||
</div>
|
||||
<div v-else-if="isTestingSTT" class="mr-2">
|
||||
<div i-solar:stop-circle-line-duotone text-lg />
|
||||
</div>
|
||||
<div v-else class="mr-2">
|
||||
<div i-solar:microphone-line-duotone text-lg />
|
||||
</div>
|
||||
{{ isTestingSTT ? 'Stop Test' : isTranscribing ? 'Transcribing...' : 'Start Speech-to-Text Test' }}
|
||||
</Button>
|
||||
</div>
|
||||
|
||||
<ErrorContainer v-if="testTranscriptionError" title="Transcription Error" :error="testTranscriptionError" />
|
||||
|
||||
<div v-if="testStatusMessage" class="border border-primary-200 rounded-lg bg-primary-50 p-3 dark:border-primary-800 dark:bg-primary-900/20">
|
||||
<div class="flex items-center gap-2 text-primary-700 dark:text-primary-400">
|
||||
<div v-if="isTranscribing" class="animate-spin text-sm" i-solar:spinner-line-duotone />
|
||||
<div v-else class="text-sm" i-solar:info-circle-line-duotone />
|
||||
<span class="text-sm font-medium">{{ testStatusMessage }}</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div v-if="shouldUseStreamInput" class="border border-blue-200 rounded-lg bg-blue-50 p-3 dark:border-blue-800 dark:bg-blue-900/20">
|
||||
<div class="flex items-center gap-2 text-blue-700 dark:text-blue-400">
|
||||
<div i-solar:info-circle-line-duotone class="text-sm" />
|
||||
<span class="text-xs">Streaming mode: Transcription will appear in real-time as you speak</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="space-y-3">
|
||||
<div>
|
||||
<label class="mb-1 block text-sm text-neutral-700 font-medium dark:text-neutral-300">
|
||||
Transcription Result
|
||||
</label>
|
||||
<div
|
||||
v-if="testTranscriptionText || testStreamingText"
|
||||
class="min-h-[100px] border border-neutral-200 rounded-lg bg-white p-3 text-sm dark:border-neutral-700 dark:bg-neutral-900"
|
||||
>
|
||||
<div v-if="testStreamingText && shouldUseStreamInput" class="text-neutral-600 dark:text-neutral-400">
|
||||
<div class="mb-2 font-medium">
|
||||
Current transcription (streaming):
|
||||
</div>
|
||||
<div class="whitespace-pre-wrap">
|
||||
{{ testStreamingText }}
|
||||
</div>
|
||||
</div>
|
||||
<div v-if="testTranscriptionText" class="text-neutral-700 dark:text-neutral-200">
|
||||
<div v-if="testStreamingText && shouldUseStreamInput" class="mb-2 mt-3 border-t border-neutral-200 pt-2 font-medium dark:border-neutral-700">
|
||||
Final transcription:
|
||||
</div>
|
||||
<div class="whitespace-pre-wrap">
|
||||
{{ testTranscriptionText }}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div
|
||||
v-else
|
||||
class="min-h-[100px] border border-neutral-300 rounded-lg border-dashed bg-neutral-50 p-3 text-sm text-neutral-400 dark:border-neutral-700 dark:bg-neutral-900/50 dark:text-neutral-500"
|
||||
>
|
||||
No transcription yet. Click "Start Speech-to-Text Test" and speak into your microphone.
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div v-if="activeTranscriptionProvider" class="text-xs text-neutral-500 dark:text-neutral-400">
|
||||
<div>Provider: <span class="font-medium">{{ configuredTranscriptionProvidersMetadata.find(p => p.id === activeTranscriptionProvider)?.localizedName || activeTranscriptionProvider }}</span></div>
|
||||
<div v-if="activeTranscriptionModel">
|
||||
Model: <span class="font-medium">{{ activeTranscriptionModel }}</span>
|
||||
</div>
|
||||
<div>Mode: <span class="font-medium">{{ shouldUseStreamInput ? 'Streaming (real-time)' : 'Recording (file-based)' }}</span></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
@@ -9,6 +9,7 @@ export * from './use-async-state'
|
||||
export * from './use-breakpoints'
|
||||
export * from './use-build-info'
|
||||
export * from './use-chat-session/summary'
|
||||
export * from './use-hearing-playground-segments'
|
||||
export * from './use-inference-preload'
|
||||
export * from './use-inference-status'
|
||||
export * from './use-lamp-flicker-animation'
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
|
||||
import { useHearingPlaygroundSegments } from './use-hearing-playground-segments'
|
||||
|
||||
describe('hearing playground segments', () => {
|
||||
it('keeps empty audio and the next transcript in separate segments', () => {
|
||||
const playground = useHearingPlaygroundSegments()
|
||||
const emptyRecording = new Blob(['noise'], { type: 'audio/wav' })
|
||||
const speechRecording = new Blob(['speech'], { type: 'audio/wav' })
|
||||
|
||||
const emptyMetadata = playground.startRecording(emptyRecording)
|
||||
playground.finishEmpty(emptyMetadata)
|
||||
|
||||
const speechMetadata = playground.startRecording(speechRecording)
|
||||
playground.finishRecording(speechMetadata, 'Second sentence.')
|
||||
|
||||
expect(playground.segments.value).toHaveLength(2)
|
||||
expect(playground.segments.value[0]).toMatchObject({
|
||||
recording: emptyRecording,
|
||||
status: 'empty',
|
||||
text: '',
|
||||
})
|
||||
expect(playground.segments.value[1]).toMatchObject({
|
||||
recording: speechRecording,
|
||||
status: 'complete',
|
||||
text: 'Second sentence.',
|
||||
})
|
||||
})
|
||||
|
||||
it('updates each segment by identity when results finish out of order', () => {
|
||||
const playground = useHearingPlaygroundSegments()
|
||||
const firstMetadata = playground.startRecording(new Blob(['first']))
|
||||
const secondMetadata = playground.startRecording(new Blob(['second']))
|
||||
|
||||
playground.finishRecording(secondMetadata, 'Second sentence.')
|
||||
playground.finishRecording(firstMetadata, 'First sentence.')
|
||||
|
||||
expect(playground.segments.value.map(segment => segment.text)).toEqual([
|
||||
'First sentence.',
|
||||
'Second sentence.',
|
||||
])
|
||||
})
|
||||
|
||||
it('replaces a volatile streaming transcript when the provider corrects it', () => {
|
||||
// ROOT CAUSE:
|
||||
//
|
||||
// Apple Speech sends complete volatile snapshots that can revise earlier characters.
|
||||
// Appending each snapshot kept both the incorrect text and its correction.
|
||||
//
|
||||
// Before: "今天天气很号 今天天气很好"
|
||||
// After: "今天天气很好"
|
||||
const playground = useHearingPlaygroundSegments()
|
||||
|
||||
playground.replaceStreamingText('今天天气很号')
|
||||
playground.replaceStreamingText('今天天气很好')
|
||||
|
||||
expect(playground.current.value).toBe('今天天气很好')
|
||||
})
|
||||
|
||||
it('clears streaming and completed transcripts when the playground resets', () => {
|
||||
const playground = useHearingPlaygroundSegments()
|
||||
const metadata = playground.startRecording(new Blob(['speech']))
|
||||
|
||||
playground.finishRecording(metadata, 'First provider result.')
|
||||
playground.replaceStreamingText('Second provider partial result')
|
||||
playground.clear()
|
||||
|
||||
expect(playground.current.value).toBe('')
|
||||
expect(playground.segments.value).toEqual([])
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,106 @@
|
||||
import { readonly, ref, shallowRef } from 'vue'
|
||||
|
||||
/** Lifecycle state of one recorded Hearing playground segment. */
|
||||
export type HearingPlaygroundSegmentStatus = 'transcribing' | 'complete' | 'empty' | 'error'
|
||||
|
||||
/** A recorded playground utterance and the transcription state that belongs to it. */
|
||||
export interface HearingPlaygroundSegment {
|
||||
/** Stable identity used to correlate asynchronous provider results. */
|
||||
id: number
|
||||
/** Audio sent to the provider. VAD-triggered recordings include retained speech padding. */
|
||||
recording?: Blob
|
||||
/** Final provider text. Empty while the request is pending or produced no text. */
|
||||
text: string
|
||||
/** User-facing failure details when transcription throws. */
|
||||
error?: string
|
||||
/** Current transcription state for this segment. */
|
||||
status: HearingPlaygroundSegmentStatus
|
||||
}
|
||||
|
||||
interface HearingPlaygroundSegmentMetadata extends Record<string, unknown> {
|
||||
playgroundSegmentId: number
|
||||
}
|
||||
|
||||
function segmentIdFrom(metadata: Record<string, unknown> | undefined): number | undefined {
|
||||
const id = metadata?.playgroundSegmentId
|
||||
return typeof id === 'number' ? id : undefined
|
||||
}
|
||||
|
||||
/**
|
||||
* Keeps playground recordings and asynchronous transcription results correlated.
|
||||
* Empty and failed results remain visible so later text cannot shift onto earlier audio.
|
||||
*/
|
||||
export function useHearingPlaygroundSegments() {
|
||||
const current = shallowRef('')
|
||||
const segments = ref<HearingPlaygroundSegment[]>([])
|
||||
let nextSegmentId = 0
|
||||
|
||||
function updateSegment(
|
||||
metadata: Record<string, unknown> | undefined,
|
||||
update: (segment: HearingPlaygroundSegment) => HearingPlaygroundSegment,
|
||||
) {
|
||||
const segmentId = segmentIdFrom(metadata)
|
||||
if (segmentId === undefined)
|
||||
return
|
||||
|
||||
segments.value = segments.value.map(segment => segment.id === segmentId ? update(segment) : segment)
|
||||
}
|
||||
|
||||
function startRecording(recording: Blob): HearingPlaygroundSegmentMetadata {
|
||||
const id = ++nextSegmentId
|
||||
segments.value = [
|
||||
...segments.value,
|
||||
{ id, recording, text: '', status: 'transcribing' },
|
||||
]
|
||||
return { playgroundSegmentId: id }
|
||||
}
|
||||
|
||||
function finishRecording(metadata: Record<string, unknown> | undefined, text: string) {
|
||||
const finalText = text.trim()
|
||||
if (!finalText) {
|
||||
finishEmpty(metadata)
|
||||
return
|
||||
}
|
||||
|
||||
updateSegment(metadata, segment => ({ ...segment, text: finalText, status: 'complete' }))
|
||||
}
|
||||
|
||||
function finishEmpty(metadata: Record<string, unknown> | undefined) {
|
||||
updateSegment(metadata, segment => ({ ...segment, text: '', status: 'empty' }))
|
||||
}
|
||||
|
||||
function finishError(metadata: Record<string, unknown> | undefined, error: string) {
|
||||
updateSegment(metadata, segment => ({ ...segment, error, status: 'error' }))
|
||||
}
|
||||
|
||||
function replaceStreamingText(text: string) {
|
||||
current.value = text.trim()
|
||||
}
|
||||
|
||||
function finishStreaming(text: string) {
|
||||
const finalText = text.trim() || current.value.trim()
|
||||
current.value = ''
|
||||
if (!finalText)
|
||||
return
|
||||
|
||||
const id = ++nextSegmentId
|
||||
segments.value = [...segments.value, { id, text: finalText, status: 'complete' }]
|
||||
}
|
||||
|
||||
function clear() {
|
||||
current.value = ''
|
||||
segments.value = []
|
||||
}
|
||||
|
||||
return {
|
||||
current: readonly(current),
|
||||
segments: readonly(segments),
|
||||
startRecording,
|
||||
finishRecording,
|
||||
finishEmpty,
|
||||
finishError,
|
||||
replaceStreamingText,
|
||||
finishStreaming,
|
||||
clear,
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user