mirror of
https://github.com/moeru-ai/airi.git
synced 2026-08-14 00:48:06 +00:00
feat(tts): add Volcengine streaming BYOK
This commit is contained in:
@@ -1405,6 +1405,20 @@ pages:
|
||||
description: App ID of the project where you can obtain in Console
|
||||
label: App ID
|
||||
title: Volcano Engine
|
||||
volcengine-streaming:
|
||||
title: Volcengine Streaming TTS
|
||||
description: Low-latency Volcengine TTS through the AIRI UnSpeech bridge
|
||||
login-required: Sign in to AIRI so the browser can securely use the WebSocket proxy.
|
||||
config-description: Configure the X-Api-Key from the Volcengine new console. The existing Volcengine V1 provider is unchanged.
|
||||
model-placeholder: Choose a streaming model
|
||||
privacy-title: How your key is used
|
||||
privacy-description: Your key stays in this device's provider settings. For each WSS session, AIRI forwards it in memory only to authenticate UnSpeech and does not store it server-side.
|
||||
preview-text: Hello, this is a preview of Volcengine bidirectional streaming speech synthesis.
|
||||
fields:
|
||||
api-key:
|
||||
label: X-Api-Key
|
||||
description: X-Api-Key from the Volcengine new console. App ID and the old-console Access Key are not supported by this streaming endpoint.
|
||||
placeholder: Enter your X-Api-Key
|
||||
volcengine-coding-plan:
|
||||
description: Volcengine Coding Plan
|
||||
title: Volcengine Coding Plan
|
||||
|
||||
@@ -1350,6 +1350,20 @@ pages:
|
||||
description: ID de App del proyecto que puedes obtener en la Consola
|
||||
label: ID de App
|
||||
title: Volcano Engine
|
||||
volcengine-streaming:
|
||||
title: Volcengine Streaming TTS
|
||||
description: Low-latency Volcengine TTS through the AIRI UnSpeech bridge
|
||||
login-required: Sign in to AIRI so the browser can securely use the WebSocket proxy.
|
||||
config-description: Configure the X-Api-Key from the Volcengine new console. The existing Volcengine V1 provider is unchanged.
|
||||
model-placeholder: Choose a streaming model
|
||||
privacy-title: How your key is used
|
||||
privacy-description: Your key stays in this device's provider settings. AIRI only forwards it in memory for each WSS session and does not store it server-side.
|
||||
preview-text: Hello, this is a preview of Volcengine bidirectional streaming speech synthesis.
|
||||
fields:
|
||||
api-key:
|
||||
label: X-Api-Key
|
||||
description: X-Api-Key from the Volcengine new console. App ID and old-console Access Key are not supported.
|
||||
placeholder: Enter your X-Api-Key
|
||||
volcengine-coding-plan:
|
||||
description: Plan de Codificación Volcengine
|
||||
title: Plan de Codificación Volcengine
|
||||
|
||||
@@ -1350,6 +1350,20 @@ pages:
|
||||
description: ID de l’application du projet, que vous pouvez obtenir dans la Console
|
||||
label: ID de l’application
|
||||
title: Volcano Engine
|
||||
volcengine-streaming:
|
||||
title: Volcengine Streaming TTS
|
||||
description: Low-latency Volcengine TTS through the AIRI UnSpeech bridge
|
||||
login-required: Sign in to AIRI so the browser can securely use the WebSocket proxy.
|
||||
config-description: Configure the X-Api-Key from the Volcengine new console. The existing Volcengine V1 provider is unchanged.
|
||||
model-placeholder: Choose a streaming model
|
||||
privacy-title: How your key is used
|
||||
privacy-description: Your key stays in this device's provider settings. AIRI only forwards it in memory for each WSS session and does not store it server-side.
|
||||
preview-text: Hello, this is a preview of Volcengine bidirectional streaming speech synthesis.
|
||||
fields:
|
||||
api-key:
|
||||
label: X-Api-Key
|
||||
description: X-Api-Key from the Volcengine new console. App ID and old-console Access Key are not supported.
|
||||
placeholder: Enter your X-Api-Key
|
||||
volcengine-coding-plan:
|
||||
description: Volcengine Coding Plan
|
||||
title: Volcengine Coding Plan
|
||||
|
||||
@@ -1350,6 +1350,20 @@ pages:
|
||||
description: コンソールで取得できるプロジェクトのApp ID
|
||||
label: App ID
|
||||
title: Volcano Engine
|
||||
volcengine-streaming:
|
||||
title: Volcengine Streaming TTS
|
||||
description: Low-latency Volcengine TTS through the AIRI UnSpeech bridge
|
||||
login-required: Sign in to AIRI so the browser can securely use the WebSocket proxy.
|
||||
config-description: Configure the X-Api-Key from the Volcengine new console. The existing Volcengine V1 provider is unchanged.
|
||||
model-placeholder: Choose a streaming model
|
||||
privacy-title: How your key is used
|
||||
privacy-description: Your key stays in this device's provider settings. AIRI only forwards it in memory for each WSS session and does not store it server-side.
|
||||
preview-text: Hello, this is a preview of Volcengine bidirectional streaming speech synthesis.
|
||||
fields:
|
||||
api-key:
|
||||
label: X-Api-Key
|
||||
description: X-Api-Key from the Volcengine new console. App ID and old-console Access Key are not supported.
|
||||
placeholder: Enter your X-Api-Key
|
||||
volcengine-coding-plan:
|
||||
description: Volcengine Coding Plan
|
||||
title: Volcengine Coding Plan
|
||||
|
||||
@@ -1350,6 +1350,20 @@ pages:
|
||||
description: 콘솔에서 얻을 수 있는 프로젝트의 애플리케이션 ID
|
||||
label: 애플리케이션 ID
|
||||
title: Volcano 엔진
|
||||
volcengine-streaming:
|
||||
title: Volcengine Streaming TTS
|
||||
description: Low-latency Volcengine TTS through the AIRI UnSpeech bridge
|
||||
login-required: Sign in to AIRI so the browser can securely use the WebSocket proxy.
|
||||
config-description: Configure the X-Api-Key from the Volcengine new console. The existing Volcengine V1 provider is unchanged.
|
||||
model-placeholder: Choose a streaming model
|
||||
privacy-title: How your key is used
|
||||
privacy-description: Your key stays in this device's provider settings. AIRI only forwards it in memory for each WSS session and does not store it server-side.
|
||||
preview-text: Hello, this is a preview of Volcengine bidirectional streaming speech synthesis.
|
||||
fields:
|
||||
api-key:
|
||||
label: X-Api-Key
|
||||
description: X-Api-Key from the Volcengine new console. App ID and old-console Access Key are not supported.
|
||||
placeholder: Enter your X-Api-Key
|
||||
volcengine-coding-plan:
|
||||
description: Volcengine Coding Plan
|
||||
title: Volcengine Coding Plan
|
||||
|
||||
@@ -1350,6 +1350,20 @@ pages:
|
||||
description: App ID проекта (получается в Console)
|
||||
label: Идентификатор приложения (App ID)
|
||||
title: Volcano Engine
|
||||
volcengine-streaming:
|
||||
title: Volcengine Streaming TTS
|
||||
description: Low-latency Volcengine TTS through the AIRI UnSpeech bridge
|
||||
login-required: Sign in to AIRI so the browser can securely use the WebSocket proxy.
|
||||
config-description: Configure the X-Api-Key from the Volcengine new console. The existing Volcengine V1 provider is unchanged.
|
||||
model-placeholder: Choose a streaming model
|
||||
privacy-title: How your key is used
|
||||
privacy-description: Your key stays in this device's provider settings. AIRI only forwards it in memory for each WSS session and does not store it server-side.
|
||||
preview-text: Hello, this is a preview of Volcengine bidirectional streaming speech synthesis.
|
||||
fields:
|
||||
api-key:
|
||||
label: X-Api-Key
|
||||
description: X-Api-Key from the Volcengine new console. App ID and old-console Access Key are not supported.
|
||||
placeholder: Enter your X-Api-Key
|
||||
volcengine-coding-plan:
|
||||
description: План кодирования Volcengine
|
||||
title: План кодирования Volcengine
|
||||
|
||||
@@ -1350,6 +1350,20 @@ pages:
|
||||
description: App ID của dự án (lấy trong Console)
|
||||
label: App ID
|
||||
title: Volcano Engine
|
||||
volcengine-streaming:
|
||||
title: Volcengine Streaming TTS
|
||||
description: Low-latency Volcengine TTS through the AIRI UnSpeech bridge
|
||||
login-required: Sign in to AIRI so the browser can securely use the WebSocket proxy.
|
||||
config-description: Configure the X-Api-Key from the Volcengine new console. The existing Volcengine V1 provider is unchanged.
|
||||
model-placeholder: Choose a streaming model
|
||||
privacy-title: How your key is used
|
||||
privacy-description: Your key stays in this device's provider settings. AIRI only forwards it in memory for each WSS session and does not store it server-side.
|
||||
preview-text: Hello, this is a preview of Volcengine bidirectional streaming speech synthesis.
|
||||
fields:
|
||||
api-key:
|
||||
label: X-Api-Key
|
||||
description: X-Api-Key from the Volcengine new console. App ID and old-console Access Key are not supported.
|
||||
placeholder: Enter your X-Api-Key
|
||||
volcengine-coding-plan:
|
||||
description: Volcengine Coding Plan
|
||||
title: Volcengine Coding Plan
|
||||
|
||||
@@ -1350,6 +1350,20 @@ pages:
|
||||
description: 可在控制台获取的 App ID
|
||||
label: App ID
|
||||
title: 火山引擎
|
||||
volcengine-streaming:
|
||||
title: 火山引擎(双向流式 TTS)
|
||||
description: 通过 AIRI UnSpeech 代理使用低延迟火山引擎语音合成
|
||||
login-required: 请先登录 AIRI,以便浏览器安全使用 WebSocket 代理。
|
||||
config-description: 配置火山引擎新控制台的 X-Api-Key。现有火山引擎 V1 提供商不会受到影响。
|
||||
model-placeholder: 选择流式模型
|
||||
privacy-title: 密钥使用方式
|
||||
privacy-description: 密钥保存在当前设备的提供商设置中。每次 WSS 会话中,AIRI 仅在内存中转发密钥用于 UnSpeech 鉴权,不会在服务端持久化。
|
||||
preview-text: 你好,这是火山引擎双向流式语音合成的试听样例。
|
||||
fields:
|
||||
api-key:
|
||||
label: X-Api-Key
|
||||
description: 火山引擎新控制台提供的 X-Api-Key。此流式接口不支持 App ID 与旧控制台 Access Key。
|
||||
placeholder: 输入 X-Api-Key
|
||||
volcengine-coding-plan:
|
||||
description: 火山引擎 Coding Plan
|
||||
title: 火山引擎 Coding Plan
|
||||
|
||||
@@ -1350,6 +1350,20 @@ pages:
|
||||
description: 可在控制台取得的 App ID
|
||||
label: 應用程式 ID
|
||||
title: 火山引擎
|
||||
volcengine-streaming:
|
||||
title: 火山引擎(雙向串流 TTS)
|
||||
description: 透過 AIRI UnSpeech 代理使用低延遲火山引擎語音合成
|
||||
login-required: 請先登入 AIRI,讓瀏覽器安全使用 WebSocket 代理。
|
||||
config-description: 設定火山引擎新控制台的 X-Api-Key。現有火山引擎 V1 提供商不受影響。
|
||||
model-placeholder: 選擇串流模型
|
||||
privacy-title: 金鑰使用方式
|
||||
privacy-description: 金鑰保存在目前裝置的提供商設定中。每次 WSS 工作階段中,AIRI 僅在記憶體轉發金鑰供 UnSpeech 驗證,不會儲存在伺服器。
|
||||
preview-text: 你好,這是火山引擎雙向串流語音合成的試聽範例。
|
||||
fields:
|
||||
api-key:
|
||||
label: X-Api-Key
|
||||
description: 火山引擎新控制台提供的 X-Api-Key。此串流介面不支援 App ID 與舊控制台 Access Key。
|
||||
placeholder: 輸入 X-Api-Key
|
||||
volcengine-coding-plan:
|
||||
description: Volcengine Coding Plan
|
||||
title: Volcengine Coding Plan
|
||||
|
||||
+4
@@ -97,6 +97,10 @@ async function handleGenerateSpeech(input: string, voiceId: string, _useSSML: bo
|
||||
throw new Error(`Streaming model id missing backend prefix: ${requestedModel}`)
|
||||
const apiResourceId = requestedModel.slice(slashIndex + 1)
|
||||
const result = await streamingSynthesize({
|
||||
connection: {
|
||||
credentialMode: 'official',
|
||||
providerId,
|
||||
},
|
||||
model: requestedModel,
|
||||
voice: voiceId,
|
||||
input,
|
||||
|
||||
@@ -0,0 +1,187 @@
|
||||
<script setup lang="ts">
|
||||
import {
|
||||
ProviderApiKeyInput,
|
||||
ProviderBasicSettings,
|
||||
ProviderSettingsContainer,
|
||||
ProviderSettingsLayout,
|
||||
SpeechPlayground,
|
||||
} from '@proj-airi/stage-ui/components'
|
||||
import { getVolcengineStreamingDefaultModel, streamingSynthesize, VOLCENGINE_STREAMING_PROVIDER_ID } from '@proj-airi/stage-ui/libs'
|
||||
import { useAuthStore } from '@proj-airi/stage-ui/stores/auth'
|
||||
import { useSpeechStore } from '@proj-airi/stage-ui/stores/modules/speech'
|
||||
import { useProvidersStore } from '@proj-airi/stage-ui/stores/providers'
|
||||
import { Button, Callout, ComboboxSelect } from '@proj-airi/ui'
|
||||
import { storeToRefs } from 'pinia'
|
||||
import { computed, onMounted, ref, watch } from 'vue'
|
||||
import { useI18n } from 'vue-i18n'
|
||||
import { useRouter } from 'vue-router'
|
||||
|
||||
const router = useRouter()
|
||||
const { t } = useI18n()
|
||||
const authStore = useAuthStore()
|
||||
const providersStore = useProvidersStore()
|
||||
const speechStore = useSpeechStore()
|
||||
const { isAuthenticated, needsLogin } = storeToRefs(authStore)
|
||||
const { providers } = storeToRefs(providersStore)
|
||||
|
||||
const providerId = VOLCENGINE_STREAMING_PROVIDER_ID
|
||||
const providerMetadata = computed(() => providersStore.getProviderMetadata(providerId))
|
||||
const providerConfig = computed(() => providersStore.getProviderConfig(providerId))
|
||||
|
||||
const apiKey = computed({
|
||||
get: () => (providers.value[providerId]?.apiKey as string | undefined) ?? '',
|
||||
set: (value: string) => {
|
||||
providers.value[providerId] ??= {}
|
||||
providers.value[providerId].apiKey = value
|
||||
},
|
||||
})
|
||||
const apiKeyConfigured = computed(() => apiKey.value.trim().length > 0)
|
||||
|
||||
const providerModels = computed(() => providersStore.getModelsForProvider(providerId))
|
||||
const modelsLoading = computed(() => providersStore.isLoadingModels[providerId] || false)
|
||||
const discoveredDefaultModel = ref<string | null>(null)
|
||||
const model = computed({
|
||||
get: () => (providerConfig.value?.model as string | undefined) ?? discoveredDefaultModel.value ?? '',
|
||||
set: (value: string) => {
|
||||
providers.value[providerId] ??= {}
|
||||
providers.value[providerId].model = value
|
||||
},
|
||||
})
|
||||
const modelOptions = computed(() => providerModels.value.map(item => ({ label: item.name, value: item.id })))
|
||||
|
||||
const availableVoices = computed(() => speechStore.availableVoices[providerId] || [])
|
||||
const voicesLoading = ref(false)
|
||||
|
||||
async function loadVoices() {
|
||||
if (!isAuthenticated.value || !model.value)
|
||||
return
|
||||
voicesLoading.value = true
|
||||
try {
|
||||
await speechStore.loadVoicesForProvider(providerId, model.value)
|
||||
}
|
||||
finally {
|
||||
voicesLoading.value = false
|
||||
}
|
||||
}
|
||||
|
||||
async function loadCatalog() {
|
||||
if (!isAuthenticated.value)
|
||||
return
|
||||
await providersStore.fetchModelsForProvider(providerId)
|
||||
discoveredDefaultModel.value = getVolcengineStreamingDefaultModel() ?? providerModels.value[0]?.id ?? null
|
||||
if (!providerConfig.value.model && discoveredDefaultModel.value)
|
||||
model.value = discoveredDefaultModel.value
|
||||
await loadVoices()
|
||||
}
|
||||
|
||||
async function handleGenerateSpeech(input: string, voiceId: string): Promise<ArrayBuffer> {
|
||||
const requestedModel = model.value
|
||||
const key = apiKey.value.trim()
|
||||
if (!key)
|
||||
throw new Error('X-Api-Key is required.')
|
||||
if (!requestedModel.includes('/'))
|
||||
throw new Error(`Streaming model id missing backend prefix: ${requestedModel}`)
|
||||
|
||||
const resourceId = requestedModel.split('/', 2)[1]
|
||||
const result = await streamingSynthesize({
|
||||
connection: {
|
||||
credentialMode: 'byok',
|
||||
providerId,
|
||||
apiKey: key,
|
||||
},
|
||||
model: requestedModel,
|
||||
voice: voiceId,
|
||||
input,
|
||||
ttsSource: 'manual_preview',
|
||||
ttsVoiceType: 'custom_configured',
|
||||
extraBody: {
|
||||
api_resource_id: resourceId,
|
||||
audio: { sample_rate: 24000, bit_rate: 64000 },
|
||||
},
|
||||
})
|
||||
return result.audio
|
||||
}
|
||||
|
||||
function handleLogin() {
|
||||
needsLogin.value = true
|
||||
}
|
||||
|
||||
onMounted(async () => {
|
||||
providersStore.initializeProvider(providerId)
|
||||
await loadCatalog()
|
||||
})
|
||||
|
||||
watch(isAuthenticated, async (authenticated) => {
|
||||
if (authenticated)
|
||||
await loadCatalog()
|
||||
})
|
||||
|
||||
watch(model, async () => {
|
||||
await loadVoices()
|
||||
})
|
||||
</script>
|
||||
|
||||
<template>
|
||||
<ProviderSettingsLayout
|
||||
:provider-name="providerMetadata.localizedName"
|
||||
:provider-icon-color="providerMetadata.iconColor"
|
||||
:on-back="() => router.back()"
|
||||
>
|
||||
<div v-if="!isAuthenticated" :class="['mx-auto max-w-2xl', 'flex flex-col gap-4']">
|
||||
<Callout theme="primary" :label="providerMetadata.localizedName">
|
||||
<div :class="['flex flex-col gap-3']">
|
||||
<p>{{ t('settings.pages.providers.provider.volcengine-streaming.login-required') }}</p>
|
||||
<Button class="w-fit" @click="handleLogin">
|
||||
{{ t('settings.dialogs.onboarding.loginAction') }}
|
||||
</Button>
|
||||
</div>
|
||||
</Callout>
|
||||
</div>
|
||||
|
||||
<div v-else :class="['flex flex-col gap-6 md:flex-row']">
|
||||
<ProviderSettingsContainer class="w-full md:w-[40%]">
|
||||
<ProviderBasicSettings
|
||||
:title="t('settings.pages.providers.common.section.basic.title')"
|
||||
:description="t('settings.pages.providers.provider.volcengine-streaming.config-description')"
|
||||
>
|
||||
<ProviderApiKeyInput
|
||||
v-model="apiKey"
|
||||
:provider-name="providerMetadata.localizedName"
|
||||
:label="t('settings.pages.providers.provider.volcengine-streaming.fields.api-key.label')"
|
||||
:description="t('settings.pages.providers.provider.volcengine-streaming.fields.api-key.description')"
|
||||
:placeholder="t('settings.pages.providers.provider.volcengine-streaming.fields.api-key.placeholder')"
|
||||
required
|
||||
/>
|
||||
|
||||
<ComboboxSelect
|
||||
v-model="model"
|
||||
:options="modelOptions"
|
||||
:disabled="modelsLoading"
|
||||
:placeholder="t('settings.pages.providers.provider.volcengine-streaming.model-placeholder')"
|
||||
/>
|
||||
</ProviderBasicSettings>
|
||||
|
||||
<Callout :label="t('settings.pages.providers.provider.volcengine-streaming.privacy-title')">
|
||||
<p>{{ t('settings.pages.providers.provider.volcengine-streaming.privacy-description') }}</p>
|
||||
</Callout>
|
||||
</ProviderSettingsContainer>
|
||||
|
||||
<div class="w-full md:w-[60%]">
|
||||
<SpeechPlayground
|
||||
:available-voices="availableVoices"
|
||||
:generate-speech="handleGenerateSpeech"
|
||||
:api-key-configured="apiKeyConfigured"
|
||||
:voices-loading="voicesLoading"
|
||||
:default-text="t('settings.pages.providers.provider.volcengine-streaming.preview-text')"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</ProviderSettingsLayout>
|
||||
</template>
|
||||
|
||||
<route lang="yaml">
|
||||
meta:
|
||||
layout: settings
|
||||
stageTransition:
|
||||
name: slide
|
||||
</route>
|
||||
@@ -36,7 +36,7 @@ import { useIOTraceBridge } from '../../composables/use-io-trace-bridge'
|
||||
import { initIOTracer } from '../../composables/use-io-tracer'
|
||||
import { useSpeechPipelineAnalytics } from '../../composables/use-speech-pipeline-analytics'
|
||||
import { Emotion, EMOTION_EmotionMotionName_value, EMOTION_VRMExpressionName_value, EmotionThinkMotionName } from '../../constants/emotions'
|
||||
import { getDefaultStreamingModel, getDefinedProvider } from '../../libs/providers/providers'
|
||||
import { getDefinedProvider } from '../../libs/providers/providers'
|
||||
import { OFFICIAL_SPEECH_PROVIDER_ID, OFFICIAL_SPEECH_STREAMING_PROVIDER_ID } from '../../libs/providers/providers/official'
|
||||
import { bindSpeakingStateToPlaybackManager } from '../../libs/speech/playback-speaking-state'
|
||||
import { createStageTtsSession } from '../../libs/speech/tts-session'
|
||||
@@ -697,17 +697,31 @@ function stopSpeechOutput(reason: string) {
|
||||
resetAssistantSpeechSurface(reason)
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves the official streaming TTS model for the current Stage session.
|
||||
*/
|
||||
function resolveStreamingSessionModel(): string | null {
|
||||
const activeModel = activeSpeechModel.value as string | undefined
|
||||
const sessionModel = activeModel?.includes('/') ? activeModel : getDefaultStreamingModel()
|
||||
const definition = activeSpeechProvider.value ? getDefinedProvider(activeSpeechProvider.value) : undefined
|
||||
const sessionModel = activeModel?.includes('/')
|
||||
? activeModel
|
||||
: definition?.capabilities?.speech?.getDefaultModel?.()
|
||||
if (!sessionModel?.includes('/'))
|
||||
return null
|
||||
return sessionModel
|
||||
}
|
||||
|
||||
function resolveStreamingConnection() {
|
||||
const providerId = activeSpeechProvider.value
|
||||
if (!providerId)
|
||||
return null
|
||||
const definition = getDefinedProvider(providerId)
|
||||
const resolver = definition?.capabilities?.speech?.resolveConnection
|
||||
if (!resolver)
|
||||
return null
|
||||
const connection = resolver(providersStore.getProviderConfig(providerId) ?? {})
|
||||
if (connection.credentialMode === 'byok' && !connection.apiKey?.trim())
|
||||
return null
|
||||
return connection
|
||||
}
|
||||
|
||||
function buildStreamingSnapshot(turnId: string): StreamingSessionSnapshot | null {
|
||||
if (speechMuted.value)
|
||||
return null
|
||||
@@ -730,12 +744,16 @@ function buildStreamingSnapshot(turnId: string): StreamingSessionSnapshot | null
|
||||
const sessionModel = resolveStreamingSessionModel()
|
||||
if (!sessionModel)
|
||||
return null
|
||||
const connection = resolveStreamingConnection()
|
||||
if (!connection)
|
||||
return null
|
||||
const apiResourceId = sessionModel.split('/', 2)[1]
|
||||
// TTS 2.0 / ICL 2.0 ship subtitles asynchronously relative to audio
|
||||
// (per the wire spec), so chunk-on-sentence-end would drop frames.
|
||||
// Buffer the entire session and decode at session.finished instead.
|
||||
const bufferEntireSession = apiResourceId.startsWith('seed-tts-2.0') || apiResourceId.startsWith('seed-icl-2.0')
|
||||
return {
|
||||
connection,
|
||||
model: sessionModel,
|
||||
voice: voiceId,
|
||||
voiceType: resolveStageVoiceType(),
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
export * from './audio/manager'
|
||||
export * from './color-from-element'
|
||||
export * from './providers'
|
||||
export * from './speech/streaming-connection'
|
||||
export * from './speech/streaming-pipeline'
|
||||
export * from './speech/streaming-session'
|
||||
export * from './speech/tts-session'
|
||||
|
||||
@@ -6,6 +6,7 @@ import './azure-openai'
|
||||
import './openai-compatible'
|
||||
import './atlascloud'
|
||||
import './volcengine-coding-plan'
|
||||
import './volcengine-streaming'
|
||||
import './byteplus'
|
||||
import './byteplus-coding-plan'
|
||||
import './n1n'
|
||||
@@ -46,3 +47,5 @@ export {
|
||||
getDefinedProvider,
|
||||
listProviders,
|
||||
} from './registry'
|
||||
|
||||
export { getVolcengineStreamingDefaultModel, VOLCENGINE_STREAMING_PROVIDER_ID } from './volcengine-streaming'
|
||||
|
||||
@@ -230,7 +230,14 @@ export const providerOfficialSpeechStreaming = defineProvider({
|
||||
// session adapter (`tts-session.ts`) picks the streaming path without
|
||||
// hard-coding provider id. Default for every other provider is `'rest'`.
|
||||
capabilities: {
|
||||
speech: { transport: 'bidirectional-ws' },
|
||||
speech: {
|
||||
transport: 'bidirectional-ws',
|
||||
resolveConnection: () => ({
|
||||
credentialMode: 'official',
|
||||
providerId: OFFICIAL_SPEECH_STREAMING_PROVIDER_ID,
|
||||
}),
|
||||
getDefaultModel: getDefaultStreamingModel,
|
||||
},
|
||||
},
|
||||
createProviderConfig: () => officialConfigSchema,
|
||||
createProvider(_config) {
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
import { describe, expect, it } from 'vitest'
|
||||
|
||||
import { providerVolcengineStreaming, VOLCENGINE_STREAMING_PROVIDER_ID } from './index'
|
||||
|
||||
describe('volcengine streaming provider', () => {
|
||||
it('keeps BYOK separate from the existing volcengine provider identity', () => {
|
||||
expect(VOLCENGINE_STREAMING_PROVIDER_ID).toBe('volcengine-streaming')
|
||||
expect(providerVolcengineStreaming.id).toBe('volcengine-streaming')
|
||||
expect(providerVolcengineStreaming.capabilities?.speech?.transport).toBe('bidirectional-ws')
|
||||
|
||||
const connection = providerVolcengineStreaming.capabilities?.speech?.resolveConnection?.({
|
||||
apiKey: ' user-x-api-key ',
|
||||
})
|
||||
expect(connection).toEqual({
|
||||
credentialMode: 'byok',
|
||||
providerId: 'volcengine-streaming',
|
||||
apiKey: 'user-x-api-key',
|
||||
})
|
||||
})
|
||||
|
||||
it('requires an X-Api-Key before the provider becomes configured', async () => {
|
||||
const validator = providerVolcengineStreaming.validators?.validateConfig?.[0]?.({ t: input => input })
|
||||
expect(validator).toBeDefined()
|
||||
|
||||
const missing = await validator!.validator({ apiKey: '' }, { t: input => input })
|
||||
const configured = await validator!.validator({ apiKey: 'user-key' }, { t: input => input })
|
||||
|
||||
expect(missing.valid).toBe(false)
|
||||
expect(missing.reason).toBe('X-Api-Key is required.')
|
||||
expect(configured.valid).toBe(true)
|
||||
expect(configured.reason).toBe('')
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,148 @@
|
||||
import type { ModelInfo, VoiceInfo } from '../../types'
|
||||
|
||||
import { createUnVolcengine } from 'unspeech'
|
||||
import { z } from 'zod'
|
||||
|
||||
import { getAuthToken } from '../../../auth'
|
||||
import { SERVER_URL } from '../../../server'
|
||||
import { defineProvider } from '../registry'
|
||||
|
||||
export const VOLCENGINE_STREAMING_PROVIDER_ID = 'volcengine-streaming'
|
||||
|
||||
const configSchema = z.object({
|
||||
apiKey: z.string(),
|
||||
})
|
||||
|
||||
type VolcengineStreamingConfig = z.input<typeof configSchema>
|
||||
|
||||
let defaultModelId: string | null = null
|
||||
|
||||
function authHeaders(): Record<string, string> {
|
||||
const headers: Record<string, string> = { Accept: 'application/json' }
|
||||
const token = getAuthToken()
|
||||
if (token)
|
||||
headers.Authorization = `Bearer ${token}`
|
||||
return headers
|
||||
}
|
||||
|
||||
export function getVolcengineStreamingDefaultModel(): string | null {
|
||||
return defaultModelId
|
||||
}
|
||||
|
||||
export const providerVolcengineStreaming = defineProvider<VolcengineStreamingConfig>({
|
||||
id: VOLCENGINE_STREAMING_PROVIDER_ID,
|
||||
order: 31,
|
||||
name: 'Volcengine Streaming TTS',
|
||||
nameLocalize: ({ t }) => t('settings.pages.providers.provider.volcengine-streaming.title'),
|
||||
description: 'Low-latency Volcengine TTS through the AIRI UnSpeech bridge.',
|
||||
descriptionLocalize: ({ t }) => t('settings.pages.providers.provider.volcengine-streaming.description'),
|
||||
tasks: ['text-to-speech'],
|
||||
iconColor: 'i-lobe-icons:volcengine',
|
||||
createProviderConfig: ({ t }) => configSchema.extend({
|
||||
apiKey: configSchema.shape.apiKey.meta({
|
||||
labelLocalized: t('settings.pages.providers.common.fields.field.api-key.label'),
|
||||
descriptionLocalized: t('settings.pages.providers.provider.volcengine-streaming.fields.api-key.description'),
|
||||
placeholderLocalized: t('settings.pages.providers.provider.volcengine-streaming.fields.api-key.placeholder'),
|
||||
type: 'password',
|
||||
}),
|
||||
}),
|
||||
createProvider(config) {
|
||||
// The provider instance satisfies the shared speech-provider boundary.
|
||||
// Runtime synthesis uses the bidirectional session capability below, not
|
||||
// this REST-compatible instance.
|
||||
const apiKey = typeof config.apiKey === 'string' ? config.apiKey.trim() : ''
|
||||
return createUnVolcengine(apiKey, 'https://unspeech.hyp3r.link/v1/')
|
||||
},
|
||||
capabilities: {
|
||||
speech: {
|
||||
transport: 'bidirectional-ws',
|
||||
resolveConnection: (config) => {
|
||||
const apiKey = typeof config.apiKey === 'string' ? config.apiKey.trim() : ''
|
||||
return {
|
||||
credentialMode: 'byok',
|
||||
providerId: VOLCENGINE_STREAMING_PROVIDER_ID,
|
||||
apiKey,
|
||||
}
|
||||
},
|
||||
getDefaultModel: getVolcengineStreamingDefaultModel,
|
||||
},
|
||||
},
|
||||
validationRequiredWhen: () => true,
|
||||
validators: {
|
||||
validateConfig: [
|
||||
({ t }) => ({
|
||||
id: 'volcengine-streaming:check-config',
|
||||
name: t('settings.pages.providers.catalog.edit.validators.openai-compatible.check-config.title'),
|
||||
validator: async (config) => {
|
||||
const apiKey = typeof config.apiKey === 'string' ? config.apiKey.trim() : ''
|
||||
const valid = apiKey.length > 0
|
||||
return {
|
||||
errors: valid ? [] : [{ error: new Error('X-Api-Key is required.') }],
|
||||
reason: valid ? '' : 'X-Api-Key is required.',
|
||||
reasonKey: '',
|
||||
valid,
|
||||
}
|
||||
},
|
||||
}),
|
||||
],
|
||||
},
|
||||
extraMethods: {
|
||||
listModels: async (): Promise<ModelInfo[]> => {
|
||||
defaultModelId = null
|
||||
const response = await globalThis.fetch(`${SERVER_URL}/api/v1/audio/models/streaming`, { headers: authHeaders() })
|
||||
if (!response.ok)
|
||||
throw new Error(`streaming models upstream ${response.status}: ${await response.text().catch(() => '')}`.slice(0, 256))
|
||||
|
||||
const data = await response.json() as {
|
||||
models?: { id: string, name?: string, description?: string }[]
|
||||
default?: string | null
|
||||
}
|
||||
if (!Array.isArray(data.models))
|
||||
throw new Error('streaming models upstream missing models[]')
|
||||
|
||||
defaultModelId = typeof data.default === 'string' && data.default.length > 0 ? data.default : null
|
||||
return data.models.map(model => ({
|
||||
id: model.id,
|
||||
name: model.name ?? model.id,
|
||||
provider: VOLCENGINE_STREAMING_PROVIDER_ID,
|
||||
description: model.description,
|
||||
}))
|
||||
},
|
||||
listVoices: async (_config, _provider, model): Promise<VoiceInfo[]> => {
|
||||
const resourceId = model?.includes('/') ? model.split('/', 2)[1] : model
|
||||
const url = new URL(`${SERVER_URL}/api/v1/audio/voices/streaming`)
|
||||
if (resourceId)
|
||||
url.searchParams.set('model', resourceId)
|
||||
|
||||
const response = await globalThis.fetch(url, { headers: authHeaders() })
|
||||
if (!response.ok)
|
||||
throw new Error(`streaming voices upstream ${response.status}: ${await response.text().catch(() => '')}`.slice(0, 256))
|
||||
|
||||
const data = await response.json() as {
|
||||
voices?: {
|
||||
id: string
|
||||
name: string
|
||||
description?: string
|
||||
labels?: Record<string, unknown>
|
||||
languages?: { code: string, title: string }[]
|
||||
preview_audio_url?: string
|
||||
}[]
|
||||
}
|
||||
if (!Array.isArray(data.voices))
|
||||
throw new Error('streaming voices upstream returned malformed body')
|
||||
|
||||
return data.voices.map((voice) => {
|
||||
const gender = typeof voice.labels?.gender === 'string' ? voice.labels.gender.toLowerCase() : undefined
|
||||
return {
|
||||
id: voice.id,
|
||||
name: voice.name,
|
||||
provider: VOLCENGINE_STREAMING_PROVIDER_ID,
|
||||
description: voice.description,
|
||||
gender,
|
||||
previewURL: voice.preview_audio_url,
|
||||
languages: Array.isArray(voice.languages) ? voice.languages : [],
|
||||
}
|
||||
})
|
||||
},
|
||||
},
|
||||
})
|
||||
@@ -95,6 +95,7 @@ const providerSourceMetadataById = {
|
||||
'speech-noop': false,
|
||||
'together-ai': paidCloud,
|
||||
'volcengine': paidCloud,
|
||||
'volcengine-streaming': paidCloud,
|
||||
'volcengine-coding-plan': paidCloud,
|
||||
'xai': paidCloud,
|
||||
'zai': paidCloud,
|
||||
|
||||
@@ -15,6 +15,8 @@ import type { MaybePromise } from 'clustr'
|
||||
import type { ComposerTranslation } from 'vue-i18n'
|
||||
import type { $ZodType } from 'zod/v4/core'
|
||||
|
||||
import type { StreamingTtsConnection } from '../speech/streaming-connection'
|
||||
|
||||
export type ProviderInstance
|
||||
= | ChatProvider
|
||||
| ChatProviderWithExtraOptions
|
||||
@@ -214,6 +216,10 @@ export interface ProviderDefinition<TConfig extends any = any> {
|
||||
*/
|
||||
speech?: {
|
||||
transport: 'rest' | 'bidirectional-ws'
|
||||
/** Resolves provider-owned credential policy for one streaming session. */
|
||||
resolveConnection?: (config: TConfig) => StreamingTtsConnection
|
||||
/** Returns the provider's discovered default streaming model. */
|
||||
getDefaultModel?: () => string | null
|
||||
}
|
||||
}
|
||||
/**
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
/**
|
||||
* Connection policy snapshotted from the active streaming speech provider.
|
||||
*
|
||||
* Official sessions use AIRI-hosted credentials and Flux. BYOK sessions send
|
||||
* the user credential inside the authenticated WSS session so the AIRI bridge
|
||||
* can inject it into UnSpeech's upstream upgrade request.
|
||||
*/
|
||||
export interface StreamingTtsConnection {
|
||||
credentialMode: 'official' | 'byok'
|
||||
providerId: 'official-provider-speech-streaming' | 'volcengine-streaming'
|
||||
/** Plaintext credential used only for a BYOK upstream handshake. */
|
||||
apiKey?: string
|
||||
}
|
||||
@@ -8,6 +8,11 @@ import { WebSocketServer } from 'ws'
|
||||
|
||||
import { createStreamingTtsPipeline } from './streaming-pipeline'
|
||||
|
||||
const officialConnection = {
|
||||
credentialMode: 'official',
|
||||
providerId: 'official-provider-speech-streaming',
|
||||
} as const
|
||||
|
||||
vi.mock('../auth', () => ({
|
||||
getAuthToken: () => 'test-jwt',
|
||||
}))
|
||||
@@ -19,6 +24,7 @@ interface MockServer {
|
||||
url: string
|
||||
receivedFrames: Array<{ kind: 'text' | 'binary', data: string | Buffer }>
|
||||
observedVoiceTypes: string[]
|
||||
observedCredentialModes: string[]
|
||||
/** Resolves when the server has observed a `start` frame from the client. */
|
||||
startObserved: Promise<void>
|
||||
stop: () => Promise<void>
|
||||
@@ -27,6 +33,7 @@ interface MockServer {
|
||||
async function startMockServer(handler: (ws: import('ws').WebSocket) => void): Promise<MockServer> {
|
||||
const receivedFrames: MockServer['receivedFrames'] = []
|
||||
const observedVoiceTypes: string[] = []
|
||||
const observedCredentialModes: string[] = []
|
||||
const httpServer = createServer()
|
||||
const wss = new WebSocketServer({ server: httpServer })
|
||||
|
||||
@@ -40,6 +47,9 @@ async function startMockServer(handler: (ws: import('ws').WebSocket) => void): P
|
||||
const voiceType = u.searchParams.get('tts_voice_type')
|
||||
if (voiceType != null)
|
||||
observedVoiceTypes.push(voiceType)
|
||||
const credentialMode = u.searchParams.get('tts_credential_mode')
|
||||
if (credentialMode != null)
|
||||
observedCredentialModes.push(credentialMode)
|
||||
|
||||
ws.on('message', (data, isBinary) => {
|
||||
const buf = Buffer.isBuffer(data) ? data : Buffer.from(data as ArrayBuffer)
|
||||
@@ -64,6 +74,7 @@ async function startMockServer(handler: (ws: import('ws').WebSocket) => void): P
|
||||
url: `http://127.0.0.1:${port}`,
|
||||
receivedFrames,
|
||||
observedVoiceTypes,
|
||||
observedCredentialModes,
|
||||
startObserved,
|
||||
async stop() {
|
||||
wss.close()
|
||||
@@ -132,6 +143,7 @@ describe('createStreamingTtsPipeline', () => {
|
||||
const onDone = vi.fn()
|
||||
|
||||
const handle = createStreamingTtsPipeline({
|
||||
connection: officialConnection,
|
||||
serverUrl: server.url,
|
||||
model: 'volcengine/seed-tts-1.0',
|
||||
voice: 'mock',
|
||||
@@ -169,6 +181,44 @@ describe('createStreamingTtsPipeline', () => {
|
||||
expect(calls[1].audio.__byteLength).toBe(chunks[2].length)
|
||||
})
|
||||
|
||||
it('opens chat streaming with the BYOK credentials frame before start', async () => {
|
||||
server = await startMockServer((ws) => {
|
||||
ws.on('message', (data, isBinary) => {
|
||||
if (isBinary)
|
||||
return
|
||||
const frame = JSON.parse(data.toString()) as { event?: string }
|
||||
if (frame.event === 'finish')
|
||||
ws.send(JSON.stringify({ event: 'session.finished', payload: { usage: { text_words: 5 } } }))
|
||||
})
|
||||
})
|
||||
|
||||
const onDone = vi.fn()
|
||||
const handle = createStreamingTtsPipeline({
|
||||
connection: {
|
||||
credentialMode: 'byok',
|
||||
providerId: 'volcengine-streaming',
|
||||
apiKey: 'chat-byok-key',
|
||||
},
|
||||
serverUrl: server.url,
|
||||
model: 'volcengine/seed-tts-2.0',
|
||||
voice: 'mock',
|
||||
audioContext: makeStubAudioContext(),
|
||||
onDone,
|
||||
})
|
||||
handle.appendText('hello')
|
||||
handle.finish()
|
||||
|
||||
await new Promise<void>((resolve) => {
|
||||
onDone.mockImplementation(() => resolve())
|
||||
setTimeout(resolve, 1500)
|
||||
})
|
||||
|
||||
const frames = server.receivedFrames.map(frame => JSON.parse(frame.data as string) as Record<string, unknown>)
|
||||
expect(frames.map(frame => frame.event)).toEqual(['credentials', 'start', 'text', 'finish'])
|
||||
expect(frames[0]?.api_key).toBe('chat-byok-key')
|
||||
expect(server.observedCredentialModes).toEqual(['byok'])
|
||||
})
|
||||
|
||||
it('buffers entire session when bufferEntireSession is true', async () => {
|
||||
const chunks = [Buffer.from([1, 2, 3, 4]), Buffer.from([5, 6, 7, 8])]
|
||||
server = await startMockServer((ws) => {
|
||||
@@ -190,6 +240,7 @@ describe('createStreamingTtsPipeline', () => {
|
||||
|
||||
const onSentence = vi.fn()
|
||||
const handle = createStreamingTtsPipeline({
|
||||
connection: officialConnection,
|
||||
serverUrl: server.url,
|
||||
model: 'volcengine/seed-tts-2.0',
|
||||
voice: 'mock',
|
||||
@@ -223,6 +274,7 @@ describe('createStreamingTtsPipeline', () => {
|
||||
const onError = vi.fn()
|
||||
const onDone = vi.fn()
|
||||
createStreamingTtsPipeline({
|
||||
connection: officialConnection,
|
||||
serverUrl: server.url,
|
||||
model: 'volcengine/seed-tts-1.0',
|
||||
voice: 'mock',
|
||||
@@ -256,6 +308,7 @@ describe('createStreamingTtsPipeline', () => {
|
||||
const onError = vi.fn()
|
||||
const onDone = vi.fn()
|
||||
createStreamingTtsPipeline({
|
||||
connection: officialConnection,
|
||||
serverUrl: server.url,
|
||||
model: 'volcengine/seed-tts-1.0',
|
||||
voice: 'mock',
|
||||
@@ -287,6 +340,7 @@ describe('createStreamingTtsPipeline', () => {
|
||||
|
||||
const onDone = vi.fn()
|
||||
const handle = createStreamingTtsPipeline({
|
||||
connection: officialConnection,
|
||||
serverUrl: server.url,
|
||||
model: 'volcengine/seed-tts-1.0',
|
||||
voice: 'mock',
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import type { StreamingTtsConnection } from './streaming-connection'
|
||||
|
||||
import { getAuthToken } from '../auth'
|
||||
import { SERVER_URL } from '../server'
|
||||
|
||||
@@ -36,6 +38,8 @@ export interface StreamingTtsPipelineEvents {
|
||||
}
|
||||
|
||||
export interface StreamingTtsPipelineOptions extends StreamingTtsPipelineEvents {
|
||||
/** Provider-owned credential and billing policy for this session. */
|
||||
connection: StreamingTtsConnection
|
||||
/** Server URL override. Defaults to {@link SERVER_URL}. */
|
||||
serverUrl?: string
|
||||
/** Override the auth token (Bearer). Defaults to {@link getAuthToken}. */
|
||||
@@ -125,6 +129,7 @@ export function createStreamingTtsPipeline(options: StreamingTtsPipelineOptions)
|
||||
ttsTrigger: options.ttsTrigger ?? 'auto',
|
||||
ttsSource: options.ttsSource ?? 'chat_auto_tts',
|
||||
ttsVoiceType: options.ttsVoiceType ?? 'unknown',
|
||||
connection: options.connection,
|
||||
})
|
||||
const ws = new WebSocket(wsUrl)
|
||||
ws.binaryType = 'arraybuffer'
|
||||
@@ -239,6 +244,13 @@ export function createStreamingTtsPipeline(options: StreamingTtsPipelineOptions)
|
||||
}
|
||||
|
||||
ws.addEventListener('open', () => {
|
||||
if (options.connection.credentialMode === 'byok') {
|
||||
ws.send(JSON.stringify({
|
||||
event: 'credentials',
|
||||
provider: 'volcengine',
|
||||
api_key: options.connection.apiKey,
|
||||
}))
|
||||
}
|
||||
const startFrame = {
|
||||
event: 'start',
|
||||
model: options.model,
|
||||
@@ -418,6 +430,7 @@ function toWebSocketUrl(
|
||||
ttsTrigger: 'auto' | 'manual'
|
||||
ttsSource: 'chat_auto_tts' | 'manual_preview' | 'settings_test'
|
||||
ttsVoiceType: 'official_default' | 'official_selected' | 'custom_configured' | 'voice_pack' | 'unknown'
|
||||
connection: StreamingTtsConnection
|
||||
},
|
||||
): string {
|
||||
const u = new URL(path, httpBase)
|
||||
@@ -426,6 +439,8 @@ function toWebSocketUrl(
|
||||
u.searchParams.set('tts_trigger', analytics.ttsTrigger)
|
||||
u.searchParams.set('tts_source', analytics.ttsSource)
|
||||
u.searchParams.set('tts_voice_type', analytics.ttsVoiceType)
|
||||
u.searchParams.set('tts_credential_mode', analytics.connection.credentialMode)
|
||||
u.searchParams.set('tts_provider_id', analytics.connection.providerId)
|
||||
return u.toString()
|
||||
}
|
||||
|
||||
|
||||
@@ -8,6 +8,11 @@ import { WebSocketServer } from 'ws'
|
||||
|
||||
import { streamingSynthesize } from './streaming-session'
|
||||
|
||||
const officialConnection = {
|
||||
credentialMode: 'official',
|
||||
providerId: 'official-provider-speech-streaming',
|
||||
} as const
|
||||
|
||||
vi.mock('../auth', () => ({
|
||||
getAuthToken: () => 'test-jwt',
|
||||
}))
|
||||
@@ -20,6 +25,8 @@ interface MockServer {
|
||||
url: string
|
||||
observedTokens: string[]
|
||||
observedVoiceTypes: string[]
|
||||
observedCredentialModes: string[]
|
||||
observedProviderIds: string[]
|
||||
closeUnexpectedly: () => void
|
||||
stop: () => Promise<void>
|
||||
}
|
||||
@@ -27,6 +34,8 @@ interface MockServer {
|
||||
async function startMockServer(handler: (ws: import('ws').WebSocket) => void): Promise<MockServer> {
|
||||
const observedTokens: string[] = []
|
||||
const observedVoiceTypes: string[] = []
|
||||
const observedCredentialModes: string[] = []
|
||||
const observedProviderIds: string[] = []
|
||||
const httpServer = createServer()
|
||||
const wss = new WebSocketServer({ server: httpServer })
|
||||
|
||||
@@ -41,6 +50,12 @@ async function startMockServer(handler: (ws: import('ws').WebSocket) => void): P
|
||||
const voiceType = u.searchParams.get('tts_voice_type')
|
||||
if (voiceType != null)
|
||||
observedVoiceTypes.push(voiceType)
|
||||
const credentialMode = u.searchParams.get('tts_credential_mode')
|
||||
if (credentialMode != null)
|
||||
observedCredentialModes.push(credentialMode)
|
||||
const providerId = u.searchParams.get('tts_provider_id')
|
||||
if (providerId != null)
|
||||
observedProviderIds.push(providerId)
|
||||
|
||||
handler(ws)
|
||||
})
|
||||
@@ -52,6 +67,8 @@ async function startMockServer(handler: (ws: import('ws').WebSocket) => void): P
|
||||
url: `http://127.0.0.1:${port}`,
|
||||
observedTokens,
|
||||
observedVoiceTypes,
|
||||
observedCredentialModes,
|
||||
observedProviderIds,
|
||||
closeUnexpectedly: () => {
|
||||
activeWs?.close(1011, 'simulated_truncation')
|
||||
},
|
||||
@@ -106,6 +123,7 @@ describe('streamingSynthesize', () => {
|
||||
})
|
||||
|
||||
const result = await streamingSynthesize({
|
||||
connection: officialConnection,
|
||||
serverUrl: server.url,
|
||||
model: 'volcengine/seed-tts-2.0',
|
||||
voice: 'mock',
|
||||
@@ -124,6 +142,41 @@ describe('streamingSynthesize', () => {
|
||||
expect(server.observedVoiceTypes).toEqual(['official_selected'])
|
||||
})
|
||||
|
||||
it('sends BYOK credentials before the UnSpeech start frame and marks the connection mode', async () => {
|
||||
const frames: Array<Record<string, unknown>> = []
|
||||
server = await startMockServer((ws) => {
|
||||
ws.on('message', (data, isBinary) => {
|
||||
if (isBinary)
|
||||
return
|
||||
const frame = JSON.parse(data.toString()) as Record<string, unknown>
|
||||
frames.push(frame)
|
||||
if (frame.event === 'finish')
|
||||
ws.send(JSON.stringify({ event: 'session.finished', payload: { usage: { text_words: 5 } } }))
|
||||
})
|
||||
})
|
||||
|
||||
await streamingSynthesize({
|
||||
connection: {
|
||||
credentialMode: 'byok',
|
||||
providerId: 'volcengine-streaming',
|
||||
apiKey: 'byok-test-key',
|
||||
},
|
||||
serverUrl: server.url,
|
||||
model: 'volcengine/seed-tts-2.0',
|
||||
voice: 'mock',
|
||||
input: 'hello',
|
||||
})
|
||||
|
||||
expect(frames.map(frame => frame.event)).toEqual(['credentials', 'start', 'text', 'finish'])
|
||||
expect(frames[0]).toEqual({
|
||||
event: 'credentials',
|
||||
provider: 'volcengine',
|
||||
api_key: 'byok-test-key',
|
||||
})
|
||||
expect(server.observedCredentialModes).toEqual(['byok'])
|
||||
expect(server.observedProviderIds).toEqual(['volcengine-streaming'])
|
||||
})
|
||||
|
||||
it('rejects when the ws closes before session.finished (codex HIGH #2)', async () => {
|
||||
// Server sends some audio chunks then closes the ws WITHOUT emitting
|
||||
// session.finished. Pre-fix behavior: streamingSynthesize would
|
||||
@@ -150,6 +203,7 @@ describe('streamingSynthesize', () => {
|
||||
})
|
||||
|
||||
await expect(streamingSynthesize({
|
||||
connection: officialConnection,
|
||||
serverUrl: server.url,
|
||||
model: 'volcengine/seed-tts-2.0',
|
||||
voice: 'mock',
|
||||
@@ -176,6 +230,7 @@ describe('streamingSynthesize', () => {
|
||||
})
|
||||
|
||||
await expect(streamingSynthesize({
|
||||
connection: officialConnection,
|
||||
serverUrl: server.url,
|
||||
model: 'volcengine/seed-tts-2.0',
|
||||
voice: 'mock',
|
||||
@@ -197,6 +252,7 @@ describe('streamingSynthesize', () => {
|
||||
|
||||
const ctrl = new AbortController()
|
||||
const promise = streamingSynthesize({
|
||||
connection: officialConnection,
|
||||
serverUrl: server.url,
|
||||
model: 'volcengine/seed-tts-2.0',
|
||||
voice: 'mock',
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import type { StreamingTtsConnection } from './streaming-connection'
|
||||
|
||||
import { getAuthToken } from '../auth'
|
||||
import { SERVER_URL } from '../server'
|
||||
|
||||
@@ -36,6 +38,8 @@ export interface StreamingTtsSessionResult {
|
||||
}
|
||||
|
||||
export interface StreamingTtsSessionOptions {
|
||||
/** Provider-owned credential and billing policy for this session. */
|
||||
connection: StreamingTtsConnection
|
||||
/** Server URL override. Defaults to {@link SERVER_URL}. */
|
||||
serverUrl?: string
|
||||
/** Override the auth token (Bearer). Defaults to {@link getAuthToken}. */
|
||||
@@ -95,6 +99,7 @@ export async function streamingSynthesize(options: StreamingTtsSessionOptions):
|
||||
ttsTrigger: options.ttsTrigger ?? 'manual',
|
||||
ttsSource: options.ttsSource ?? 'manual_preview',
|
||||
ttsVoiceType: options.ttsVoiceType ?? 'unknown',
|
||||
connection: options.connection,
|
||||
})
|
||||
|
||||
const audioChunks: ArrayBuffer[] = []
|
||||
@@ -143,6 +148,13 @@ export async function streamingSynthesize(options: StreamingTtsSessionOptions):
|
||||
}
|
||||
|
||||
ws.addEventListener('open', () => {
|
||||
if (options.connection.credentialMode === 'byok') {
|
||||
ws.send(JSON.stringify({
|
||||
event: 'credentials',
|
||||
provider: 'volcengine',
|
||||
api_key: options.connection.apiKey,
|
||||
}))
|
||||
}
|
||||
const startFrame = {
|
||||
event: 'start',
|
||||
model: options.model,
|
||||
@@ -243,6 +255,7 @@ function toWebSocketUrl(
|
||||
ttsTrigger: 'auto' | 'manual'
|
||||
ttsSource: 'chat_auto_tts' | 'manual_preview' | 'settings_test'
|
||||
ttsVoiceType: 'official_default' | 'official_selected' | 'custom_configured' | 'voice_pack' | 'unknown'
|
||||
connection: StreamingTtsConnection
|
||||
},
|
||||
): string {
|
||||
const u = new URL(path, httpBase)
|
||||
@@ -251,6 +264,8 @@ function toWebSocketUrl(
|
||||
u.searchParams.set('tts_trigger', analytics.ttsTrigger)
|
||||
u.searchParams.set('tts_source', analytics.ttsSource)
|
||||
u.searchParams.set('tts_voice_type', analytics.ttsVoiceType)
|
||||
u.searchParams.set('tts_credential_mode', analytics.connection.credentialMode)
|
||||
u.searchParams.set('tts_provider_id', analytics.connection.providerId)
|
||||
return u.toString()
|
||||
}
|
||||
|
||||
|
||||
@@ -46,6 +46,10 @@ function makePlaybackManagerStub<TAudio = AudioBuffer>(): PlaybackManagerSubset<
|
||||
|
||||
function makeStreamingSnapshot(overrides: Partial<StreamingSessionSnapshot> = {}): StreamingSessionSnapshot {
|
||||
return {
|
||||
connection: {
|
||||
credentialMode: 'official',
|
||||
providerId: 'official-provider-speech-streaming',
|
||||
},
|
||||
model: 'volcengine/seed-tts-2.0',
|
||||
voice: 'mock-voice',
|
||||
voiceType: 'official_selected',
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import type { IntentHandle, IntentOptions, PlaybackItem } from '@proj-airi/pipelines-audio'
|
||||
|
||||
import type { StreamingTtsConnection } from './streaming-connection'
|
||||
import type { StreamingTtsPipelineOptions } from './streaming-pipeline'
|
||||
|
||||
import { createStreamingTtsPipeline } from './streaming-pipeline'
|
||||
@@ -69,6 +70,7 @@ function fromIntent(intent: IntentHandleSubset): StageTtsSession {
|
||||
* for cancelling and re-opening.
|
||||
*/
|
||||
export interface StreamingSessionSnapshot {
|
||||
connection: StreamingTtsConnection
|
||||
model: string
|
||||
voice: string
|
||||
voiceType: 'official_default' | 'official_selected' | 'custom_configured' | 'voice_pack' | 'unknown'
|
||||
@@ -157,6 +159,7 @@ export function createStreamingTtsSession<TAudio = AudioBuffer>(
|
||||
let terminated = false
|
||||
|
||||
const handle = pipelineFactory({
|
||||
connection: snapshot.connection,
|
||||
model: snapshot.model,
|
||||
voice: snapshot.voice,
|
||||
ttsVoiceType: snapshot.voiceType,
|
||||
|
||||
@@ -12,7 +12,8 @@ import { useI18n } from 'vue-i18n'
|
||||
import { toXml } from 'xast-util-to-xml'
|
||||
import { x } from 'xastscript'
|
||||
|
||||
import { getDefaultSpeechModel, getDefaultStreamingModel, OFFICIAL_SPEECH_PROVIDER_ID, OFFICIAL_SPEECH_STREAMING_PROVIDER_ID, setupOfficialSpeechAutoPick } from '../../libs/providers/providers/official'
|
||||
import { getDefinedProvider } from '../../libs/providers/providers'
|
||||
import { getDefaultSpeechModel, OFFICIAL_SPEECH_PROVIDER_ID, OFFICIAL_SPEECH_STREAMING_PROVIDER_ID, setupOfficialSpeechAutoPick } from '../../libs/providers/providers/official'
|
||||
import { useProvidersStore } from '../providers'
|
||||
|
||||
export function toSignedPercent(value: number): string {
|
||||
@@ -141,18 +142,15 @@ export const useSpeechStore = defineStore('speech', () => {
|
||||
activeSpeechVoice.value = undefined
|
||||
}
|
||||
|
||||
// Streaming TTS voices are model-scoped: the server only returns recommended
|
||||
// voices for an explicit `?model=`. Ensure the active model is a valid
|
||||
// streaming model id so voice loading gets the right recommendations (parity
|
||||
// with the HTTP provider's auto-pick). Reseeds the server-curated default
|
||||
// both when no model is selected AND when `activeSpeechModel` still holds a
|
||||
// stale id from a previously-active provider (the global model ref is shared
|
||||
// across providers, and the per-surface reset may not have run yet). No-op
|
||||
// for non-streaming providers.
|
||||
// Streaming TTS voices are model-scoped. Resolve the default through the
|
||||
// active provider capability so official and BYOK transports share the same
|
||||
// state transition without Stage knowing provider ids.
|
||||
function ensureStreamingDefaultModel() {
|
||||
if (activeSpeechProvider.value !== OFFICIAL_SPEECH_STREAMING_PROVIDER_ID)
|
||||
const providerId = activeSpeechProvider.value
|
||||
const speechCapability = getDefinedProvider(providerId)?.capabilities?.speech
|
||||
if (speechCapability?.transport !== 'bidirectional-ws')
|
||||
return
|
||||
const streamingModels = providersStore.getModelsForProvider(OFFICIAL_SPEECH_STREAMING_PROVIDER_ID)
|
||||
const streamingModels = providersStore.getModelsForProvider(providerId)
|
||||
const hasValidSelection = !!activeSpeechModel.value && streamingModels.some(m => m.id === activeSpeechModel.value)
|
||||
if (hasValidSelection)
|
||||
return
|
||||
@@ -160,7 +158,7 @@ export const useSpeechStore = defineStore('speech', () => {
|
||||
// When no default can be resolved yet (catalog not loaded), clear it to ''
|
||||
// so callers pass `undefined` (server returns the full streaming catalog)
|
||||
// rather than forwarding a stale non-streaming model id as `?model=`.
|
||||
const nextModel = getDefaultStreamingModel() ?? streamingModels[0]?.id ?? ''
|
||||
const nextModel = speechCapability.getDefaultModel?.() ?? streamingModels[0]?.id ?? ''
|
||||
if (activeSpeechModel.value === nextModel)
|
||||
return
|
||||
activeSpeechModel.value = nextModel
|
||||
|
||||
@@ -4,7 +4,7 @@ import type { AuthInstance } from './libs/auth'
|
||||
import type { Database } from './libs/db'
|
||||
import type { Env } from './libs/env'
|
||||
import type { OtelInstance } from './otel'
|
||||
import type { StreamingTtsVoiceType } from './routes/audio-speech-ws/session'
|
||||
import type { StreamingTtsProviderId, StreamingTtsVoiceType } from './routes/audio-speech-ws/session'
|
||||
import type { ConfigKVService } from './services/adapters/config-kv'
|
||||
import type { AdminFluxGrantsService } from './services/domain/admin/flux-grants'
|
||||
import type { AdminRouterConfigService } from './services/domain/admin/router-config'
|
||||
@@ -222,6 +222,8 @@ export async function buildApp(deps: AppDeps) {
|
||||
trigger: c.req.query('tts_trigger') === 'auto' ? 'auto' : 'manual',
|
||||
source: parseTtsSource(c.req.query('tts_source'), 'audio.speech.ws'),
|
||||
voiceType: parseTtsVoiceType(c.req.query('tts_voice_type')),
|
||||
credentialMode: c.req.query('tts_credential_mode') === 'byok' ? 'byok' : 'official',
|
||||
providerId: parseStreamingTtsProviderId(c.req.query('tts_provider_id')),
|
||||
})
|
||||
}))
|
||||
|
||||
@@ -503,6 +505,19 @@ function parseTtsVoiceType(
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Bounds client-provided provider identity before it reaches product events.
|
||||
*/
|
||||
function parseStreamingTtsProviderId(value: string | undefined): StreamingTtsProviderId {
|
||||
switch (value) {
|
||||
case 'official-provider-speech-streaming':
|
||||
case 'volcengine-streaming':
|
||||
return value
|
||||
default:
|
||||
return 'unknown'
|
||||
}
|
||||
}
|
||||
|
||||
export type AppType = Awaited<ReturnType<typeof buildApp>>['app']
|
||||
|
||||
export async function createApp() {
|
||||
|
||||
@@ -23,8 +23,9 @@ export type { AudioSpeechWsHandlersOptions } from './types'
|
||||
* Expects:
|
||||
* - The route handler has already resolved auth via the `?token=` query
|
||||
* (see app.ts wiring) and passes a verified `userId` in.
|
||||
* - The client sends a `start` control frame first. The session validates the
|
||||
* requested streaming model and voice before dialing upstream.
|
||||
* - Official clients send a `start` control frame first. BYOK clients send an
|
||||
* AIRI-private `credentials` frame followed by `start`. The session consumes
|
||||
* credentials locally, then validates model and voice before dialing upstream.
|
||||
*
|
||||
* Returns:
|
||||
* - A function that takes `userId` and returns hono `WSEvents`. Each call
|
||||
|
||||
@@ -312,6 +312,66 @@ describe('audio-speech-ws route', () => {
|
||||
}))
|
||||
})
|
||||
|
||||
it('uses the client BYOK credential without forwarding it as a frame or charging Flux', async () => {
|
||||
upstream = await startMockUpstream([
|
||||
{ kind: 'json', payload: { event: 'session.finished', payload: { usage: { text_words: 21 } } } },
|
||||
])
|
||||
// A zero balance would reject the official path. BYOK must not run that
|
||||
// pre-flight or any final Flux accumulation.
|
||||
const deps = makeFakeDeps({ upstreamURL: upstream.url, restBaseURL: upstream.restBaseURL, fluxBalance: 0 })
|
||||
const handlers = createAudioSpeechWsHandlers(deps as any)
|
||||
const events = handlers('user-byok', {
|
||||
credentialMode: 'byok',
|
||||
providerId: 'volcengine-streaming',
|
||||
voiceType: 'custom_configured',
|
||||
})
|
||||
const client = makeMockClientWs()
|
||||
|
||||
await driveClientSession(events, client, [
|
||||
JSON.stringify({ event: 'credentials', provider: 'volcengine', api_key: 'user-owned-x-api-key' }),
|
||||
JSON.stringify({ event: 'start', model: 'volcengine/seed-tts-2.0', voice: 'mock' }),
|
||||
JSON.stringify({ event: 'text', text: 'BYOK streaming' }),
|
||||
JSON.stringify({ event: 'finish' }),
|
||||
])
|
||||
await new Promise(r => setTimeout(r, 200))
|
||||
|
||||
expect(upstream.observedAuth).toBe('Bearer user-owned-x-api-key')
|
||||
expect(upstream.receivedFrames).toHaveLength(3)
|
||||
expect(upstream.receivedFrames.map(frame => JSON.parse(frame.data as string).event)).toEqual(['start', 'text', 'finish'])
|
||||
expect(deps.fluxService.getFlux).not.toHaveBeenCalled()
|
||||
expect(deps.ttsMeter.assertCanAfford).not.toHaveBeenCalled()
|
||||
expect(deps.ttsMeter.accumulate).not.toHaveBeenCalled()
|
||||
expect(deps.productEventService.track).toHaveBeenCalledWith(expect.objectContaining({
|
||||
action: 'speech_succeeded',
|
||||
metadata: expect.objectContaining({
|
||||
credential_mode: 'byok',
|
||||
provider_id: 'volcengine-streaming',
|
||||
flux_consumed: 0,
|
||||
}),
|
||||
}))
|
||||
})
|
||||
|
||||
it('does not fall back to the operator key when a BYOK session omits credentials', async () => {
|
||||
upstream = await startMockUpstream([])
|
||||
const deps = makeFakeDeps({ upstreamURL: upstream.url, restBaseURL: upstream.restBaseURL, fluxBalance: 100 })
|
||||
const handlers = createAudioSpeechWsHandlers(deps as any)
|
||||
const events = handlers('user-byok', {
|
||||
credentialMode: 'byok',
|
||||
providerId: 'volcengine-streaming',
|
||||
})
|
||||
const client = makeMockClientWs()
|
||||
|
||||
await driveClientSession(events, client, [
|
||||
JSON.stringify({ event: 'start', model: 'volcengine/seed-tts-2.0', voice: 'mock' }),
|
||||
])
|
||||
|
||||
expect(client.closeCode).toBe(1008)
|
||||
expect(client.closeReason).toBe('invalid_credentials_frame')
|
||||
expect(upstream.observedAuth).toBeUndefined()
|
||||
expect(upstream.receivedFrames).toHaveLength(0)
|
||||
expect(deps.envelopeCrypto.decryptKey).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('refuses the session with insufficient_flux when the user is broke', async () => {
|
||||
upstream = await startMockUpstream([])
|
||||
const deps = makeFakeDeps({ upstreamURL: upstream.url, restBaseURL: upstream.restBaseURL, fluxBalance: 0 })
|
||||
|
||||
@@ -44,7 +44,7 @@ const tracer = trace.getTracer('audio-speech-ws')
|
||||
export interface AudioSpeechSessionState {
|
||||
/** Stores the accepted client websocket. */
|
||||
attachClient: (ws: WSContext) => void
|
||||
/** Reads config, checks balance, decrypts the upstream key, and dials upstream after the start frame is accepted. */
|
||||
/** Resolves the selected credential policy and dials upstream after the start frame is accepted. */
|
||||
dialUpstream: () => Promise<void>
|
||||
/** Forwards a client frame or queues it while the upstream connection opens. */
|
||||
handleClientMessage: (message: { data: unknown }, ws: WSContext) => void
|
||||
@@ -55,11 +55,15 @@ export interface AudioSpeechSessionState {
|
||||
export type StreamingTtsTrigger = 'auto' | 'manual'
|
||||
export type StreamingTtsSource = 'audio.speech.ws' | 'chat_auto_tts' | 'manual_preview' | 'settings_test'
|
||||
export type StreamingTtsVoiceType = 'official_default' | 'official_selected' | 'custom_configured' | 'voice_pack' | 'unknown'
|
||||
export type StreamingTtsCredentialMode = 'official' | 'byok'
|
||||
export type StreamingTtsProviderId = 'official-provider-speech-streaming' | 'volcengine-streaming' | 'unknown'
|
||||
|
||||
export interface AudioSpeechSessionAnalytics {
|
||||
trigger?: StreamingTtsTrigger
|
||||
source?: StreamingTtsSource
|
||||
voiceType?: StreamingTtsVoiceType
|
||||
credentialMode?: StreamingTtsCredentialMode
|
||||
providerId?: StreamingTtsProviderId
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -71,7 +75,9 @@ export interface AudioSpeechSessionAnalytics {
|
||||
* are handled at session end.
|
||||
*
|
||||
* Expects:
|
||||
* - `UNSPEECH_UPSTREAM.streaming` has a base URL and at least one encrypted key.
|
||||
* - `UNSPEECH_UPSTREAM.streaming` has a base URL.
|
||||
* - Official sessions have an operator key; BYOK sessions send one private
|
||||
* credentials frame before the UnSpeech start frame.
|
||||
*
|
||||
* Returns:
|
||||
* - A connection-scoped state object with no global peer registry.
|
||||
@@ -94,7 +100,7 @@ export function createSessionState(
|
||||
let upstreamWs: WebSocket | null = null
|
||||
let upstreamReady = false
|
||||
let closed = false
|
||||
let billed = false
|
||||
let completed = false
|
||||
let startFrameAccepted = false
|
||||
let startValidationStarted = false
|
||||
let dialStarted = false
|
||||
@@ -102,6 +108,8 @@ export function createSessionState(
|
||||
let preflightFluxBalance: number | undefined
|
||||
let modelLabel = STREAM_MODEL_LABEL_FALLBACK
|
||||
let voiceLabel: string | undefined
|
||||
let byokKey: Buffer | null = null
|
||||
let credentialFrameAccepted = analytics.credentialMode === 'official'
|
||||
/**
|
||||
* Frames the client sent before the upstream finished dialing. Buffered to
|
||||
* avoid silently dropping the `start` frame; flushed in arrival order once
|
||||
@@ -127,7 +135,7 @@ export function createSessionState(
|
||||
model: modelLabel,
|
||||
metadata: {
|
||||
trigger: analytics.trigger,
|
||||
...streamingVoiceMetadata(voiceLabel, analytics.voiceType),
|
||||
...streamingSessionMetadata(voiceLabel, analytics),
|
||||
},
|
||||
})
|
||||
|
||||
@@ -142,51 +150,63 @@ export function createSessionState(
|
||||
}
|
||||
|
||||
const upstreamConfig = unspeech?.streaming
|
||||
if (!upstreamConfig || !upstreamConfig.baseURL || upstreamConfig.keys.length === 0) {
|
||||
if (!upstreamConfig?.baseURL || (analytics.credentialMode === 'official' && upstreamConfig.keys.length === 0)) {
|
||||
closeWithError(1008, 'streaming_tts_not_configured')
|
||||
return
|
||||
}
|
||||
|
||||
// Pre-flight balance check: refuse before dialing if the user cannot
|
||||
// afford the worst-case session.
|
||||
try {
|
||||
const flux = await opts.fluxService.getFlux(userId)
|
||||
preflightFluxBalance = flux.flux
|
||||
await opts.ttsMeter.assertCanAfford(userId, STREAMING_PREFLIGHT_CHARS_ESTIMATE, flux.flux)
|
||||
}
|
||||
catch (err) {
|
||||
log.withError(err).withFields({ userId }).warn('pre-flight rejected streaming tts')
|
||||
// assertCanAfford throws PaymentRequiredError (402) — translate to ws
|
||||
// policy-violation close. The client can read the close code/reason to
|
||||
// surface a 'top up' prompt.
|
||||
if (isPaymentRequiredError(err))
|
||||
closeWithBlockedPreflight(1008, 'insufficient_flux')
|
||||
else
|
||||
closeWithError(1011, 'flux_preflight_failed')
|
||||
return
|
||||
if (analytics.credentialMode === 'official') {
|
||||
// Official sessions consume AIRI-hosted credentials and Flux. BYOK
|
||||
// sessions are explicitly excluded so a missing user key can never
|
||||
// fall back to the operator account or charge the user twice.
|
||||
try {
|
||||
const flux = await opts.fluxService.getFlux(userId)
|
||||
preflightFluxBalance = flux.flux
|
||||
await opts.ttsMeter.assertCanAfford(userId, STREAMING_PREFLIGHT_CHARS_ESTIMATE, flux.flux)
|
||||
}
|
||||
catch (err) {
|
||||
log.withError(err).withFields({ userId }).warn('pre-flight rejected streaming tts')
|
||||
// assertCanAfford throws PaymentRequiredError (402) — translate to ws
|
||||
// policy-violation close. The client can read the close code/reason to
|
||||
// surface a 'top up' prompt.
|
||||
if (isPaymentRequiredError(err))
|
||||
closeWithBlockedPreflight(1008, 'insufficient_flux')
|
||||
else
|
||||
closeWithError(1011, 'flux_preflight_failed')
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Decrypt the first key. Streaming surface does not do per-attempt key
|
||||
// rotation: a live ws cannot transparently switch upstream mid-session
|
||||
// without breaking audio continuity. Fallback policy belongs at the
|
||||
// session-retry layer (next client connect), not inline.
|
||||
const entry = upstreamConfig.keys[0]
|
||||
let keyPlaintext: Buffer
|
||||
try {
|
||||
keyPlaintext = opts.envelopeCrypto.decryptKey(entry.ciphertext, {
|
||||
modelName: STREAM_MODEL_LABEL_FALLBACK,
|
||||
keyEntryId: entry.id,
|
||||
})
|
||||
if (analytics.credentialMode === 'byok') {
|
||||
if (!byokKey) {
|
||||
closeWithError(1008, 'byok_credentials_required')
|
||||
return
|
||||
}
|
||||
keyPlaintext = byokKey
|
||||
byokKey = null
|
||||
}
|
||||
catch (err) {
|
||||
log.withError(err).withFields({ keyEntryId: entry.id }).error('decrypt failed for streaming tts key')
|
||||
closeWithError(1011, 'decrypt_failed')
|
||||
return
|
||||
else {
|
||||
// Streaming sessions do not rotate keys mid-connection because that
|
||||
// would break audio continuity. Retry policy belongs to the next client
|
||||
// connection, never to an in-flight session.
|
||||
const entry = upstreamConfig.keys[0]
|
||||
try {
|
||||
keyPlaintext = opts.envelopeCrypto.decryptKey(entry.ciphertext, {
|
||||
modelName: STREAM_MODEL_LABEL_FALLBACK,
|
||||
keyEntryId: entry.id,
|
||||
})
|
||||
}
|
||||
catch (err) {
|
||||
log.withError(err).withFields({ keyEntryId: entry.id }).error('decrypt failed for streaming tts key')
|
||||
closeWithError(1011, 'decrypt_failed')
|
||||
return
|
||||
}
|
||||
span.setAttribute(AIRI_ATTR_GEN_AI_GATEWAY_KEY_ID, entry.id)
|
||||
}
|
||||
|
||||
const upstreamURL = upstreamConfig.baseURL
|
||||
span.setAttribute(AIRI_ATTR_GEN_AI_GATEWAY_UPSTREAM_URL, upstreamURL)
|
||||
span.setAttribute(AIRI_ATTR_GEN_AI_GATEWAY_KEY_ID, entry.id)
|
||||
|
||||
let upstream: WebSocket
|
||||
try {
|
||||
@@ -196,6 +216,11 @@ export function createSessionState(
|
||||
},
|
||||
})
|
||||
}
|
||||
catch (err) {
|
||||
log.withError(err).withFields({ userId }).warn('failed to dial streaming tts upstream')
|
||||
closeWithError(1011, 'upstream_dial_failed')
|
||||
return
|
||||
}
|
||||
finally {
|
||||
// Wipe plaintext immediately — the ws lib has already serialized the
|
||||
// header into its outgoing handshake buffer.
|
||||
@@ -242,7 +267,7 @@ export function createSessionState(
|
||||
metadata: {
|
||||
duration_ms: Date.now() - startedAt,
|
||||
trigger: analytics.trigger,
|
||||
...streamingVoiceMetadata(voiceLabel, analytics.voiceType),
|
||||
...streamingSessionMetadata(voiceLabel, analytics),
|
||||
},
|
||||
})
|
||||
try {
|
||||
@@ -270,6 +295,26 @@ export function createSessionState(
|
||||
? Buffer.from(message.data)
|
||||
: Buffer.from(message.data as ArrayBufferLike)
|
||||
|
||||
if (!credentialFrameAccepted) {
|
||||
if (isBinary || typeof payload !== 'string') {
|
||||
closeWithError(1008, 'invalid_credentials_frame')
|
||||
return
|
||||
}
|
||||
|
||||
const credentials = parseCredentialsFrame(payload)
|
||||
if (!credentials) {
|
||||
closeWithError(1008, 'invalid_credentials_frame')
|
||||
return
|
||||
}
|
||||
|
||||
// This AIRI-private frame is deliberately consumed here and never
|
||||
// placed in pendingClientFrames, so the upstream UnSpeech protocol can
|
||||
// never observe or log the plaintext credential as an application frame.
|
||||
byokKey = Buffer.from(credentials.apiKey, 'utf8')
|
||||
credentialFrameAccepted = true
|
||||
return
|
||||
}
|
||||
|
||||
if (!startValidationStarted) {
|
||||
if (isBinary || typeof payload !== 'string') {
|
||||
closeWithError(1008, 'invalid_start_frame')
|
||||
@@ -382,10 +427,7 @@ export function createSessionState(
|
||||
// the client-text-frame estimate accumulated in handleClientMessage.
|
||||
const usageChars = readUsageChars(evt.payload)
|
||||
const billUnits = usageChars ?? totalInputChars
|
||||
if (billUnits > 0)
|
||||
void billSession(billUnits, 'session.finished')
|
||||
else
|
||||
finalize()
|
||||
void completeSession(billUnits, 'session.finished')
|
||||
break
|
||||
}
|
||||
case 'error': {
|
||||
@@ -433,7 +475,7 @@ export function createSessionState(
|
||||
}
|
||||
|
||||
const upstreamConfig = unspeech?.streaming
|
||||
if (!unspeech?.restBaseURL || !upstreamConfig?.baseURL || upstreamConfig.keys.length === 0) {
|
||||
if (!unspeech?.restBaseURL || !upstreamConfig?.baseURL || (analytics.credentialMode === 'official' && upstreamConfig.keys.length === 0)) {
|
||||
closeWithError(1008, 'streaming_tts_not_configured')
|
||||
return false
|
||||
}
|
||||
@@ -470,42 +512,44 @@ export function createSessionState(
|
||||
return true
|
||||
}
|
||||
|
||||
async function billSession(units: number, reason: string) {
|
||||
if (billed)
|
||||
async function completeSession(units: number, reason: string) {
|
||||
if (completed)
|
||||
return
|
||||
billed = true
|
||||
completed = true
|
||||
span.setAttribute(GEN_AI_ATTR_REQUEST_MODEL, modelLabel)
|
||||
|
||||
let flux: Awaited<ReturnType<FluxService['getFlux']>>
|
||||
try {
|
||||
flux = await opts.fluxService.getFlux(userId)
|
||||
}
|
||||
catch (err) {
|
||||
log.withError(err).withFields({ userId }).warn('flux read failed at session end')
|
||||
finalize()
|
||||
return
|
||||
}
|
||||
|
||||
let fluxConsumed = 0
|
||||
try {
|
||||
const result = await otelContext.with(trace.setSpan(otelContext.active(), span), () =>
|
||||
opts.ttsMeter.accumulate({
|
||||
userId,
|
||||
units,
|
||||
currentBalance: flux.flux,
|
||||
requestId,
|
||||
metadata: { model: modelLabel },
|
||||
}))
|
||||
fluxConsumed = result.fluxDebited
|
||||
span.setAttribute(AIRI_ATTR_BILLING_FLUX_CONSUMED, fluxConsumed)
|
||||
}
|
||||
catch (err) {
|
||||
// Billing failure is surfaced but does not retroactively reject the
|
||||
// already-delivered audio — the user got the audio, the meter retains
|
||||
// the debt for the next request to settle (per FluxMeter rollback path).
|
||||
log.withError(err).withFields({ userId, units, reason }).error('billing accumulate failed for streaming tts')
|
||||
span.recordException(err as Error)
|
||||
span.setStatus({ code: SpanStatusCode.ERROR, message: 'billing_failed' })
|
||||
if (analytics.credentialMode === 'official' && units > 0) {
|
||||
let flux: Awaited<ReturnType<FluxService['getFlux']>>
|
||||
try {
|
||||
flux = await opts.fluxService.getFlux(userId)
|
||||
}
|
||||
catch (err) {
|
||||
log.withError(err).withFields({ userId }).warn('flux read failed at session end')
|
||||
finalize()
|
||||
return
|
||||
}
|
||||
|
||||
try {
|
||||
const result = await otelContext.with(trace.setSpan(otelContext.active(), span), () =>
|
||||
opts.ttsMeter.accumulate({
|
||||
userId,
|
||||
units,
|
||||
currentBalance: flux.flux,
|
||||
requestId,
|
||||
metadata: { model: modelLabel },
|
||||
}))
|
||||
fluxConsumed = result.fluxDebited
|
||||
span.setAttribute(AIRI_ATTR_BILLING_FLUX_CONSUMED, fluxConsumed)
|
||||
}
|
||||
catch (err) {
|
||||
// Billing failure is surfaced but does not retroactively reject the
|
||||
// already-delivered audio — the user got the audio, the meter retains
|
||||
// the debt for the next request to settle (per FluxMeter rollback path).
|
||||
log.withError(err).withFields({ userId, units, reason }).error('billing accumulate failed for streaming tts')
|
||||
span.recordException(err as Error)
|
||||
span.setStatus({ code: SpanStatusCode.ERROR, message: 'billing_failed' })
|
||||
}
|
||||
}
|
||||
|
||||
const durationMs = Date.now() - startedAt
|
||||
@@ -534,7 +578,7 @@ export function createSessionState(
|
||||
duration_ms: durationMs,
|
||||
flux_consumed: fluxConsumed,
|
||||
trigger: analytics.trigger,
|
||||
...streamingVoiceMetadata(voiceLabel, analytics.voiceType),
|
||||
...streamingSessionMetadata(voiceLabel, analytics),
|
||||
},
|
||||
})
|
||||
|
||||
@@ -545,6 +589,7 @@ export function createSessionState(
|
||||
if (closed)
|
||||
return
|
||||
closed = true
|
||||
wipeByokKey()
|
||||
try {
|
||||
upstreamWs?.close()
|
||||
}
|
||||
@@ -572,7 +617,7 @@ export function createSessionState(
|
||||
close_code: code,
|
||||
duration_ms: Date.now() - startedAt,
|
||||
trigger: analytics.trigger,
|
||||
...streamingVoiceMetadata(voiceLabel, analytics.voiceType),
|
||||
...streamingSessionMetadata(voiceLabel, analytics),
|
||||
},
|
||||
})
|
||||
if (clientWs) {
|
||||
@@ -586,6 +631,7 @@ export function createSessionState(
|
||||
catch {}
|
||||
}
|
||||
closed = true
|
||||
wipeByokKey()
|
||||
span.end()
|
||||
}
|
||||
|
||||
@@ -608,7 +654,7 @@ export function createSessionState(
|
||||
close_code: code,
|
||||
duration_ms: Date.now() - startedAt,
|
||||
trigger: analytics.trigger,
|
||||
...streamingVoiceMetadata(voiceLabel, analytics.voiceType),
|
||||
...streamingSessionMetadata(voiceLabel, analytics),
|
||||
},
|
||||
})
|
||||
if (clientWs) {
|
||||
@@ -622,9 +668,15 @@ export function createSessionState(
|
||||
catch {}
|
||||
}
|
||||
closed = true
|
||||
wipeByokKey()
|
||||
span.end()
|
||||
}
|
||||
|
||||
function wipeByokKey() {
|
||||
byokKey?.fill(0)
|
||||
byokKey = null
|
||||
}
|
||||
|
||||
return {
|
||||
attachClient,
|
||||
dialUpstream,
|
||||
@@ -638,6 +690,8 @@ function normalizeAnalytics(input: AudioSpeechSessionAnalytics): Required<AudioS
|
||||
trigger: normalizeTrigger(input.trigger),
|
||||
source: normalizeSource(input.source),
|
||||
voiceType: normalizeVoiceType(input.voiceType),
|
||||
credentialMode: input.credentialMode === 'byok' ? 'byok' : 'official',
|
||||
providerId: normalizeProviderId(input.providerId),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -682,6 +736,27 @@ function streamingVoiceMetadata(voiceId: string | undefined, voiceType: Streamin
|
||||
}
|
||||
}
|
||||
|
||||
function streamingSessionMetadata(
|
||||
voiceId: string | undefined,
|
||||
analytics: Required<AudioSpeechSessionAnalytics>,
|
||||
): Record<string, unknown> {
|
||||
return {
|
||||
...streamingVoiceMetadata(voiceId, analytics.voiceType),
|
||||
credential_mode: analytics.credentialMode,
|
||||
provider_id: analytics.providerId,
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeProviderId(providerId: AudioSpeechSessionAnalytics['providerId']): StreamingTtsProviderId {
|
||||
switch (providerId) {
|
||||
case 'official-provider-speech-streaming':
|
||||
case 'volcengine-streaming':
|
||||
return providerId
|
||||
default:
|
||||
return 'unknown'
|
||||
}
|
||||
}
|
||||
|
||||
function isPaymentRequiredError(err: unknown): boolean {
|
||||
if (err instanceof ApiError)
|
||||
return err.statusCode === 402
|
||||
@@ -697,6 +772,29 @@ interface StreamingTtsStartFrame {
|
||||
voice: string
|
||||
}
|
||||
|
||||
interface StreamingTtsCredentialsFrame {
|
||||
event: 'credentials'
|
||||
provider: 'volcengine'
|
||||
apiKey: string
|
||||
}
|
||||
|
||||
function parseCredentialsFrame(rawText: string): StreamingTtsCredentialsFrame | null {
|
||||
try {
|
||||
const parsed = JSON.parse(rawText) as Record<string, unknown>
|
||||
if (parsed.event !== 'credentials' || parsed.provider !== 'volcengine')
|
||||
return null
|
||||
if (typeof parsed.api_key !== 'string')
|
||||
return null
|
||||
const apiKey = parsed.api_key.trim()
|
||||
if (apiKey.length === 0 || apiKey.length > 4096)
|
||||
return null
|
||||
return { event: 'credentials', provider: 'volcengine', apiKey }
|
||||
}
|
||||
catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
function parseStartFrame(rawText: string): StreamingTtsStartFrame | null {
|
||||
try {
|
||||
const parsed = JSON.parse(rawText) as Record<string, unknown>
|
||||
|
||||
@@ -9,13 +9,13 @@ import type { EnvelopeCrypto } from '../../utils/envelope-crypto'
|
||||
* Dependencies required by the streaming speech websocket proxy.
|
||||
*/
|
||||
export interface AudioSpeechWsHandlersOptions {
|
||||
/** Reads upstream websocket URL and encrypted API keys. */
|
||||
/** Reads the UnSpeech websocket URL and operator credentials for official sessions. */
|
||||
configKV: ConfigKVService
|
||||
/** Decrypts the selected upstream API key before the websocket handshake. */
|
||||
envelopeCrypto: EnvelopeCrypto
|
||||
/** Reads the user's current Flux balance for pre-flight and final billing. */
|
||||
/** Reads the user's current Flux balance for official-session billing. */
|
||||
fluxService: FluxService
|
||||
/** Applies pre-flight affordability checks and final streaming TTS billing. */
|
||||
/** Applies pre-flight checks and final billing to official sessions only. */
|
||||
ttsMeter: FluxMeter
|
||||
/** Persists request accounting after a stream finishes. */
|
||||
requestLogService: RequestLogService
|
||||
|
||||
Reference in New Issue
Block a user