feat(tts): add Volcengine streaming BYOK

This commit is contained in:
RainbowBird
2026-08-04 17:15:17 +08:00
parent f66a18956b
commit efdf13c24c
32 changed files with 968 additions and 102 deletions
@@ -1405,6 +1405,20 @@ pages:
description: App ID of the project where you can obtain in Console
label: App ID
title: Volcano Engine
volcengine-streaming:
title: Volcengine Streaming TTS
description: Low-latency Volcengine TTS through the AIRI UnSpeech bridge
login-required: Sign in to AIRI so the browser can securely use the WebSocket proxy.
config-description: Configure the X-Api-Key from the Volcengine new console. The existing Volcengine V1 provider is unchanged.
model-placeholder: Choose a streaming model
privacy-title: How your key is used
privacy-description: Your key stays in this device's provider settings. For each WSS session, AIRI forwards it in memory only to authenticate UnSpeech and does not store it server-side.
preview-text: Hello, this is a preview of Volcengine bidirectional streaming speech synthesis.
fields:
api-key:
label: X-Api-Key
description: X-Api-Key from the Volcengine new console. App ID and the old-console Access Key are not supported by this streaming endpoint.
placeholder: Enter your X-Api-Key
volcengine-coding-plan:
description: Volcengine Coding Plan
title: Volcengine Coding Plan
@@ -1350,6 +1350,20 @@ pages:
description: ID de App del proyecto que puedes obtener en la Consola
label: ID de App
title: Volcano Engine
volcengine-streaming:
title: Volcengine Streaming TTS
description: Low-latency Volcengine TTS through the AIRI UnSpeech bridge
login-required: Sign in to AIRI so the browser can securely use the WebSocket proxy.
config-description: Configure the X-Api-Key from the Volcengine new console. The existing Volcengine V1 provider is unchanged.
model-placeholder: Choose a streaming model
privacy-title: How your key is used
privacy-description: Your key stays in this device's provider settings. AIRI only forwards it in memory for each WSS session and does not store it server-side.
preview-text: Hello, this is a preview of Volcengine bidirectional streaming speech synthesis.
fields:
api-key:
label: X-Api-Key
description: X-Api-Key from the Volcengine new console. App ID and old-console Access Key are not supported.
placeholder: Enter your X-Api-Key
volcengine-coding-plan:
description: Plan de Codificación Volcengine
title: Plan de Codificación Volcengine
@@ -1350,6 +1350,20 @@ pages:
description: ID de lapplication du projet, que vous pouvez obtenir dans la Console
label: ID de lapplication
title: Volcano Engine
volcengine-streaming:
title: Volcengine Streaming TTS
description: Low-latency Volcengine TTS through the AIRI UnSpeech bridge
login-required: Sign in to AIRI so the browser can securely use the WebSocket proxy.
config-description: Configure the X-Api-Key from the Volcengine new console. The existing Volcengine V1 provider is unchanged.
model-placeholder: Choose a streaming model
privacy-title: How your key is used
privacy-description: Your key stays in this device's provider settings. AIRI only forwards it in memory for each WSS session and does not store it server-side.
preview-text: Hello, this is a preview of Volcengine bidirectional streaming speech synthesis.
fields:
api-key:
label: X-Api-Key
description: X-Api-Key from the Volcengine new console. App ID and old-console Access Key are not supported.
placeholder: Enter your X-Api-Key
volcengine-coding-plan:
description: Volcengine Coding Plan
title: Volcengine Coding Plan
@@ -1350,6 +1350,20 @@ pages:
description: コンソールで取得できるプロジェクトのApp ID
label: App ID
title: Volcano Engine
volcengine-streaming:
title: Volcengine Streaming TTS
description: Low-latency Volcengine TTS through the AIRI UnSpeech bridge
login-required: Sign in to AIRI so the browser can securely use the WebSocket proxy.
config-description: Configure the X-Api-Key from the Volcengine new console. The existing Volcengine V1 provider is unchanged.
model-placeholder: Choose a streaming model
privacy-title: How your key is used
privacy-description: Your key stays in this device's provider settings. AIRI only forwards it in memory for each WSS session and does not store it server-side.
preview-text: Hello, this is a preview of Volcengine bidirectional streaming speech synthesis.
fields:
api-key:
label: X-Api-Key
description: X-Api-Key from the Volcengine new console. App ID and old-console Access Key are not supported.
placeholder: Enter your X-Api-Key
volcengine-coding-plan:
description: Volcengine Coding Plan
title: Volcengine Coding Plan
@@ -1350,6 +1350,20 @@ pages:
description: 콘솔에서 얻을 수 있는 프로젝트의 애플리케이션 ID
label: 애플리케이션 ID
title: Volcano 엔진
volcengine-streaming:
title: Volcengine Streaming TTS
description: Low-latency Volcengine TTS through the AIRI UnSpeech bridge
login-required: Sign in to AIRI so the browser can securely use the WebSocket proxy.
config-description: Configure the X-Api-Key from the Volcengine new console. The existing Volcengine V1 provider is unchanged.
model-placeholder: Choose a streaming model
privacy-title: How your key is used
privacy-description: Your key stays in this device's provider settings. AIRI only forwards it in memory for each WSS session and does not store it server-side.
preview-text: Hello, this is a preview of Volcengine bidirectional streaming speech synthesis.
fields:
api-key:
label: X-Api-Key
description: X-Api-Key from the Volcengine new console. App ID and old-console Access Key are not supported.
placeholder: Enter your X-Api-Key
volcengine-coding-plan:
description: Volcengine Coding Plan
title: Volcengine Coding Plan
@@ -1350,6 +1350,20 @@ pages:
description: App ID проекта (получается в Console)
label: Идентификатор приложения (App ID)
title: Volcano Engine
volcengine-streaming:
title: Volcengine Streaming TTS
description: Low-latency Volcengine TTS through the AIRI UnSpeech bridge
login-required: Sign in to AIRI so the browser can securely use the WebSocket proxy.
config-description: Configure the X-Api-Key from the Volcengine new console. The existing Volcengine V1 provider is unchanged.
model-placeholder: Choose a streaming model
privacy-title: How your key is used
privacy-description: Your key stays in this device's provider settings. AIRI only forwards it in memory for each WSS session and does not store it server-side.
preview-text: Hello, this is a preview of Volcengine bidirectional streaming speech synthesis.
fields:
api-key:
label: X-Api-Key
description: X-Api-Key from the Volcengine new console. App ID and old-console Access Key are not supported.
placeholder: Enter your X-Api-Key
volcengine-coding-plan:
description: План кодирования Volcengine
title: План кодирования Volcengine
@@ -1350,6 +1350,20 @@ pages:
description: App ID của dự án (lấy trong Console)
label: App ID
title: Volcano Engine
volcengine-streaming:
title: Volcengine Streaming TTS
description: Low-latency Volcengine TTS through the AIRI UnSpeech bridge
login-required: Sign in to AIRI so the browser can securely use the WebSocket proxy.
config-description: Configure the X-Api-Key from the Volcengine new console. The existing Volcengine V1 provider is unchanged.
model-placeholder: Choose a streaming model
privacy-title: How your key is used
privacy-description: Your key stays in this device's provider settings. AIRI only forwards it in memory for each WSS session and does not store it server-side.
preview-text: Hello, this is a preview of Volcengine bidirectional streaming speech synthesis.
fields:
api-key:
label: X-Api-Key
description: X-Api-Key from the Volcengine new console. App ID and old-console Access Key are not supported.
placeholder: Enter your X-Api-Key
volcengine-coding-plan:
description: Volcengine Coding Plan
title: Volcengine Coding Plan
@@ -1350,6 +1350,20 @@ pages:
description: 可在控制台获取的 App ID
label: App ID
title: 火山引擎
volcengine-streaming:
title: 火山引擎(双向流式 TTS
description: 通过 AIRI UnSpeech 代理使用低延迟火山引擎语音合成
login-required: 请先登录 AIRI,以便浏览器安全使用 WebSocket 代理。
config-description: 配置火山引擎新控制台的 X-Api-Key。现有火山引擎 V1 提供商不会受到影响。
model-placeholder: 选择流式模型
privacy-title: 密钥使用方式
privacy-description: 密钥保存在当前设备的提供商设置中。每次 WSS 会话中,AIRI 仅在内存中转发密钥用于 UnSpeech 鉴权,不会在服务端持久化。
preview-text: 你好,这是火山引擎双向流式语音合成的试听样例。
fields:
api-key:
label: X-Api-Key
description: 火山引擎新控制台提供的 X-Api-Key。此流式接口不支持 App ID 与旧控制台 Access Key。
placeholder: 输入 X-Api-Key
volcengine-coding-plan:
description: 火山引擎 Coding Plan
title: 火山引擎 Coding Plan
@@ -1350,6 +1350,20 @@ pages:
description: 可在控制台取得的 App ID
label: 應用程式 ID
title: 火山引擎
volcengine-streaming:
title: 火山引擎(雙向串流 TTS
description: 透過 AIRI UnSpeech 代理使用低延遲火山引擎語音合成
login-required: 請先登入 AIRI,讓瀏覽器安全使用 WebSocket 代理。
config-description: 設定火山引擎新控制台的 X-Api-Key。現有火山引擎 V1 提供商不受影響。
model-placeholder: 選擇串流模型
privacy-title: 金鑰使用方式
privacy-description: 金鑰保存在目前裝置的提供商設定中。每次 WSS 工作階段中,AIRI 僅在記憶體轉發金鑰供 UnSpeech 驗證,不會儲存在伺服器。
preview-text: 你好,這是火山引擎雙向串流語音合成的試聽範例。
fields:
api-key:
label: X-Api-Key
description: 火山引擎新控制台提供的 X-Api-Key。此串流介面不支援 App ID 與舊控制台 Access Key。
placeholder: 輸入 X-Api-Key
volcengine-coding-plan:
description: Volcengine Coding Plan
title: Volcengine Coding Plan
@@ -97,6 +97,10 @@ async function handleGenerateSpeech(input: string, voiceId: string, _useSSML: bo
throw new Error(`Streaming model id missing backend prefix: ${requestedModel}`)
const apiResourceId = requestedModel.slice(slashIndex + 1)
const result = await streamingSynthesize({
connection: {
credentialMode: 'official',
providerId,
},
model: requestedModel,
voice: voiceId,
input,
@@ -0,0 +1,187 @@
<script setup lang="ts">
import {
ProviderApiKeyInput,
ProviderBasicSettings,
ProviderSettingsContainer,
ProviderSettingsLayout,
SpeechPlayground,
} from '@proj-airi/stage-ui/components'
import { getVolcengineStreamingDefaultModel, streamingSynthesize, VOLCENGINE_STREAMING_PROVIDER_ID } from '@proj-airi/stage-ui/libs'
import { useAuthStore } from '@proj-airi/stage-ui/stores/auth'
import { useSpeechStore } from '@proj-airi/stage-ui/stores/modules/speech'
import { useProvidersStore } from '@proj-airi/stage-ui/stores/providers'
import { Button, Callout, ComboboxSelect } from '@proj-airi/ui'
import { storeToRefs } from 'pinia'
import { computed, onMounted, ref, watch } from 'vue'
import { useI18n } from 'vue-i18n'
import { useRouter } from 'vue-router'
const router = useRouter()
const { t } = useI18n()
const authStore = useAuthStore()
const providersStore = useProvidersStore()
const speechStore = useSpeechStore()
const { isAuthenticated, needsLogin } = storeToRefs(authStore)
const { providers } = storeToRefs(providersStore)
const providerId = VOLCENGINE_STREAMING_PROVIDER_ID
const providerMetadata = computed(() => providersStore.getProviderMetadata(providerId))
const providerConfig = computed(() => providersStore.getProviderConfig(providerId))
const apiKey = computed({
get: () => (providers.value[providerId]?.apiKey as string | undefined) ?? '',
set: (value: string) => {
providers.value[providerId] ??= {}
providers.value[providerId].apiKey = value
},
})
const apiKeyConfigured = computed(() => apiKey.value.trim().length > 0)
const providerModels = computed(() => providersStore.getModelsForProvider(providerId))
const modelsLoading = computed(() => providersStore.isLoadingModels[providerId] || false)
const discoveredDefaultModel = ref<string | null>(null)
const model = computed({
get: () => (providerConfig.value?.model as string | undefined) ?? discoveredDefaultModel.value ?? '',
set: (value: string) => {
providers.value[providerId] ??= {}
providers.value[providerId].model = value
},
})
const modelOptions = computed(() => providerModels.value.map(item => ({ label: item.name, value: item.id })))
const availableVoices = computed(() => speechStore.availableVoices[providerId] || [])
const voicesLoading = ref(false)
async function loadVoices() {
if (!isAuthenticated.value || !model.value)
return
voicesLoading.value = true
try {
await speechStore.loadVoicesForProvider(providerId, model.value)
}
finally {
voicesLoading.value = false
}
}
async function loadCatalog() {
if (!isAuthenticated.value)
return
await providersStore.fetchModelsForProvider(providerId)
discoveredDefaultModel.value = getVolcengineStreamingDefaultModel() ?? providerModels.value[0]?.id ?? null
if (!providerConfig.value.model && discoveredDefaultModel.value)
model.value = discoveredDefaultModel.value
await loadVoices()
}
async function handleGenerateSpeech(input: string, voiceId: string): Promise<ArrayBuffer> {
const requestedModel = model.value
const key = apiKey.value.trim()
if (!key)
throw new Error('X-Api-Key is required.')
if (!requestedModel.includes('/'))
throw new Error(`Streaming model id missing backend prefix: ${requestedModel}`)
const resourceId = requestedModel.split('/', 2)[1]
const result = await streamingSynthesize({
connection: {
credentialMode: 'byok',
providerId,
apiKey: key,
},
model: requestedModel,
voice: voiceId,
input,
ttsSource: 'manual_preview',
ttsVoiceType: 'custom_configured',
extraBody: {
api_resource_id: resourceId,
audio: { sample_rate: 24000, bit_rate: 64000 },
},
})
return result.audio
}
function handleLogin() {
needsLogin.value = true
}
onMounted(async () => {
providersStore.initializeProvider(providerId)
await loadCatalog()
})
watch(isAuthenticated, async (authenticated) => {
if (authenticated)
await loadCatalog()
})
watch(model, async () => {
await loadVoices()
})
</script>
<template>
<ProviderSettingsLayout
:provider-name="providerMetadata.localizedName"
:provider-icon-color="providerMetadata.iconColor"
:on-back="() => router.back()"
>
<div v-if="!isAuthenticated" :class="['mx-auto max-w-2xl', 'flex flex-col gap-4']">
<Callout theme="primary" :label="providerMetadata.localizedName">
<div :class="['flex flex-col gap-3']">
<p>{{ t('settings.pages.providers.provider.volcengine-streaming.login-required') }}</p>
<Button class="w-fit" @click="handleLogin">
{{ t('settings.dialogs.onboarding.loginAction') }}
</Button>
</div>
</Callout>
</div>
<div v-else :class="['flex flex-col gap-6 md:flex-row']">
<ProviderSettingsContainer class="w-full md:w-[40%]">
<ProviderBasicSettings
:title="t('settings.pages.providers.common.section.basic.title')"
:description="t('settings.pages.providers.provider.volcengine-streaming.config-description')"
>
<ProviderApiKeyInput
v-model="apiKey"
:provider-name="providerMetadata.localizedName"
:label="t('settings.pages.providers.provider.volcengine-streaming.fields.api-key.label')"
:description="t('settings.pages.providers.provider.volcengine-streaming.fields.api-key.description')"
:placeholder="t('settings.pages.providers.provider.volcengine-streaming.fields.api-key.placeholder')"
required
/>
<ComboboxSelect
v-model="model"
:options="modelOptions"
:disabled="modelsLoading"
:placeholder="t('settings.pages.providers.provider.volcengine-streaming.model-placeholder')"
/>
</ProviderBasicSettings>
<Callout :label="t('settings.pages.providers.provider.volcengine-streaming.privacy-title')">
<p>{{ t('settings.pages.providers.provider.volcengine-streaming.privacy-description') }}</p>
</Callout>
</ProviderSettingsContainer>
<div class="w-full md:w-[60%]">
<SpeechPlayground
:available-voices="availableVoices"
:generate-speech="handleGenerateSpeech"
:api-key-configured="apiKeyConfigured"
:voices-loading="voicesLoading"
:default-text="t('settings.pages.providers.provider.volcengine-streaming.preview-text')"
/>
</div>
</div>
</ProviderSettingsLayout>
</template>
<route lang="yaml">
meta:
layout: settings
stageTransition:
name: slide
</route>
@@ -36,7 +36,7 @@ import { useIOTraceBridge } from '../../composables/use-io-trace-bridge'
import { initIOTracer } from '../../composables/use-io-tracer'
import { useSpeechPipelineAnalytics } from '../../composables/use-speech-pipeline-analytics'
import { Emotion, EMOTION_EmotionMotionName_value, EMOTION_VRMExpressionName_value, EmotionThinkMotionName } from '../../constants/emotions'
import { getDefaultStreamingModel, getDefinedProvider } from '../../libs/providers/providers'
import { getDefinedProvider } from '../../libs/providers/providers'
import { OFFICIAL_SPEECH_PROVIDER_ID, OFFICIAL_SPEECH_STREAMING_PROVIDER_ID } from '../../libs/providers/providers/official'
import { bindSpeakingStateToPlaybackManager } from '../../libs/speech/playback-speaking-state'
import { createStageTtsSession } from '../../libs/speech/tts-session'
@@ -697,17 +697,31 @@ function stopSpeechOutput(reason: string) {
resetAssistantSpeechSurface(reason)
}
/**
* Resolves the official streaming TTS model for the current Stage session.
*/
function resolveStreamingSessionModel(): string | null {
const activeModel = activeSpeechModel.value as string | undefined
const sessionModel = activeModel?.includes('/') ? activeModel : getDefaultStreamingModel()
const definition = activeSpeechProvider.value ? getDefinedProvider(activeSpeechProvider.value) : undefined
const sessionModel = activeModel?.includes('/')
? activeModel
: definition?.capabilities?.speech?.getDefaultModel?.()
if (!sessionModel?.includes('/'))
return null
return sessionModel
}
function resolveStreamingConnection() {
const providerId = activeSpeechProvider.value
if (!providerId)
return null
const definition = getDefinedProvider(providerId)
const resolver = definition?.capabilities?.speech?.resolveConnection
if (!resolver)
return null
const connection = resolver(providersStore.getProviderConfig(providerId) ?? {})
if (connection.credentialMode === 'byok' && !connection.apiKey?.trim())
return null
return connection
}
function buildStreamingSnapshot(turnId: string): StreamingSessionSnapshot | null {
if (speechMuted.value)
return null
@@ -730,12 +744,16 @@ function buildStreamingSnapshot(turnId: string): StreamingSessionSnapshot | null
const sessionModel = resolveStreamingSessionModel()
if (!sessionModel)
return null
const connection = resolveStreamingConnection()
if (!connection)
return null
const apiResourceId = sessionModel.split('/', 2)[1]
// TTS 2.0 / ICL 2.0 ship subtitles asynchronously relative to audio
// (per the wire spec), so chunk-on-sentence-end would drop frames.
// Buffer the entire session and decode at session.finished instead.
const bufferEntireSession = apiResourceId.startsWith('seed-tts-2.0') || apiResourceId.startsWith('seed-icl-2.0')
return {
connection,
model: sessionModel,
voice: voiceId,
voiceType: resolveStageVoiceType(),
+1
View File
@@ -1,6 +1,7 @@
export * from './audio/manager'
export * from './color-from-element'
export * from './providers'
export * from './speech/streaming-connection'
export * from './speech/streaming-pipeline'
export * from './speech/streaming-session'
export * from './speech/tts-session'
@@ -6,6 +6,7 @@ import './azure-openai'
import './openai-compatible'
import './atlascloud'
import './volcengine-coding-plan'
import './volcengine-streaming'
import './byteplus'
import './byteplus-coding-plan'
import './n1n'
@@ -46,3 +47,5 @@ export {
getDefinedProvider,
listProviders,
} from './registry'
export { getVolcengineStreamingDefaultModel, VOLCENGINE_STREAMING_PROVIDER_ID } from './volcengine-streaming'
@@ -230,7 +230,14 @@ export const providerOfficialSpeechStreaming = defineProvider({
// session adapter (`tts-session.ts`) picks the streaming path without
// hard-coding provider id. Default for every other provider is `'rest'`.
capabilities: {
speech: { transport: 'bidirectional-ws' },
speech: {
transport: 'bidirectional-ws',
resolveConnection: () => ({
credentialMode: 'official',
providerId: OFFICIAL_SPEECH_STREAMING_PROVIDER_ID,
}),
getDefaultModel: getDefaultStreamingModel,
},
},
createProviderConfig: () => officialConfigSchema,
createProvider(_config) {
@@ -0,0 +1,33 @@
import { describe, expect, it } from 'vitest'
import { providerVolcengineStreaming, VOLCENGINE_STREAMING_PROVIDER_ID } from './index'
describe('volcengine streaming provider', () => {
it('keeps BYOK separate from the existing volcengine provider identity', () => {
expect(VOLCENGINE_STREAMING_PROVIDER_ID).toBe('volcengine-streaming')
expect(providerVolcengineStreaming.id).toBe('volcengine-streaming')
expect(providerVolcengineStreaming.capabilities?.speech?.transport).toBe('bidirectional-ws')
const connection = providerVolcengineStreaming.capabilities?.speech?.resolveConnection?.({
apiKey: ' user-x-api-key ',
})
expect(connection).toEqual({
credentialMode: 'byok',
providerId: 'volcengine-streaming',
apiKey: 'user-x-api-key',
})
})
it('requires an X-Api-Key before the provider becomes configured', async () => {
const validator = providerVolcengineStreaming.validators?.validateConfig?.[0]?.({ t: input => input })
expect(validator).toBeDefined()
const missing = await validator!.validator({ apiKey: '' }, { t: input => input })
const configured = await validator!.validator({ apiKey: 'user-key' }, { t: input => input })
expect(missing.valid).toBe(false)
expect(missing.reason).toBe('X-Api-Key is required.')
expect(configured.valid).toBe(true)
expect(configured.reason).toBe('')
})
})
@@ -0,0 +1,148 @@
import type { ModelInfo, VoiceInfo } from '../../types'
import { createUnVolcengine } from 'unspeech'
import { z } from 'zod'
import { getAuthToken } from '../../../auth'
import { SERVER_URL } from '../../../server'
import { defineProvider } from '../registry'
export const VOLCENGINE_STREAMING_PROVIDER_ID = 'volcengine-streaming'
const configSchema = z.object({
apiKey: z.string(),
})
type VolcengineStreamingConfig = z.input<typeof configSchema>
let defaultModelId: string | null = null
function authHeaders(): Record<string, string> {
const headers: Record<string, string> = { Accept: 'application/json' }
const token = getAuthToken()
if (token)
headers.Authorization = `Bearer ${token}`
return headers
}
export function getVolcengineStreamingDefaultModel(): string | null {
return defaultModelId
}
export const providerVolcengineStreaming = defineProvider<VolcengineStreamingConfig>({
id: VOLCENGINE_STREAMING_PROVIDER_ID,
order: 31,
name: 'Volcengine Streaming TTS',
nameLocalize: ({ t }) => t('settings.pages.providers.provider.volcengine-streaming.title'),
description: 'Low-latency Volcengine TTS through the AIRI UnSpeech bridge.',
descriptionLocalize: ({ t }) => t('settings.pages.providers.provider.volcengine-streaming.description'),
tasks: ['text-to-speech'],
iconColor: 'i-lobe-icons:volcengine',
createProviderConfig: ({ t }) => configSchema.extend({
apiKey: configSchema.shape.apiKey.meta({
labelLocalized: t('settings.pages.providers.common.fields.field.api-key.label'),
descriptionLocalized: t('settings.pages.providers.provider.volcengine-streaming.fields.api-key.description'),
placeholderLocalized: t('settings.pages.providers.provider.volcengine-streaming.fields.api-key.placeholder'),
type: 'password',
}),
}),
createProvider(config) {
// The provider instance satisfies the shared speech-provider boundary.
// Runtime synthesis uses the bidirectional session capability below, not
// this REST-compatible instance.
const apiKey = typeof config.apiKey === 'string' ? config.apiKey.trim() : ''
return createUnVolcengine(apiKey, 'https://unspeech.hyp3r.link/v1/')
},
capabilities: {
speech: {
transport: 'bidirectional-ws',
resolveConnection: (config) => {
const apiKey = typeof config.apiKey === 'string' ? config.apiKey.trim() : ''
return {
credentialMode: 'byok',
providerId: VOLCENGINE_STREAMING_PROVIDER_ID,
apiKey,
}
},
getDefaultModel: getVolcengineStreamingDefaultModel,
},
},
validationRequiredWhen: () => true,
validators: {
validateConfig: [
({ t }) => ({
id: 'volcengine-streaming:check-config',
name: t('settings.pages.providers.catalog.edit.validators.openai-compatible.check-config.title'),
validator: async (config) => {
const apiKey = typeof config.apiKey === 'string' ? config.apiKey.trim() : ''
const valid = apiKey.length > 0
return {
errors: valid ? [] : [{ error: new Error('X-Api-Key is required.') }],
reason: valid ? '' : 'X-Api-Key is required.',
reasonKey: '',
valid,
}
},
}),
],
},
extraMethods: {
listModels: async (): Promise<ModelInfo[]> => {
defaultModelId = null
const response = await globalThis.fetch(`${SERVER_URL}/api/v1/audio/models/streaming`, { headers: authHeaders() })
if (!response.ok)
throw new Error(`streaming models upstream ${response.status}: ${await response.text().catch(() => '')}`.slice(0, 256))
const data = await response.json() as {
models?: { id: string, name?: string, description?: string }[]
default?: string | null
}
if (!Array.isArray(data.models))
throw new Error('streaming models upstream missing models[]')
defaultModelId = typeof data.default === 'string' && data.default.length > 0 ? data.default : null
return data.models.map(model => ({
id: model.id,
name: model.name ?? model.id,
provider: VOLCENGINE_STREAMING_PROVIDER_ID,
description: model.description,
}))
},
listVoices: async (_config, _provider, model): Promise<VoiceInfo[]> => {
const resourceId = model?.includes('/') ? model.split('/', 2)[1] : model
const url = new URL(`${SERVER_URL}/api/v1/audio/voices/streaming`)
if (resourceId)
url.searchParams.set('model', resourceId)
const response = await globalThis.fetch(url, { headers: authHeaders() })
if (!response.ok)
throw new Error(`streaming voices upstream ${response.status}: ${await response.text().catch(() => '')}`.slice(0, 256))
const data = await response.json() as {
voices?: {
id: string
name: string
description?: string
labels?: Record<string, unknown>
languages?: { code: string, title: string }[]
preview_audio_url?: string
}[]
}
if (!Array.isArray(data.voices))
throw new Error('streaming voices upstream returned malformed body')
return data.voices.map((voice) => {
const gender = typeof voice.labels?.gender === 'string' ? voice.labels.gender.toLowerCase() : undefined
return {
id: voice.id,
name: voice.name,
provider: VOLCENGINE_STREAMING_PROVIDER_ID,
description: voice.description,
gender,
previewURL: voice.preview_audio_url,
languages: Array.isArray(voice.languages) ? voice.languages : [],
}
})
},
},
})
@@ -95,6 +95,7 @@ const providerSourceMetadataById = {
'speech-noop': false,
'together-ai': paidCloud,
'volcengine': paidCloud,
'volcengine-streaming': paidCloud,
'volcengine-coding-plan': paidCloud,
'xai': paidCloud,
'zai': paidCloud,
@@ -15,6 +15,8 @@ import type { MaybePromise } from 'clustr'
import type { ComposerTranslation } from 'vue-i18n'
import type { $ZodType } from 'zod/v4/core'
import type { StreamingTtsConnection } from '../speech/streaming-connection'
export type ProviderInstance
= | ChatProvider
| ChatProviderWithExtraOptions
@@ -214,6 +216,10 @@ export interface ProviderDefinition<TConfig extends any = any> {
*/
speech?: {
transport: 'rest' | 'bidirectional-ws'
/** Resolves provider-owned credential policy for one streaming session. */
resolveConnection?: (config: TConfig) => StreamingTtsConnection
/** Returns the provider's discovered default streaming model. */
getDefaultModel?: () => string | null
}
}
/**
@@ -0,0 +1,13 @@
/**
* Connection policy snapshotted from the active streaming speech provider.
*
* Official sessions use AIRI-hosted credentials and Flux. BYOK sessions send
* the user credential inside the authenticated WSS session so the AIRI bridge
* can inject it into UnSpeech's upstream upgrade request.
*/
export interface StreamingTtsConnection {
credentialMode: 'official' | 'byok'
providerId: 'official-provider-speech-streaming' | 'volcengine-streaming'
/** Plaintext credential used only for a BYOK upstream handshake. */
apiKey?: string
}
@@ -8,6 +8,11 @@ import { WebSocketServer } from 'ws'
import { createStreamingTtsPipeline } from './streaming-pipeline'
const officialConnection = {
credentialMode: 'official',
providerId: 'official-provider-speech-streaming',
} as const
vi.mock('../auth', () => ({
getAuthToken: () => 'test-jwt',
}))
@@ -19,6 +24,7 @@ interface MockServer {
url: string
receivedFrames: Array<{ kind: 'text' | 'binary', data: string | Buffer }>
observedVoiceTypes: string[]
observedCredentialModes: string[]
/** Resolves when the server has observed a `start` frame from the client. */
startObserved: Promise<void>
stop: () => Promise<void>
@@ -27,6 +33,7 @@ interface MockServer {
async function startMockServer(handler: (ws: import('ws').WebSocket) => void): Promise<MockServer> {
const receivedFrames: MockServer['receivedFrames'] = []
const observedVoiceTypes: string[] = []
const observedCredentialModes: string[] = []
const httpServer = createServer()
const wss = new WebSocketServer({ server: httpServer })
@@ -40,6 +47,9 @@ async function startMockServer(handler: (ws: import('ws').WebSocket) => void): P
const voiceType = u.searchParams.get('tts_voice_type')
if (voiceType != null)
observedVoiceTypes.push(voiceType)
const credentialMode = u.searchParams.get('tts_credential_mode')
if (credentialMode != null)
observedCredentialModes.push(credentialMode)
ws.on('message', (data, isBinary) => {
const buf = Buffer.isBuffer(data) ? data : Buffer.from(data as ArrayBuffer)
@@ -64,6 +74,7 @@ async function startMockServer(handler: (ws: import('ws').WebSocket) => void): P
url: `http://127.0.0.1:${port}`,
receivedFrames,
observedVoiceTypes,
observedCredentialModes,
startObserved,
async stop() {
wss.close()
@@ -132,6 +143,7 @@ describe('createStreamingTtsPipeline', () => {
const onDone = vi.fn()
const handle = createStreamingTtsPipeline({
connection: officialConnection,
serverUrl: server.url,
model: 'volcengine/seed-tts-1.0',
voice: 'mock',
@@ -169,6 +181,44 @@ describe('createStreamingTtsPipeline', () => {
expect(calls[1].audio.__byteLength).toBe(chunks[2].length)
})
it('opens chat streaming with the BYOK credentials frame before start', async () => {
server = await startMockServer((ws) => {
ws.on('message', (data, isBinary) => {
if (isBinary)
return
const frame = JSON.parse(data.toString()) as { event?: string }
if (frame.event === 'finish')
ws.send(JSON.stringify({ event: 'session.finished', payload: { usage: { text_words: 5 } } }))
})
})
const onDone = vi.fn()
const handle = createStreamingTtsPipeline({
connection: {
credentialMode: 'byok',
providerId: 'volcengine-streaming',
apiKey: 'chat-byok-key',
},
serverUrl: server.url,
model: 'volcengine/seed-tts-2.0',
voice: 'mock',
audioContext: makeStubAudioContext(),
onDone,
})
handle.appendText('hello')
handle.finish()
await new Promise<void>((resolve) => {
onDone.mockImplementation(() => resolve())
setTimeout(resolve, 1500)
})
const frames = server.receivedFrames.map(frame => JSON.parse(frame.data as string) as Record<string, unknown>)
expect(frames.map(frame => frame.event)).toEqual(['credentials', 'start', 'text', 'finish'])
expect(frames[0]?.api_key).toBe('chat-byok-key')
expect(server.observedCredentialModes).toEqual(['byok'])
})
it('buffers entire session when bufferEntireSession is true', async () => {
const chunks = [Buffer.from([1, 2, 3, 4]), Buffer.from([5, 6, 7, 8])]
server = await startMockServer((ws) => {
@@ -190,6 +240,7 @@ describe('createStreamingTtsPipeline', () => {
const onSentence = vi.fn()
const handle = createStreamingTtsPipeline({
connection: officialConnection,
serverUrl: server.url,
model: 'volcengine/seed-tts-2.0',
voice: 'mock',
@@ -223,6 +274,7 @@ describe('createStreamingTtsPipeline', () => {
const onError = vi.fn()
const onDone = vi.fn()
createStreamingTtsPipeline({
connection: officialConnection,
serverUrl: server.url,
model: 'volcengine/seed-tts-1.0',
voice: 'mock',
@@ -256,6 +308,7 @@ describe('createStreamingTtsPipeline', () => {
const onError = vi.fn()
const onDone = vi.fn()
createStreamingTtsPipeline({
connection: officialConnection,
serverUrl: server.url,
model: 'volcengine/seed-tts-1.0',
voice: 'mock',
@@ -287,6 +340,7 @@ describe('createStreamingTtsPipeline', () => {
const onDone = vi.fn()
const handle = createStreamingTtsPipeline({
connection: officialConnection,
serverUrl: server.url,
model: 'volcengine/seed-tts-1.0',
voice: 'mock',
@@ -1,3 +1,5 @@
import type { StreamingTtsConnection } from './streaming-connection'
import { getAuthToken } from '../auth'
import { SERVER_URL } from '../server'
@@ -36,6 +38,8 @@ export interface StreamingTtsPipelineEvents {
}
export interface StreamingTtsPipelineOptions extends StreamingTtsPipelineEvents {
/** Provider-owned credential and billing policy for this session. */
connection: StreamingTtsConnection
/** Server URL override. Defaults to {@link SERVER_URL}. */
serverUrl?: string
/** Override the auth token (Bearer). Defaults to {@link getAuthToken}. */
@@ -125,6 +129,7 @@ export function createStreamingTtsPipeline(options: StreamingTtsPipelineOptions)
ttsTrigger: options.ttsTrigger ?? 'auto',
ttsSource: options.ttsSource ?? 'chat_auto_tts',
ttsVoiceType: options.ttsVoiceType ?? 'unknown',
connection: options.connection,
})
const ws = new WebSocket(wsUrl)
ws.binaryType = 'arraybuffer'
@@ -239,6 +244,13 @@ export function createStreamingTtsPipeline(options: StreamingTtsPipelineOptions)
}
ws.addEventListener('open', () => {
if (options.connection.credentialMode === 'byok') {
ws.send(JSON.stringify({
event: 'credentials',
provider: 'volcengine',
api_key: options.connection.apiKey,
}))
}
const startFrame = {
event: 'start',
model: options.model,
@@ -418,6 +430,7 @@ function toWebSocketUrl(
ttsTrigger: 'auto' | 'manual'
ttsSource: 'chat_auto_tts' | 'manual_preview' | 'settings_test'
ttsVoiceType: 'official_default' | 'official_selected' | 'custom_configured' | 'voice_pack' | 'unknown'
connection: StreamingTtsConnection
},
): string {
const u = new URL(path, httpBase)
@@ -426,6 +439,8 @@ function toWebSocketUrl(
u.searchParams.set('tts_trigger', analytics.ttsTrigger)
u.searchParams.set('tts_source', analytics.ttsSource)
u.searchParams.set('tts_voice_type', analytics.ttsVoiceType)
u.searchParams.set('tts_credential_mode', analytics.connection.credentialMode)
u.searchParams.set('tts_provider_id', analytics.connection.providerId)
return u.toString()
}
@@ -8,6 +8,11 @@ import { WebSocketServer } from 'ws'
import { streamingSynthesize } from './streaming-session'
const officialConnection = {
credentialMode: 'official',
providerId: 'official-provider-speech-streaming',
} as const
vi.mock('../auth', () => ({
getAuthToken: () => 'test-jwt',
}))
@@ -20,6 +25,8 @@ interface MockServer {
url: string
observedTokens: string[]
observedVoiceTypes: string[]
observedCredentialModes: string[]
observedProviderIds: string[]
closeUnexpectedly: () => void
stop: () => Promise<void>
}
@@ -27,6 +34,8 @@ interface MockServer {
async function startMockServer(handler: (ws: import('ws').WebSocket) => void): Promise<MockServer> {
const observedTokens: string[] = []
const observedVoiceTypes: string[] = []
const observedCredentialModes: string[] = []
const observedProviderIds: string[] = []
const httpServer = createServer()
const wss = new WebSocketServer({ server: httpServer })
@@ -41,6 +50,12 @@ async function startMockServer(handler: (ws: import('ws').WebSocket) => void): P
const voiceType = u.searchParams.get('tts_voice_type')
if (voiceType != null)
observedVoiceTypes.push(voiceType)
const credentialMode = u.searchParams.get('tts_credential_mode')
if (credentialMode != null)
observedCredentialModes.push(credentialMode)
const providerId = u.searchParams.get('tts_provider_id')
if (providerId != null)
observedProviderIds.push(providerId)
handler(ws)
})
@@ -52,6 +67,8 @@ async function startMockServer(handler: (ws: import('ws').WebSocket) => void): P
url: `http://127.0.0.1:${port}`,
observedTokens,
observedVoiceTypes,
observedCredentialModes,
observedProviderIds,
closeUnexpectedly: () => {
activeWs?.close(1011, 'simulated_truncation')
},
@@ -106,6 +123,7 @@ describe('streamingSynthesize', () => {
})
const result = await streamingSynthesize({
connection: officialConnection,
serverUrl: server.url,
model: 'volcengine/seed-tts-2.0',
voice: 'mock',
@@ -124,6 +142,41 @@ describe('streamingSynthesize', () => {
expect(server.observedVoiceTypes).toEqual(['official_selected'])
})
it('sends BYOK credentials before the UnSpeech start frame and marks the connection mode', async () => {
const frames: Array<Record<string, unknown>> = []
server = await startMockServer((ws) => {
ws.on('message', (data, isBinary) => {
if (isBinary)
return
const frame = JSON.parse(data.toString()) as Record<string, unknown>
frames.push(frame)
if (frame.event === 'finish')
ws.send(JSON.stringify({ event: 'session.finished', payload: { usage: { text_words: 5 } } }))
})
})
await streamingSynthesize({
connection: {
credentialMode: 'byok',
providerId: 'volcengine-streaming',
apiKey: 'byok-test-key',
},
serverUrl: server.url,
model: 'volcengine/seed-tts-2.0',
voice: 'mock',
input: 'hello',
})
expect(frames.map(frame => frame.event)).toEqual(['credentials', 'start', 'text', 'finish'])
expect(frames[0]).toEqual({
event: 'credentials',
provider: 'volcengine',
api_key: 'byok-test-key',
})
expect(server.observedCredentialModes).toEqual(['byok'])
expect(server.observedProviderIds).toEqual(['volcengine-streaming'])
})
it('rejects when the ws closes before session.finished (codex HIGH #2)', async () => {
// Server sends some audio chunks then closes the ws WITHOUT emitting
// session.finished. Pre-fix behavior: streamingSynthesize would
@@ -150,6 +203,7 @@ describe('streamingSynthesize', () => {
})
await expect(streamingSynthesize({
connection: officialConnection,
serverUrl: server.url,
model: 'volcengine/seed-tts-2.0',
voice: 'mock',
@@ -176,6 +230,7 @@ describe('streamingSynthesize', () => {
})
await expect(streamingSynthesize({
connection: officialConnection,
serverUrl: server.url,
model: 'volcengine/seed-tts-2.0',
voice: 'mock',
@@ -197,6 +252,7 @@ describe('streamingSynthesize', () => {
const ctrl = new AbortController()
const promise = streamingSynthesize({
connection: officialConnection,
serverUrl: server.url,
model: 'volcengine/seed-tts-2.0',
voice: 'mock',
@@ -1,3 +1,5 @@
import type { StreamingTtsConnection } from './streaming-connection'
import { getAuthToken } from '../auth'
import { SERVER_URL } from '../server'
@@ -36,6 +38,8 @@ export interface StreamingTtsSessionResult {
}
export interface StreamingTtsSessionOptions {
/** Provider-owned credential and billing policy for this session. */
connection: StreamingTtsConnection
/** Server URL override. Defaults to {@link SERVER_URL}. */
serverUrl?: string
/** Override the auth token (Bearer). Defaults to {@link getAuthToken}. */
@@ -95,6 +99,7 @@ export async function streamingSynthesize(options: StreamingTtsSessionOptions):
ttsTrigger: options.ttsTrigger ?? 'manual',
ttsSource: options.ttsSource ?? 'manual_preview',
ttsVoiceType: options.ttsVoiceType ?? 'unknown',
connection: options.connection,
})
const audioChunks: ArrayBuffer[] = []
@@ -143,6 +148,13 @@ export async function streamingSynthesize(options: StreamingTtsSessionOptions):
}
ws.addEventListener('open', () => {
if (options.connection.credentialMode === 'byok') {
ws.send(JSON.stringify({
event: 'credentials',
provider: 'volcengine',
api_key: options.connection.apiKey,
}))
}
const startFrame = {
event: 'start',
model: options.model,
@@ -243,6 +255,7 @@ function toWebSocketUrl(
ttsTrigger: 'auto' | 'manual'
ttsSource: 'chat_auto_tts' | 'manual_preview' | 'settings_test'
ttsVoiceType: 'official_default' | 'official_selected' | 'custom_configured' | 'voice_pack' | 'unknown'
connection: StreamingTtsConnection
},
): string {
const u = new URL(path, httpBase)
@@ -251,6 +264,8 @@ function toWebSocketUrl(
u.searchParams.set('tts_trigger', analytics.ttsTrigger)
u.searchParams.set('tts_source', analytics.ttsSource)
u.searchParams.set('tts_voice_type', analytics.ttsVoiceType)
u.searchParams.set('tts_credential_mode', analytics.connection.credentialMode)
u.searchParams.set('tts_provider_id', analytics.connection.providerId)
return u.toString()
}
@@ -46,6 +46,10 @@ function makePlaybackManagerStub<TAudio = AudioBuffer>(): PlaybackManagerSubset<
function makeStreamingSnapshot(overrides: Partial<StreamingSessionSnapshot> = {}): StreamingSessionSnapshot {
return {
connection: {
credentialMode: 'official',
providerId: 'official-provider-speech-streaming',
},
model: 'volcengine/seed-tts-2.0',
voice: 'mock-voice',
voiceType: 'official_selected',
@@ -1,5 +1,6 @@
import type { IntentHandle, IntentOptions, PlaybackItem } from '@proj-airi/pipelines-audio'
import type { StreamingTtsConnection } from './streaming-connection'
import type { StreamingTtsPipelineOptions } from './streaming-pipeline'
import { createStreamingTtsPipeline } from './streaming-pipeline'
@@ -69,6 +70,7 @@ function fromIntent(intent: IntentHandleSubset): StageTtsSession {
* for cancelling and re-opening.
*/
export interface StreamingSessionSnapshot {
connection: StreamingTtsConnection
model: string
voice: string
voiceType: 'official_default' | 'official_selected' | 'custom_configured' | 'voice_pack' | 'unknown'
@@ -157,6 +159,7 @@ export function createStreamingTtsSession<TAudio = AudioBuffer>(
let terminated = false
const handle = pipelineFactory({
connection: snapshot.connection,
model: snapshot.model,
voice: snapshot.voice,
ttsVoiceType: snapshot.voiceType,
+10 -12
View File
@@ -12,7 +12,8 @@ import { useI18n } from 'vue-i18n'
import { toXml } from 'xast-util-to-xml'
import { x } from 'xastscript'
import { getDefaultSpeechModel, getDefaultStreamingModel, OFFICIAL_SPEECH_PROVIDER_ID, OFFICIAL_SPEECH_STREAMING_PROVIDER_ID, setupOfficialSpeechAutoPick } from '../../libs/providers/providers/official'
import { getDefinedProvider } from '../../libs/providers/providers'
import { getDefaultSpeechModel, OFFICIAL_SPEECH_PROVIDER_ID, OFFICIAL_SPEECH_STREAMING_PROVIDER_ID, setupOfficialSpeechAutoPick } from '../../libs/providers/providers/official'
import { useProvidersStore } from '../providers'
export function toSignedPercent(value: number): string {
@@ -141,18 +142,15 @@ export const useSpeechStore = defineStore('speech', () => {
activeSpeechVoice.value = undefined
}
// Streaming TTS voices are model-scoped: the server only returns recommended
// voices for an explicit `?model=`. Ensure the active model is a valid
// streaming model id so voice loading gets the right recommendations (parity
// with the HTTP provider's auto-pick). Reseeds the server-curated default
// both when no model is selected AND when `activeSpeechModel` still holds a
// stale id from a previously-active provider (the global model ref is shared
// across providers, and the per-surface reset may not have run yet). No-op
// for non-streaming providers.
// Streaming TTS voices are model-scoped. Resolve the default through the
// active provider capability so official and BYOK transports share the same
// state transition without Stage knowing provider ids.
function ensureStreamingDefaultModel() {
if (activeSpeechProvider.value !== OFFICIAL_SPEECH_STREAMING_PROVIDER_ID)
const providerId = activeSpeechProvider.value
const speechCapability = getDefinedProvider(providerId)?.capabilities?.speech
if (speechCapability?.transport !== 'bidirectional-ws')
return
const streamingModels = providersStore.getModelsForProvider(OFFICIAL_SPEECH_STREAMING_PROVIDER_ID)
const streamingModels = providersStore.getModelsForProvider(providerId)
const hasValidSelection = !!activeSpeechModel.value && streamingModels.some(m => m.id === activeSpeechModel.value)
if (hasValidSelection)
return
@@ -160,7 +158,7 @@ export const useSpeechStore = defineStore('speech', () => {
// When no default can be resolved yet (catalog not loaded), clear it to ''
// so callers pass `undefined` (server returns the full streaming catalog)
// rather than forwarding a stale non-streaming model id as `?model=`.
const nextModel = getDefaultStreamingModel() ?? streamingModels[0]?.id ?? ''
const nextModel = speechCapability.getDefaultModel?.() ?? streamingModels[0]?.id ?? ''
if (activeSpeechModel.value === nextModel)
return
activeSpeechModel.value = nextModel
+16 -1
View File
@@ -4,7 +4,7 @@ import type { AuthInstance } from './libs/auth'
import type { Database } from './libs/db'
import type { Env } from './libs/env'
import type { OtelInstance } from './otel'
import type { StreamingTtsVoiceType } from './routes/audio-speech-ws/session'
import type { StreamingTtsProviderId, StreamingTtsVoiceType } from './routes/audio-speech-ws/session'
import type { ConfigKVService } from './services/adapters/config-kv'
import type { AdminFluxGrantsService } from './services/domain/admin/flux-grants'
import type { AdminRouterConfigService } from './services/domain/admin/router-config'
@@ -222,6 +222,8 @@ export async function buildApp(deps: AppDeps) {
trigger: c.req.query('tts_trigger') === 'auto' ? 'auto' : 'manual',
source: parseTtsSource(c.req.query('tts_source'), 'audio.speech.ws'),
voiceType: parseTtsVoiceType(c.req.query('tts_voice_type')),
credentialMode: c.req.query('tts_credential_mode') === 'byok' ? 'byok' : 'official',
providerId: parseStreamingTtsProviderId(c.req.query('tts_provider_id')),
})
}))
@@ -503,6 +505,19 @@ function parseTtsVoiceType(
}
}
/**
* Bounds client-provided provider identity before it reaches product events.
*/
function parseStreamingTtsProviderId(value: string | undefined): StreamingTtsProviderId {
switch (value) {
case 'official-provider-speech-streaming':
case 'volcengine-streaming':
return value
default:
return 'unknown'
}
}
export type AppType = Awaited<ReturnType<typeof buildApp>>['app']
export async function createApp() {
@@ -23,8 +23,9 @@ export type { AudioSpeechWsHandlersOptions } from './types'
* Expects:
* - The route handler has already resolved auth via the `?token=` query
* (see app.ts wiring) and passes a verified `userId` in.
* - The client sends a `start` control frame first. The session validates the
* requested streaming model and voice before dialing upstream.
* - Official clients send a `start` control frame first. BYOK clients send an
* AIRI-private `credentials` frame followed by `start`. The session consumes
* credentials locally, then validates model and voice before dialing upstream.
*
* Returns:
* - A function that takes `userId` and returns hono `WSEvents`. Each call
@@ -312,6 +312,66 @@ describe('audio-speech-ws route', () => {
}))
})
it('uses the client BYOK credential without forwarding it as a frame or charging Flux', async () => {
upstream = await startMockUpstream([
{ kind: 'json', payload: { event: 'session.finished', payload: { usage: { text_words: 21 } } } },
])
// A zero balance would reject the official path. BYOK must not run that
// pre-flight or any final Flux accumulation.
const deps = makeFakeDeps({ upstreamURL: upstream.url, restBaseURL: upstream.restBaseURL, fluxBalance: 0 })
const handlers = createAudioSpeechWsHandlers(deps as any)
const events = handlers('user-byok', {
credentialMode: 'byok',
providerId: 'volcengine-streaming',
voiceType: 'custom_configured',
})
const client = makeMockClientWs()
await driveClientSession(events, client, [
JSON.stringify({ event: 'credentials', provider: 'volcengine', api_key: 'user-owned-x-api-key' }),
JSON.stringify({ event: 'start', model: 'volcengine/seed-tts-2.0', voice: 'mock' }),
JSON.stringify({ event: 'text', text: 'BYOK streaming' }),
JSON.stringify({ event: 'finish' }),
])
await new Promise(r => setTimeout(r, 200))
expect(upstream.observedAuth).toBe('Bearer user-owned-x-api-key')
expect(upstream.receivedFrames).toHaveLength(3)
expect(upstream.receivedFrames.map(frame => JSON.parse(frame.data as string).event)).toEqual(['start', 'text', 'finish'])
expect(deps.fluxService.getFlux).not.toHaveBeenCalled()
expect(deps.ttsMeter.assertCanAfford).not.toHaveBeenCalled()
expect(deps.ttsMeter.accumulate).not.toHaveBeenCalled()
expect(deps.productEventService.track).toHaveBeenCalledWith(expect.objectContaining({
action: 'speech_succeeded',
metadata: expect.objectContaining({
credential_mode: 'byok',
provider_id: 'volcengine-streaming',
flux_consumed: 0,
}),
}))
})
it('does not fall back to the operator key when a BYOK session omits credentials', async () => {
upstream = await startMockUpstream([])
const deps = makeFakeDeps({ upstreamURL: upstream.url, restBaseURL: upstream.restBaseURL, fluxBalance: 100 })
const handlers = createAudioSpeechWsHandlers(deps as any)
const events = handlers('user-byok', {
credentialMode: 'byok',
providerId: 'volcengine-streaming',
})
const client = makeMockClientWs()
await driveClientSession(events, client, [
JSON.stringify({ event: 'start', model: 'volcengine/seed-tts-2.0', voice: 'mock' }),
])
expect(client.closeCode).toBe(1008)
expect(client.closeReason).toBe('invalid_credentials_frame')
expect(upstream.observedAuth).toBeUndefined()
expect(upstream.receivedFrames).toHaveLength(0)
expect(deps.envelopeCrypto.decryptKey).not.toHaveBeenCalled()
})
it('refuses the session with insufficient_flux when the user is broke', async () => {
upstream = await startMockUpstream([])
const deps = makeFakeDeps({ upstreamURL: upstream.url, restBaseURL: upstream.restBaseURL, fluxBalance: 0 })
@@ -44,7 +44,7 @@ const tracer = trace.getTracer('audio-speech-ws')
export interface AudioSpeechSessionState {
/** Stores the accepted client websocket. */
attachClient: (ws: WSContext) => void
/** Reads config, checks balance, decrypts the upstream key, and dials upstream after the start frame is accepted. */
/** Resolves the selected credential policy and dials upstream after the start frame is accepted. */
dialUpstream: () => Promise<void>
/** Forwards a client frame or queues it while the upstream connection opens. */
handleClientMessage: (message: { data: unknown }, ws: WSContext) => void
@@ -55,11 +55,15 @@ export interface AudioSpeechSessionState {
export type StreamingTtsTrigger = 'auto' | 'manual'
export type StreamingTtsSource = 'audio.speech.ws' | 'chat_auto_tts' | 'manual_preview' | 'settings_test'
export type StreamingTtsVoiceType = 'official_default' | 'official_selected' | 'custom_configured' | 'voice_pack' | 'unknown'
export type StreamingTtsCredentialMode = 'official' | 'byok'
export type StreamingTtsProviderId = 'official-provider-speech-streaming' | 'volcengine-streaming' | 'unknown'
export interface AudioSpeechSessionAnalytics {
trigger?: StreamingTtsTrigger
source?: StreamingTtsSource
voiceType?: StreamingTtsVoiceType
credentialMode?: StreamingTtsCredentialMode
providerId?: StreamingTtsProviderId
}
/**
@@ -71,7 +75,9 @@ export interface AudioSpeechSessionAnalytics {
* are handled at session end.
*
* Expects:
* - `UNSPEECH_UPSTREAM.streaming` has a base URL and at least one encrypted key.
* - `UNSPEECH_UPSTREAM.streaming` has a base URL.
* - Official sessions have an operator key; BYOK sessions send one private
* credentials frame before the UnSpeech start frame.
*
* Returns:
* - A connection-scoped state object with no global peer registry.
@@ -94,7 +100,7 @@ export function createSessionState(
let upstreamWs: WebSocket | null = null
let upstreamReady = false
let closed = false
let billed = false
let completed = false
let startFrameAccepted = false
let startValidationStarted = false
let dialStarted = false
@@ -102,6 +108,8 @@ export function createSessionState(
let preflightFluxBalance: number | undefined
let modelLabel = STREAM_MODEL_LABEL_FALLBACK
let voiceLabel: string | undefined
let byokKey: Buffer | null = null
let credentialFrameAccepted = analytics.credentialMode === 'official'
/**
* Frames the client sent before the upstream finished dialing. Buffered to
* avoid silently dropping the `start` frame; flushed in arrival order once
@@ -127,7 +135,7 @@ export function createSessionState(
model: modelLabel,
metadata: {
trigger: analytics.trigger,
...streamingVoiceMetadata(voiceLabel, analytics.voiceType),
...streamingSessionMetadata(voiceLabel, analytics),
},
})
@@ -142,51 +150,63 @@ export function createSessionState(
}
const upstreamConfig = unspeech?.streaming
if (!upstreamConfig || !upstreamConfig.baseURL || upstreamConfig.keys.length === 0) {
if (!upstreamConfig?.baseURL || (analytics.credentialMode === 'official' && upstreamConfig.keys.length === 0)) {
closeWithError(1008, 'streaming_tts_not_configured')
return
}
// Pre-flight balance check: refuse before dialing if the user cannot
// afford the worst-case session.
try {
const flux = await opts.fluxService.getFlux(userId)
preflightFluxBalance = flux.flux
await opts.ttsMeter.assertCanAfford(userId, STREAMING_PREFLIGHT_CHARS_ESTIMATE, flux.flux)
}
catch (err) {
log.withError(err).withFields({ userId }).warn('pre-flight rejected streaming tts')
// assertCanAfford throws PaymentRequiredError (402) — translate to ws
// policy-violation close. The client can read the close code/reason to
// surface a 'top up' prompt.
if (isPaymentRequiredError(err))
closeWithBlockedPreflight(1008, 'insufficient_flux')
else
closeWithError(1011, 'flux_preflight_failed')
return
if (analytics.credentialMode === 'official') {
// Official sessions consume AIRI-hosted credentials and Flux. BYOK
// sessions are explicitly excluded so a missing user key can never
// fall back to the operator account or charge the user twice.
try {
const flux = await opts.fluxService.getFlux(userId)
preflightFluxBalance = flux.flux
await opts.ttsMeter.assertCanAfford(userId, STREAMING_PREFLIGHT_CHARS_ESTIMATE, flux.flux)
}
catch (err) {
log.withError(err).withFields({ userId }).warn('pre-flight rejected streaming tts')
// assertCanAfford throws PaymentRequiredError (402) — translate to ws
// policy-violation close. The client can read the close code/reason to
// surface a 'top up' prompt.
if (isPaymentRequiredError(err))
closeWithBlockedPreflight(1008, 'insufficient_flux')
else
closeWithError(1011, 'flux_preflight_failed')
return
}
}
// Decrypt the first key. Streaming surface does not do per-attempt key
// rotation: a live ws cannot transparently switch upstream mid-session
// without breaking audio continuity. Fallback policy belongs at the
// session-retry layer (next client connect), not inline.
const entry = upstreamConfig.keys[0]
let keyPlaintext: Buffer
try {
keyPlaintext = opts.envelopeCrypto.decryptKey(entry.ciphertext, {
modelName: STREAM_MODEL_LABEL_FALLBACK,
keyEntryId: entry.id,
})
if (analytics.credentialMode === 'byok') {
if (!byokKey) {
closeWithError(1008, 'byok_credentials_required')
return
}
keyPlaintext = byokKey
byokKey = null
}
catch (err) {
log.withError(err).withFields({ keyEntryId: entry.id }).error('decrypt failed for streaming tts key')
closeWithError(1011, 'decrypt_failed')
return
else {
// Streaming sessions do not rotate keys mid-connection because that
// would break audio continuity. Retry policy belongs to the next client
// connection, never to an in-flight session.
const entry = upstreamConfig.keys[0]
try {
keyPlaintext = opts.envelopeCrypto.decryptKey(entry.ciphertext, {
modelName: STREAM_MODEL_LABEL_FALLBACK,
keyEntryId: entry.id,
})
}
catch (err) {
log.withError(err).withFields({ keyEntryId: entry.id }).error('decrypt failed for streaming tts key')
closeWithError(1011, 'decrypt_failed')
return
}
span.setAttribute(AIRI_ATTR_GEN_AI_GATEWAY_KEY_ID, entry.id)
}
const upstreamURL = upstreamConfig.baseURL
span.setAttribute(AIRI_ATTR_GEN_AI_GATEWAY_UPSTREAM_URL, upstreamURL)
span.setAttribute(AIRI_ATTR_GEN_AI_GATEWAY_KEY_ID, entry.id)
let upstream: WebSocket
try {
@@ -196,6 +216,11 @@ export function createSessionState(
},
})
}
catch (err) {
log.withError(err).withFields({ userId }).warn('failed to dial streaming tts upstream')
closeWithError(1011, 'upstream_dial_failed')
return
}
finally {
// Wipe plaintext immediately — the ws lib has already serialized the
// header into its outgoing handshake buffer.
@@ -242,7 +267,7 @@ export function createSessionState(
metadata: {
duration_ms: Date.now() - startedAt,
trigger: analytics.trigger,
...streamingVoiceMetadata(voiceLabel, analytics.voiceType),
...streamingSessionMetadata(voiceLabel, analytics),
},
})
try {
@@ -270,6 +295,26 @@ export function createSessionState(
? Buffer.from(message.data)
: Buffer.from(message.data as ArrayBufferLike)
if (!credentialFrameAccepted) {
if (isBinary || typeof payload !== 'string') {
closeWithError(1008, 'invalid_credentials_frame')
return
}
const credentials = parseCredentialsFrame(payload)
if (!credentials) {
closeWithError(1008, 'invalid_credentials_frame')
return
}
// This AIRI-private frame is deliberately consumed here and never
// placed in pendingClientFrames, so the upstream UnSpeech protocol can
// never observe or log the plaintext credential as an application frame.
byokKey = Buffer.from(credentials.apiKey, 'utf8')
credentialFrameAccepted = true
return
}
if (!startValidationStarted) {
if (isBinary || typeof payload !== 'string') {
closeWithError(1008, 'invalid_start_frame')
@@ -382,10 +427,7 @@ export function createSessionState(
// the client-text-frame estimate accumulated in handleClientMessage.
const usageChars = readUsageChars(evt.payload)
const billUnits = usageChars ?? totalInputChars
if (billUnits > 0)
void billSession(billUnits, 'session.finished')
else
finalize()
void completeSession(billUnits, 'session.finished')
break
}
case 'error': {
@@ -433,7 +475,7 @@ export function createSessionState(
}
const upstreamConfig = unspeech?.streaming
if (!unspeech?.restBaseURL || !upstreamConfig?.baseURL || upstreamConfig.keys.length === 0) {
if (!unspeech?.restBaseURL || !upstreamConfig?.baseURL || (analytics.credentialMode === 'official' && upstreamConfig.keys.length === 0)) {
closeWithError(1008, 'streaming_tts_not_configured')
return false
}
@@ -470,42 +512,44 @@ export function createSessionState(
return true
}
async function billSession(units: number, reason: string) {
if (billed)
async function completeSession(units: number, reason: string) {
if (completed)
return
billed = true
completed = true
span.setAttribute(GEN_AI_ATTR_REQUEST_MODEL, modelLabel)
let flux: Awaited<ReturnType<FluxService['getFlux']>>
try {
flux = await opts.fluxService.getFlux(userId)
}
catch (err) {
log.withError(err).withFields({ userId }).warn('flux read failed at session end')
finalize()
return
}
let fluxConsumed = 0
try {
const result = await otelContext.with(trace.setSpan(otelContext.active(), span), () =>
opts.ttsMeter.accumulate({
userId,
units,
currentBalance: flux.flux,
requestId,
metadata: { model: modelLabel },
}))
fluxConsumed = result.fluxDebited
span.setAttribute(AIRI_ATTR_BILLING_FLUX_CONSUMED, fluxConsumed)
}
catch (err) {
// Billing failure is surfaced but does not retroactively reject the
// already-delivered audio — the user got the audio, the meter retains
// the debt for the next request to settle (per FluxMeter rollback path).
log.withError(err).withFields({ userId, units, reason }).error('billing accumulate failed for streaming tts')
span.recordException(err as Error)
span.setStatus({ code: SpanStatusCode.ERROR, message: 'billing_failed' })
if (analytics.credentialMode === 'official' && units > 0) {
let flux: Awaited<ReturnType<FluxService['getFlux']>>
try {
flux = await opts.fluxService.getFlux(userId)
}
catch (err) {
log.withError(err).withFields({ userId }).warn('flux read failed at session end')
finalize()
return
}
try {
const result = await otelContext.with(trace.setSpan(otelContext.active(), span), () =>
opts.ttsMeter.accumulate({
userId,
units,
currentBalance: flux.flux,
requestId,
metadata: { model: modelLabel },
}))
fluxConsumed = result.fluxDebited
span.setAttribute(AIRI_ATTR_BILLING_FLUX_CONSUMED, fluxConsumed)
}
catch (err) {
// Billing failure is surfaced but does not retroactively reject the
// already-delivered audio — the user got the audio, the meter retains
// the debt for the next request to settle (per FluxMeter rollback path).
log.withError(err).withFields({ userId, units, reason }).error('billing accumulate failed for streaming tts')
span.recordException(err as Error)
span.setStatus({ code: SpanStatusCode.ERROR, message: 'billing_failed' })
}
}
const durationMs = Date.now() - startedAt
@@ -534,7 +578,7 @@ export function createSessionState(
duration_ms: durationMs,
flux_consumed: fluxConsumed,
trigger: analytics.trigger,
...streamingVoiceMetadata(voiceLabel, analytics.voiceType),
...streamingSessionMetadata(voiceLabel, analytics),
},
})
@@ -545,6 +589,7 @@ export function createSessionState(
if (closed)
return
closed = true
wipeByokKey()
try {
upstreamWs?.close()
}
@@ -572,7 +617,7 @@ export function createSessionState(
close_code: code,
duration_ms: Date.now() - startedAt,
trigger: analytics.trigger,
...streamingVoiceMetadata(voiceLabel, analytics.voiceType),
...streamingSessionMetadata(voiceLabel, analytics),
},
})
if (clientWs) {
@@ -586,6 +631,7 @@ export function createSessionState(
catch {}
}
closed = true
wipeByokKey()
span.end()
}
@@ -608,7 +654,7 @@ export function createSessionState(
close_code: code,
duration_ms: Date.now() - startedAt,
trigger: analytics.trigger,
...streamingVoiceMetadata(voiceLabel, analytics.voiceType),
...streamingSessionMetadata(voiceLabel, analytics),
},
})
if (clientWs) {
@@ -622,9 +668,15 @@ export function createSessionState(
catch {}
}
closed = true
wipeByokKey()
span.end()
}
function wipeByokKey() {
byokKey?.fill(0)
byokKey = null
}
return {
attachClient,
dialUpstream,
@@ -638,6 +690,8 @@ function normalizeAnalytics(input: AudioSpeechSessionAnalytics): Required<AudioS
trigger: normalizeTrigger(input.trigger),
source: normalizeSource(input.source),
voiceType: normalizeVoiceType(input.voiceType),
credentialMode: input.credentialMode === 'byok' ? 'byok' : 'official',
providerId: normalizeProviderId(input.providerId),
}
}
@@ -682,6 +736,27 @@ function streamingVoiceMetadata(voiceId: string | undefined, voiceType: Streamin
}
}
function streamingSessionMetadata(
voiceId: string | undefined,
analytics: Required<AudioSpeechSessionAnalytics>,
): Record<string, unknown> {
return {
...streamingVoiceMetadata(voiceId, analytics.voiceType),
credential_mode: analytics.credentialMode,
provider_id: analytics.providerId,
}
}
function normalizeProviderId(providerId: AudioSpeechSessionAnalytics['providerId']): StreamingTtsProviderId {
switch (providerId) {
case 'official-provider-speech-streaming':
case 'volcengine-streaming':
return providerId
default:
return 'unknown'
}
}
function isPaymentRequiredError(err: unknown): boolean {
if (err instanceof ApiError)
return err.statusCode === 402
@@ -697,6 +772,29 @@ interface StreamingTtsStartFrame {
voice: string
}
interface StreamingTtsCredentialsFrame {
event: 'credentials'
provider: 'volcengine'
apiKey: string
}
function parseCredentialsFrame(rawText: string): StreamingTtsCredentialsFrame | null {
try {
const parsed = JSON.parse(rawText) as Record<string, unknown>
if (parsed.event !== 'credentials' || parsed.provider !== 'volcengine')
return null
if (typeof parsed.api_key !== 'string')
return null
const apiKey = parsed.api_key.trim()
if (apiKey.length === 0 || apiKey.length > 4096)
return null
return { event: 'credentials', provider: 'volcengine', apiKey }
}
catch {
return null
}
}
function parseStartFrame(rawText: string): StreamingTtsStartFrame | null {
try {
const parsed = JSON.parse(rawText) as Record<string, unknown>
@@ -9,13 +9,13 @@ import type { EnvelopeCrypto } from '../../utils/envelope-crypto'
* Dependencies required by the streaming speech websocket proxy.
*/
export interface AudioSpeechWsHandlersOptions {
/** Reads upstream websocket URL and encrypted API keys. */
/** Reads the UnSpeech websocket URL and operator credentials for official sessions. */
configKV: ConfigKVService
/** Decrypts the selected upstream API key before the websocket handshake. */
envelopeCrypto: EnvelopeCrypto
/** Reads the user's current Flux balance for pre-flight and final billing. */
/** Reads the user's current Flux balance for official-session billing. */
fluxService: FluxService
/** Applies pre-flight affordability checks and final streaming TTS billing. */
/** Applies pre-flight checks and final billing to official sessions only. */
ttsMeter: FluxMeter
/** Persists request accounting after a stream finishes. */
requestLogService: RequestLogService