From efdf13c24c46fbfe67c023447c584130cb889e48 Mon Sep 17 00:00:00 2001 From: RainbowBird Date: Tue, 4 Aug 2026 17:15:17 +0800 Subject: [PATCH] feat(tts): add Volcengine streaming BYOK --- packages/i18n/src/locales/en/settings.yaml | 14 + packages/i18n/src/locales/es/settings.yaml | 14 + packages/i18n/src/locales/fr/settings.yaml | 14 + packages/i18n/src/locales/ja/settings.yaml | 14 + packages/i18n/src/locales/ko/settings.yaml | 14 + packages/i18n/src/locales/ru/settings.yaml | 14 + packages/i18n/src/locales/vi/settings.yaml | 14 + .../i18n/src/locales/zh-Hans/settings.yaml | 14 + .../i18n/src/locales/zh-Hant/settings.yaml | 14 + .../official-provider-speech-streaming.vue | 4 + .../providers/speech/volcengine-streaming.vue | 187 +++++++++++++ .../stage-ui/src/components/scenes/Stage.vue | 28 +- packages/stage-ui/src/libs/index.ts | 1 + .../src/libs/providers/providers/index.ts | 3 + .../providers/providers/official/index.ts | 9 +- .../volcengine-streaming/index.test.ts | 33 +++ .../providers/volcengine-streaming/index.ts | 148 ++++++++++ .../src/libs/providers/source-metadata.ts | 1 + packages/stage-ui/src/libs/providers/types.ts | 6 + .../src/libs/speech/streaming-connection.ts | 13 + .../libs/speech/streaming-pipeline.test.ts | 54 ++++ .../src/libs/speech/streaming-pipeline.ts | 15 ++ .../src/libs/speech/streaming-session.test.ts | 56 ++++ .../src/libs/speech/streaming-session.ts | 15 ++ .../src/libs/speech/tts-session.test.ts | 4 + .../stage-ui/src/libs/speech/tts-session.ts | 3 + .../stage-ui/src/stores/modules/speech.ts | 22 +- server/apps/api/src/app.ts | 17 +- .../api/src/routes/audio-speech-ws/index.ts | 5 +- .../src/routes/audio-speech-ws/route.test.ts | 60 +++++ .../api/src/routes/audio-speech-ws/session.ts | 254 ++++++++++++------ .../api/src/routes/audio-speech-ws/types.ts | 6 +- 32 files changed, 968 insertions(+), 102 deletions(-) create mode 100644 packages/stage-pages/src/pages/settings/providers/speech/volcengine-streaming.vue create mode 100644 packages/stage-ui/src/libs/providers/providers/volcengine-streaming/index.test.ts create mode 100644 packages/stage-ui/src/libs/providers/providers/volcengine-streaming/index.ts create mode 100644 packages/stage-ui/src/libs/speech/streaming-connection.ts diff --git a/packages/i18n/src/locales/en/settings.yaml b/packages/i18n/src/locales/en/settings.yaml index 6d91d0d04..03d58c011 100644 --- a/packages/i18n/src/locales/en/settings.yaml +++ b/packages/i18n/src/locales/en/settings.yaml @@ -1405,6 +1405,20 @@ pages: description: App ID of the project where you can obtain in Console label: App ID title: Volcano Engine + volcengine-streaming: + title: Volcengine Streaming TTS + description: Low-latency Volcengine TTS through the AIRI UnSpeech bridge + login-required: Sign in to AIRI so the browser can securely use the WebSocket proxy. + config-description: Configure the X-Api-Key from the Volcengine new console. The existing Volcengine V1 provider is unchanged. + model-placeholder: Choose a streaming model + privacy-title: How your key is used + privacy-description: Your key stays in this device's provider settings. For each WSS session, AIRI forwards it in memory only to authenticate UnSpeech and does not store it server-side. + preview-text: Hello, this is a preview of Volcengine bidirectional streaming speech synthesis. + fields: + api-key: + label: X-Api-Key + description: X-Api-Key from the Volcengine new console. App ID and the old-console Access Key are not supported by this streaming endpoint. + placeholder: Enter your X-Api-Key volcengine-coding-plan: description: Volcengine Coding Plan title: Volcengine Coding Plan diff --git a/packages/i18n/src/locales/es/settings.yaml b/packages/i18n/src/locales/es/settings.yaml index 5956a172f..5d69d9a16 100644 --- a/packages/i18n/src/locales/es/settings.yaml +++ b/packages/i18n/src/locales/es/settings.yaml @@ -1350,6 +1350,20 @@ pages: description: ID de App del proyecto que puedes obtener en la Consola label: ID de App title: Volcano Engine + volcengine-streaming: + title: Volcengine Streaming TTS + description: Low-latency Volcengine TTS through the AIRI UnSpeech bridge + login-required: Sign in to AIRI so the browser can securely use the WebSocket proxy. + config-description: Configure the X-Api-Key from the Volcengine new console. The existing Volcengine V1 provider is unchanged. + model-placeholder: Choose a streaming model + privacy-title: How your key is used + privacy-description: Your key stays in this device's provider settings. AIRI only forwards it in memory for each WSS session and does not store it server-side. + preview-text: Hello, this is a preview of Volcengine bidirectional streaming speech synthesis. + fields: + api-key: + label: X-Api-Key + description: X-Api-Key from the Volcengine new console. App ID and old-console Access Key are not supported. + placeholder: Enter your X-Api-Key volcengine-coding-plan: description: Plan de Codificación Volcengine title: Plan de Codificación Volcengine diff --git a/packages/i18n/src/locales/fr/settings.yaml b/packages/i18n/src/locales/fr/settings.yaml index 94461ada4..75731a6d2 100644 --- a/packages/i18n/src/locales/fr/settings.yaml +++ b/packages/i18n/src/locales/fr/settings.yaml @@ -1350,6 +1350,20 @@ pages: description: ID de l’application du projet, que vous pouvez obtenir dans la Console label: ID de l’application title: Volcano Engine + volcengine-streaming: + title: Volcengine Streaming TTS + description: Low-latency Volcengine TTS through the AIRI UnSpeech bridge + login-required: Sign in to AIRI so the browser can securely use the WebSocket proxy. + config-description: Configure the X-Api-Key from the Volcengine new console. The existing Volcengine V1 provider is unchanged. + model-placeholder: Choose a streaming model + privacy-title: How your key is used + privacy-description: Your key stays in this device's provider settings. AIRI only forwards it in memory for each WSS session and does not store it server-side. + preview-text: Hello, this is a preview of Volcengine bidirectional streaming speech synthesis. + fields: + api-key: + label: X-Api-Key + description: X-Api-Key from the Volcengine new console. App ID and old-console Access Key are not supported. + placeholder: Enter your X-Api-Key volcengine-coding-plan: description: Volcengine Coding Plan title: Volcengine Coding Plan diff --git a/packages/i18n/src/locales/ja/settings.yaml b/packages/i18n/src/locales/ja/settings.yaml index f0a0502c3..27704f45e 100644 --- a/packages/i18n/src/locales/ja/settings.yaml +++ b/packages/i18n/src/locales/ja/settings.yaml @@ -1350,6 +1350,20 @@ pages: description: コンソールで取得できるプロジェクトのApp ID label: App ID title: Volcano Engine + volcengine-streaming: + title: Volcengine Streaming TTS + description: Low-latency Volcengine TTS through the AIRI UnSpeech bridge + login-required: Sign in to AIRI so the browser can securely use the WebSocket proxy. + config-description: Configure the X-Api-Key from the Volcengine new console. The existing Volcengine V1 provider is unchanged. + model-placeholder: Choose a streaming model + privacy-title: How your key is used + privacy-description: Your key stays in this device's provider settings. AIRI only forwards it in memory for each WSS session and does not store it server-side. + preview-text: Hello, this is a preview of Volcengine bidirectional streaming speech synthesis. + fields: + api-key: + label: X-Api-Key + description: X-Api-Key from the Volcengine new console. App ID and old-console Access Key are not supported. + placeholder: Enter your X-Api-Key volcengine-coding-plan: description: Volcengine Coding Plan title: Volcengine Coding Plan diff --git a/packages/i18n/src/locales/ko/settings.yaml b/packages/i18n/src/locales/ko/settings.yaml index 4e9f88d6a..cf25599a3 100644 --- a/packages/i18n/src/locales/ko/settings.yaml +++ b/packages/i18n/src/locales/ko/settings.yaml @@ -1350,6 +1350,20 @@ pages: description: 콘솔에서 얻을 수 있는 프로젝트의 애플리케이션 ID label: 애플리케이션 ID title: Volcano 엔진 + volcengine-streaming: + title: Volcengine Streaming TTS + description: Low-latency Volcengine TTS through the AIRI UnSpeech bridge + login-required: Sign in to AIRI so the browser can securely use the WebSocket proxy. + config-description: Configure the X-Api-Key from the Volcengine new console. The existing Volcengine V1 provider is unchanged. + model-placeholder: Choose a streaming model + privacy-title: How your key is used + privacy-description: Your key stays in this device's provider settings. AIRI only forwards it in memory for each WSS session and does not store it server-side. + preview-text: Hello, this is a preview of Volcengine bidirectional streaming speech synthesis. + fields: + api-key: + label: X-Api-Key + description: X-Api-Key from the Volcengine new console. App ID and old-console Access Key are not supported. + placeholder: Enter your X-Api-Key volcengine-coding-plan: description: Volcengine Coding Plan title: Volcengine Coding Plan diff --git a/packages/i18n/src/locales/ru/settings.yaml b/packages/i18n/src/locales/ru/settings.yaml index 42a71d2ad..3e851c072 100644 --- a/packages/i18n/src/locales/ru/settings.yaml +++ b/packages/i18n/src/locales/ru/settings.yaml @@ -1350,6 +1350,20 @@ pages: description: App ID проекта (получается в Console) label: Идентификатор приложения (App ID) title: Volcano Engine + volcengine-streaming: + title: Volcengine Streaming TTS + description: Low-latency Volcengine TTS through the AIRI UnSpeech bridge + login-required: Sign in to AIRI so the browser can securely use the WebSocket proxy. + config-description: Configure the X-Api-Key from the Volcengine new console. The existing Volcengine V1 provider is unchanged. + model-placeholder: Choose a streaming model + privacy-title: How your key is used + privacy-description: Your key stays in this device's provider settings. AIRI only forwards it in memory for each WSS session and does not store it server-side. + preview-text: Hello, this is a preview of Volcengine bidirectional streaming speech synthesis. + fields: + api-key: + label: X-Api-Key + description: X-Api-Key from the Volcengine new console. App ID and old-console Access Key are not supported. + placeholder: Enter your X-Api-Key volcengine-coding-plan: description: План кодирования Volcengine title: План кодирования Volcengine diff --git a/packages/i18n/src/locales/vi/settings.yaml b/packages/i18n/src/locales/vi/settings.yaml index 614d35049..329f42bf7 100644 --- a/packages/i18n/src/locales/vi/settings.yaml +++ b/packages/i18n/src/locales/vi/settings.yaml @@ -1350,6 +1350,20 @@ pages: description: App ID của dự án (lấy trong Console) label: App ID title: Volcano Engine + volcengine-streaming: + title: Volcengine Streaming TTS + description: Low-latency Volcengine TTS through the AIRI UnSpeech bridge + login-required: Sign in to AIRI so the browser can securely use the WebSocket proxy. + config-description: Configure the X-Api-Key from the Volcengine new console. The existing Volcengine V1 provider is unchanged. + model-placeholder: Choose a streaming model + privacy-title: How your key is used + privacy-description: Your key stays in this device's provider settings. AIRI only forwards it in memory for each WSS session and does not store it server-side. + preview-text: Hello, this is a preview of Volcengine bidirectional streaming speech synthesis. + fields: + api-key: + label: X-Api-Key + description: X-Api-Key from the Volcengine new console. App ID and old-console Access Key are not supported. + placeholder: Enter your X-Api-Key volcengine-coding-plan: description: Volcengine Coding Plan title: Volcengine Coding Plan diff --git a/packages/i18n/src/locales/zh-Hans/settings.yaml b/packages/i18n/src/locales/zh-Hans/settings.yaml index ec236e14a..396c47d3f 100644 --- a/packages/i18n/src/locales/zh-Hans/settings.yaml +++ b/packages/i18n/src/locales/zh-Hans/settings.yaml @@ -1350,6 +1350,20 @@ pages: description: 可在控制台获取的 App ID label: App ID title: 火山引擎 + volcengine-streaming: + title: 火山引擎（双向流式 TTS） + description: 通过 AIRI UnSpeech 代理使用低延迟火山引擎语音合成 + login-required: 请先登录 AIRI，以便浏览器安全使用 WebSocket 代理。 + config-description: 配置火山引擎新控制台的 X-Api-Key。现有火山引擎 V1 提供商不会受到影响。 + model-placeholder: 选择流式模型 + privacy-title: 密钥使用方式 + privacy-description: 密钥保存在当前设备的提供商设置中。每次 WSS 会话中，AIRI 仅在内存中转发密钥用于 UnSpeech 鉴权，不会在服务端持久化。 + preview-text: 你好，这是火山引擎双向流式语音合成的试听样例。 + fields: + api-key: + label: X-Api-Key + description: 火山引擎新控制台提供的 X-Api-Key。此流式接口不支持 App ID 与旧控制台 Access Key。 + placeholder: 输入 X-Api-Key volcengine-coding-plan: description: 火山引擎 Coding Plan title: 火山引擎 Coding Plan diff --git a/packages/i18n/src/locales/zh-Hant/settings.yaml b/packages/i18n/src/locales/zh-Hant/settings.yaml index 2e36c952e..f54d21140 100644 --- a/packages/i18n/src/locales/zh-Hant/settings.yaml +++ b/packages/i18n/src/locales/zh-Hant/settings.yaml @@ -1350,6 +1350,20 @@ pages: description: 可在控制台取得的 App ID label: 應用程式 ID title: 火山引擎 + volcengine-streaming: + title: 火山引擎（雙向串流 TTS） + description: 透過 AIRI UnSpeech 代理使用低延遲火山引擎語音合成 + login-required: 請先登入 AIRI，讓瀏覽器安全使用 WebSocket 代理。 + config-description: 設定火山引擎新控制台的 X-Api-Key。現有火山引擎 V1 提供商不受影響。 + model-placeholder: 選擇串流模型 + privacy-title: 金鑰使用方式 + privacy-description: 金鑰保存在目前裝置的提供商設定中。每次 WSS 工作階段中，AIRI 僅在記憶體轉發金鑰供 UnSpeech 驗證，不會儲存在伺服器。 + preview-text: 你好，這是火山引擎雙向串流語音合成的試聽範例。 + fields: + api-key: + label: X-Api-Key + description: 火山引擎新控制台提供的 X-Api-Key。此串流介面不支援 App ID 與舊控制台 Access Key。 + placeholder: 輸入 X-Api-Key volcengine-coding-plan: description: Volcengine Coding Plan title: Volcengine Coding Plan diff --git a/packages/stage-pages/src/pages/settings/providers/speech/official-provider-speech-streaming.vue b/packages/stage-pages/src/pages/settings/providers/speech/official-provider-speech-streaming.vue index 4d5cf617b..a3f3a002e 100644 --- a/packages/stage-pages/src/pages/settings/providers/speech/official-provider-speech-streaming.vue +++ b/packages/stage-pages/src/pages/settings/providers/speech/official-provider-speech-streaming.vue @@ -97,6 +97,10 @@ async function handleGenerateSpeech(input: string, voiceId: string, _useSSML: bo throw new Error(`Streaming model id missing backend prefix: ${requestedModel}`) const apiResourceId = requestedModel.slice(slashIndex + 1) const result = await streamingSynthesize({ + connection: { + credentialMode: 'official', + providerId, + }, model: requestedModel, voice: voiceId, input, diff --git a/packages/stage-pages/src/pages/settings/providers/speech/volcengine-streaming.vue b/packages/stage-pages/src/pages/settings/providers/speech/volcengine-streaming.vue new file mode 100644 index 000000000..06bbd5c27 --- /dev/null +++ b/packages/stage-pages/src/pages/settings/providers/speech/volcengine-streaming.vue @@ -0,0 +1,187 @@ + + + + + +meta: + layout: settings + stageTransition: + name: slide + diff --git a/packages/stage-ui/src/components/scenes/Stage.vue b/packages/stage-ui/src/components/scenes/Stage.vue index 764bc8ca7..1aa4d85dd 100644 --- a/packages/stage-ui/src/components/scenes/Stage.vue +++ b/packages/stage-ui/src/components/scenes/Stage.vue @@ -36,7 +36,7 @@ import { useIOTraceBridge } from '../../composables/use-io-trace-bridge' import { initIOTracer } from '../../composables/use-io-tracer' import { useSpeechPipelineAnalytics } from '../../composables/use-speech-pipeline-analytics' import { Emotion, EMOTION_EmotionMotionName_value, EMOTION_VRMExpressionName_value, EmotionThinkMotionName } from '../../constants/emotions' -import { getDefaultStreamingModel, getDefinedProvider } from '../../libs/providers/providers' +import { getDefinedProvider } from '../../libs/providers/providers' import { OFFICIAL_SPEECH_PROVIDER_ID, OFFICIAL_SPEECH_STREAMING_PROVIDER_ID } from '../../libs/providers/providers/official' import { bindSpeakingStateToPlaybackManager } from '../../libs/speech/playback-speaking-state' import { createStageTtsSession } from '../../libs/speech/tts-session' @@ -697,17 +697,31 @@ function stopSpeechOutput(reason: string) { resetAssistantSpeechSurface(reason) } -/** - * Resolves the official streaming TTS model for the current Stage session. - */ function resolveStreamingSessionModel(): string | null { const activeModel = activeSpeechModel.value as string | undefined - const sessionModel = activeModel?.includes('/') ? activeModel : getDefaultStreamingModel() + const definition = activeSpeechProvider.value ? getDefinedProvider(activeSpeechProvider.value) : undefined + const sessionModel = activeModel?.includes('/') + ? activeModel + : definition?.capabilities?.speech?.getDefaultModel?.() if (!sessionModel?.includes('/')) return null return sessionModel } +function resolveStreamingConnection() { + const providerId = activeSpeechProvider.value + if (!providerId) + return null + const definition = getDefinedProvider(providerId) + const resolver = definition?.capabilities?.speech?.resolveConnection + if (!resolver) + return null + const connection = resolver(providersStore.getProviderConfig(providerId) ?? {}) + if (connection.credentialMode === 'byok' && !connection.apiKey?.trim()) + return null + return connection +} + function buildStreamingSnapshot(turnId: string): StreamingSessionSnapshot | null { if (speechMuted.value) return null @@ -730,12 +744,16 @@ function buildStreamingSnapshot(turnId: string): StreamingSessionSnapshot | null const sessionModel = resolveStreamingSessionModel() if (!sessionModel) return null + const connection = resolveStreamingConnection() + if (!connection) + return null const apiResourceId = sessionModel.split('/', 2)[1] // TTS 2.0 / ICL 2.0 ship subtitles asynchronously relative to audio // (per the wire spec), so chunk-on-sentence-end would drop frames. // Buffer the entire session and decode at session.finished instead. const bufferEntireSession = apiResourceId.startsWith('seed-tts-2.0') || apiResourceId.startsWith('seed-icl-2.0') return { + connection, model: sessionModel, voice: voiceId, voiceType: resolveStageVoiceType(), diff --git a/packages/stage-ui/src/libs/index.ts b/packages/stage-ui/src/libs/index.ts index fe0d6f090..d90c5b3e6 100644 --- a/packages/stage-ui/src/libs/index.ts +++ b/packages/stage-ui/src/libs/index.ts @@ -1,6 +1,7 @@ export * from './audio/manager' export * from './color-from-element' export * from './providers' +export * from './speech/streaming-connection' export * from './speech/streaming-pipeline' export * from './speech/streaming-session' export * from './speech/tts-session' diff --git a/packages/stage-ui/src/libs/providers/providers/index.ts b/packages/stage-ui/src/libs/providers/providers/index.ts index bcebf2b79..9a53699c0 100644 --- a/packages/stage-ui/src/libs/providers/providers/index.ts +++ b/packages/stage-ui/src/libs/providers/providers/index.ts @@ -6,6 +6,7 @@ import './azure-openai' import './openai-compatible' import './atlascloud' import './volcengine-coding-plan' +import './volcengine-streaming' import './byteplus' import './byteplus-coding-plan' import './n1n' @@ -46,3 +47,5 @@ export { getDefinedProvider, listProviders, } from './registry' + +export { getVolcengineStreamingDefaultModel, VOLCENGINE_STREAMING_PROVIDER_ID } from './volcengine-streaming' diff --git a/packages/stage-ui/src/libs/providers/providers/official/index.ts b/packages/stage-ui/src/libs/providers/providers/official/index.ts index f3e0319cb..32fdbf89c 100644 --- a/packages/stage-ui/src/libs/providers/providers/official/index.ts +++ b/packages/stage-ui/src/libs/providers/providers/official/index.ts @@ -230,7 +230,14 @@ export const providerOfficialSpeechStreaming = defineProvider({ // session adapter (`tts-session.ts`) picks the streaming path without // hard-coding provider id. Default for every other provider is `'rest'`. capabilities: { - speech: { transport: 'bidirectional-ws' }, + speech: { + transport: 'bidirectional-ws', + resolveConnection: () => ({ + credentialMode: 'official', + providerId: OFFICIAL_SPEECH_STREAMING_PROVIDER_ID, + }), + getDefaultModel: getDefaultStreamingModel, + }, }, createProviderConfig: () => officialConfigSchema, createProvider(_config) { diff --git a/packages/stage-ui/src/libs/providers/providers/volcengine-streaming/index.test.ts b/packages/stage-ui/src/libs/providers/providers/volcengine-streaming/index.test.ts new file mode 100644 index 000000000..c55a1b539 --- /dev/null +++ b/packages/stage-ui/src/libs/providers/providers/volcengine-streaming/index.test.ts @@ -0,0 +1,33 @@ +import { describe, expect, it } from 'vitest' + +import { providerVolcengineStreaming, VOLCENGINE_STREAMING_PROVIDER_ID } from './index' + +describe('volcengine streaming provider', () => { + it('keeps BYOK separate from the existing volcengine provider identity', () => { + expect(VOLCENGINE_STREAMING_PROVIDER_ID).toBe('volcengine-streaming') + expect(providerVolcengineStreaming.id).toBe('volcengine-streaming') + expect(providerVolcengineStreaming.capabilities?.speech?.transport).toBe('bidirectional-ws') + + const connection = providerVolcengineStreaming.capabilities?.speech?.resolveConnection?.({ + apiKey: ' user-x-api-key ', + }) + expect(connection).toEqual({ + credentialMode: 'byok', + providerId: 'volcengine-streaming', + apiKey: 'user-x-api-key', + }) + }) + + it('requires an X-Api-Key before the provider becomes configured', async () => { + const validator = providerVolcengineStreaming.validators?.validateConfig?.[0]?.({ t: input => input }) + expect(validator).toBeDefined() + + const missing = await validator!.validator({ apiKey: '' }, { t: input => input }) + const configured = await validator!.validator({ apiKey: 'user-key' }, { t: input => input }) + + expect(missing.valid).toBe(false) + expect(missing.reason).toBe('X-Api-Key is required.') + expect(configured.valid).toBe(true) + expect(configured.reason).toBe('') + }) +}) diff --git a/packages/stage-ui/src/libs/providers/providers/volcengine-streaming/index.ts b/packages/stage-ui/src/libs/providers/providers/volcengine-streaming/index.ts new file mode 100644 index 000000000..4b10274f7 --- /dev/null +++ b/packages/stage-ui/src/libs/providers/providers/volcengine-streaming/index.ts @@ -0,0 +1,148 @@ +import type { ModelInfo, VoiceInfo } from '../../types' + +import { createUnVolcengine } from 'unspeech' +import { z } from 'zod' + +import { getAuthToken } from '../../../auth' +import { SERVER_URL } from '../../../server' +import { defineProvider } from '../registry' + +export const VOLCENGINE_STREAMING_PROVIDER_ID = 'volcengine-streaming' + +const configSchema = z.object({ + apiKey: z.string(), +}) + +type VolcengineStreamingConfig = z.input + +let defaultModelId: string | null = null + +function authHeaders(): Record { + const headers: Record = { Accept: 'application/json' } + const token = getAuthToken() + if (token) + headers.Authorization = `Bearer ${token}` + return headers +} + +export function getVolcengineStreamingDefaultModel(): string | null { + return defaultModelId +} + +export const providerVolcengineStreaming = defineProvider({ + id: VOLCENGINE_STREAMING_PROVIDER_ID, + order: 31, + name: 'Volcengine Streaming TTS', + nameLocalize: ({ t }) => t('settings.pages.providers.provider.volcengine-streaming.title'), + description: 'Low-latency Volcengine TTS through the AIRI UnSpeech bridge.', + descriptionLocalize: ({ t }) => t('settings.pages.providers.provider.volcengine-streaming.description'), + tasks: ['text-to-speech'], + iconColor: 'i-lobe-icons:volcengine', + createProviderConfig: ({ t }) => configSchema.extend({ + apiKey: configSchema.shape.apiKey.meta({ + labelLocalized: t('settings.pages.providers.common.fields.field.api-key.label'), + descriptionLocalized: t('settings.pages.providers.provider.volcengine-streaming.fields.api-key.description'), + placeholderLocalized: t('settings.pages.providers.provider.volcengine-streaming.fields.api-key.placeholder'), + type: 'password', + }), + }), + createProvider(config) { + // The provider instance satisfies the shared speech-provider boundary. + // Runtime synthesis uses the bidirectional session capability below, not + // this REST-compatible instance. + const apiKey = typeof config.apiKey === 'string' ? config.apiKey.trim() : '' + return createUnVolcengine(apiKey, 'https://unspeech.hyp3r.link/v1/') + }, + capabilities: { + speech: { + transport: 'bidirectional-ws', + resolveConnection: (config) => { + const apiKey = typeof config.apiKey === 'string' ? config.apiKey.trim() : '' + return { + credentialMode: 'byok', + providerId: VOLCENGINE_STREAMING_PROVIDER_ID, + apiKey, + } + }, + getDefaultModel: getVolcengineStreamingDefaultModel, + }, + }, + validationRequiredWhen: () => true, + validators: { + validateConfig: [ + ({ t }) => ({ + id: 'volcengine-streaming:check-config', + name: t('settings.pages.providers.catalog.edit.validators.openai-compatible.check-config.title'), + validator: async (config) => { + const apiKey = typeof config.apiKey === 'string' ? config.apiKey.trim() : '' + const valid = apiKey.length > 0 + return { + errors: valid ? [] : [{ error: new Error('X-Api-Key is required.') }], + reason: valid ? '' : 'X-Api-Key is required.', + reasonKey: '', + valid, + } + }, + }), + ], + }, + extraMethods: { + listModels: async (): Promise => { + defaultModelId = null + const response = await globalThis.fetch(`${SERVER_URL}/api/v1/audio/models/streaming`, { headers: authHeaders() }) + if (!response.ok) + throw new Error(`streaming models upstream ${response.status}: ${await response.text().catch(() => '')}`.slice(0, 256)) + + const data = await response.json() as { + models?: { id: string, name?: string, description?: string }[] + default?: string | null + } + if (!Array.isArray(data.models)) + throw new Error('streaming models upstream missing models[]') + + defaultModelId = typeof data.default === 'string' && data.default.length > 0 ? data.default : null + return data.models.map(model => ({ + id: model.id, + name: model.name ?? model.id, + provider: VOLCENGINE_STREAMING_PROVIDER_ID, + description: model.description, + })) + }, + listVoices: async (_config, _provider, model): Promise => { + const resourceId = model?.includes('/') ? model.split('/', 2)[1] : model + const url = new URL(`${SERVER_URL}/api/v1/audio/voices/streaming`) + if (resourceId) + url.searchParams.set('model', resourceId) + + const response = await globalThis.fetch(url, { headers: authHeaders() }) + if (!response.ok) + throw new Error(`streaming voices upstream ${response.status}: ${await response.text().catch(() => '')}`.slice(0, 256)) + + const data = await response.json() as { + voices?: { + id: string + name: string + description?: string + labels?: Record + languages?: { code: string, title: string }[] + preview_audio_url?: string + }[] + } + if (!Array.isArray(data.voices)) + throw new Error('streaming voices upstream returned malformed body') + + return data.voices.map((voice) => { + const gender = typeof voice.labels?.gender === 'string' ? voice.labels.gender.toLowerCase() : undefined + return { + id: voice.id, + name: voice.name, + provider: VOLCENGINE_STREAMING_PROVIDER_ID, + description: voice.description, + gender, + previewURL: voice.preview_audio_url, + languages: Array.isArray(voice.languages) ? voice.languages : [], + } + }) + }, + }, +}) diff --git a/packages/stage-ui/src/libs/providers/source-metadata.ts b/packages/stage-ui/src/libs/providers/source-metadata.ts index ae697ec5d..69f7da305 100644 --- a/packages/stage-ui/src/libs/providers/source-metadata.ts +++ b/packages/stage-ui/src/libs/providers/source-metadata.ts @@ -95,6 +95,7 @@ const providerSourceMetadataById = { 'speech-noop': false, 'together-ai': paidCloud, 'volcengine': paidCloud, + 'volcengine-streaming': paidCloud, 'volcengine-coding-plan': paidCloud, 'xai': paidCloud, 'zai': paidCloud, diff --git a/packages/stage-ui/src/libs/providers/types.ts b/packages/stage-ui/src/libs/providers/types.ts index 84e4fd433..d3712db68 100644 --- a/packages/stage-ui/src/libs/providers/types.ts +++ b/packages/stage-ui/src/libs/providers/types.ts @@ -15,6 +15,8 @@ import type { MaybePromise } from 'clustr' import type { ComposerTranslation } from 'vue-i18n' import type { $ZodType } from 'zod/v4/core' +import type { StreamingTtsConnection } from '../speech/streaming-connection' + export type ProviderInstance = | ChatProvider | ChatProviderWithExtraOptions @@ -214,6 +216,10 @@ export interface ProviderDefinition { */ speech?: { transport: 'rest' | 'bidirectional-ws' + /** Resolves provider-owned credential policy for one streaming session. */ + resolveConnection?: (config: TConfig) => StreamingTtsConnection + /** Returns the provider's discovered default streaming model. */ + getDefaultModel?: () => string | null } } /** diff --git a/packages/stage-ui/src/libs/speech/streaming-connection.ts b/packages/stage-ui/src/libs/speech/streaming-connection.ts new file mode 100644 index 000000000..d49c2c851 --- /dev/null +++ b/packages/stage-ui/src/libs/speech/streaming-connection.ts @@ -0,0 +1,13 @@ +/** + * Connection policy snapshotted from the active streaming speech provider. + * + * Official sessions use AIRI-hosted credentials and Flux. BYOK sessions send + * the user credential inside the authenticated WSS session so the AIRI bridge + * can inject it into UnSpeech's upstream upgrade request. + */ +export interface StreamingTtsConnection { + credentialMode: 'official' | 'byok' + providerId: 'official-provider-speech-streaming' | 'volcengine-streaming' + /** Plaintext credential used only for a BYOK upstream handshake. */ + apiKey?: string +} diff --git a/packages/stage-ui/src/libs/speech/streaming-pipeline.test.ts b/packages/stage-ui/src/libs/speech/streaming-pipeline.test.ts index 71891b232..10e7eb4aa 100644 --- a/packages/stage-ui/src/libs/speech/streaming-pipeline.test.ts +++ b/packages/stage-ui/src/libs/speech/streaming-pipeline.test.ts @@ -8,6 +8,11 @@ import { WebSocketServer } from 'ws' import { createStreamingTtsPipeline } from './streaming-pipeline' +const officialConnection = { + credentialMode: 'official', + providerId: 'official-provider-speech-streaming', +} as const + vi.mock('../auth', () => ({ getAuthToken: () => 'test-jwt', })) @@ -19,6 +24,7 @@ interface MockServer { url: string receivedFrames: Array<{ kind: 'text' | 'binary', data: string | Buffer }> observedVoiceTypes: string[] + observedCredentialModes: string[] /** Resolves when the server has observed a `start` frame from the client. */ startObserved: Promise stop: () => Promise @@ -27,6 +33,7 @@ interface MockServer { async function startMockServer(handler: (ws: import('ws').WebSocket) => void): Promise { const receivedFrames: MockServer['receivedFrames'] = [] const observedVoiceTypes: string[] = [] + const observedCredentialModes: string[] = [] const httpServer = createServer() const wss = new WebSocketServer({ server: httpServer }) @@ -40,6 +47,9 @@ async function startMockServer(handler: (ws: import('ws').WebSocket) => void): P const voiceType = u.searchParams.get('tts_voice_type') if (voiceType != null) observedVoiceTypes.push(voiceType) + const credentialMode = u.searchParams.get('tts_credential_mode') + if (credentialMode != null) + observedCredentialModes.push(credentialMode) ws.on('message', (data, isBinary) => { const buf = Buffer.isBuffer(data) ? data : Buffer.from(data as ArrayBuffer) @@ -64,6 +74,7 @@ async function startMockServer(handler: (ws: import('ws').WebSocket) => void): P url: `http://127.0.0.1:${port}`, receivedFrames, observedVoiceTypes, + observedCredentialModes, startObserved, async stop() { wss.close() @@ -132,6 +143,7 @@ describe('createStreamingTtsPipeline', () => { const onDone = vi.fn() const handle = createStreamingTtsPipeline({ + connection: officialConnection, serverUrl: server.url, model: 'volcengine/seed-tts-1.0', voice: 'mock', @@ -169,6 +181,44 @@ describe('createStreamingTtsPipeline', () => { expect(calls[1].audio.__byteLength).toBe(chunks[2].length) }) + it('opens chat streaming with the BYOK credentials frame before start', async () => { + server = await startMockServer((ws) => { + ws.on('message', (data, isBinary) => { + if (isBinary) + return + const frame = JSON.parse(data.toString()) as { event?: string } + if (frame.event === 'finish') + ws.send(JSON.stringify({ event: 'session.finished', payload: { usage: { text_words: 5 } } })) + }) + }) + + const onDone = vi.fn() + const handle = createStreamingTtsPipeline({ + connection: { + credentialMode: 'byok', + providerId: 'volcengine-streaming', + apiKey: 'chat-byok-key', + }, + serverUrl: server.url, + model: 'volcengine/seed-tts-2.0', + voice: 'mock', + audioContext: makeStubAudioContext(), + onDone, + }) + handle.appendText('hello') + handle.finish() + + await new Promise((resolve) => { + onDone.mockImplementation(() => resolve()) + setTimeout(resolve, 1500) + }) + + const frames = server.receivedFrames.map(frame => JSON.parse(frame.data as string) as Record) + expect(frames.map(frame => frame.event)).toEqual(['credentials', 'start', 'text', 'finish']) + expect(frames[0]?.api_key).toBe('chat-byok-key') + expect(server.observedCredentialModes).toEqual(['byok']) + }) + it('buffers entire session when bufferEntireSession is true', async () => { const chunks = [Buffer.from([1, 2, 3, 4]), Buffer.from([5, 6, 7, 8])] server = await startMockServer((ws) => { @@ -190,6 +240,7 @@ describe('createStreamingTtsPipeline', () => { const onSentence = vi.fn() const handle = createStreamingTtsPipeline({ + connection: officialConnection, serverUrl: server.url, model: 'volcengine/seed-tts-2.0', voice: 'mock', @@ -223,6 +274,7 @@ describe('createStreamingTtsPipeline', () => { const onError = vi.fn() const onDone = vi.fn() createStreamingTtsPipeline({ + connection: officialConnection, serverUrl: server.url, model: 'volcengine/seed-tts-1.0', voice: 'mock', @@ -256,6 +308,7 @@ describe('createStreamingTtsPipeline', () => { const onError = vi.fn() const onDone = vi.fn() createStreamingTtsPipeline({ + connection: officialConnection, serverUrl: server.url, model: 'volcengine/seed-tts-1.0', voice: 'mock', @@ -287,6 +340,7 @@ describe('createStreamingTtsPipeline', () => { const onDone = vi.fn() const handle = createStreamingTtsPipeline({ + connection: officialConnection, serverUrl: server.url, model: 'volcengine/seed-tts-1.0', voice: 'mock', diff --git a/packages/stage-ui/src/libs/speech/streaming-pipeline.ts b/packages/stage-ui/src/libs/speech/streaming-pipeline.ts index 247f7a83c..b621c2c2e 100644 --- a/packages/stage-ui/src/libs/speech/streaming-pipeline.ts +++ b/packages/stage-ui/src/libs/speech/streaming-pipeline.ts @@ -1,3 +1,5 @@ +import type { StreamingTtsConnection } from './streaming-connection' + import { getAuthToken } from '../auth' import { SERVER_URL } from '../server' @@ -36,6 +38,8 @@ export interface StreamingTtsPipelineEvents { } export interface StreamingTtsPipelineOptions extends StreamingTtsPipelineEvents { + /** Provider-owned credential and billing policy for this session. */ + connection: StreamingTtsConnection /** Server URL override. Defaults to {@link SERVER_URL}. */ serverUrl?: string /** Override the auth token (Bearer). Defaults to {@link getAuthToken}. */ @@ -125,6 +129,7 @@ export function createStreamingTtsPipeline(options: StreamingTtsPipelineOptions) ttsTrigger: options.ttsTrigger ?? 'auto', ttsSource: options.ttsSource ?? 'chat_auto_tts', ttsVoiceType: options.ttsVoiceType ?? 'unknown', + connection: options.connection, }) const ws = new WebSocket(wsUrl) ws.binaryType = 'arraybuffer' @@ -239,6 +244,13 @@ export function createStreamingTtsPipeline(options: StreamingTtsPipelineOptions) } ws.addEventListener('open', () => { + if (options.connection.credentialMode === 'byok') { + ws.send(JSON.stringify({ + event: 'credentials', + provider: 'volcengine', + api_key: options.connection.apiKey, + })) + } const startFrame = { event: 'start', model: options.model, @@ -418,6 +430,7 @@ function toWebSocketUrl( ttsTrigger: 'auto' | 'manual' ttsSource: 'chat_auto_tts' | 'manual_preview' | 'settings_test' ttsVoiceType: 'official_default' | 'official_selected' | 'custom_configured' | 'voice_pack' | 'unknown' + connection: StreamingTtsConnection }, ): string { const u = new URL(path, httpBase) @@ -426,6 +439,8 @@ function toWebSocketUrl( u.searchParams.set('tts_trigger', analytics.ttsTrigger) u.searchParams.set('tts_source', analytics.ttsSource) u.searchParams.set('tts_voice_type', analytics.ttsVoiceType) + u.searchParams.set('tts_credential_mode', analytics.connection.credentialMode) + u.searchParams.set('tts_provider_id', analytics.connection.providerId) return u.toString() } diff --git a/packages/stage-ui/src/libs/speech/streaming-session.test.ts b/packages/stage-ui/src/libs/speech/streaming-session.test.ts index f905bc5b7..f6afc4354 100644 --- a/packages/stage-ui/src/libs/speech/streaming-session.test.ts +++ b/packages/stage-ui/src/libs/speech/streaming-session.test.ts @@ -8,6 +8,11 @@ import { WebSocketServer } from 'ws' import { streamingSynthesize } from './streaming-session' +const officialConnection = { + credentialMode: 'official', + providerId: 'official-provider-speech-streaming', +} as const + vi.mock('../auth', () => ({ getAuthToken: () => 'test-jwt', })) @@ -20,6 +25,8 @@ interface MockServer { url: string observedTokens: string[] observedVoiceTypes: string[] + observedCredentialModes: string[] + observedProviderIds: string[] closeUnexpectedly: () => void stop: () => Promise } @@ -27,6 +34,8 @@ interface MockServer { async function startMockServer(handler: (ws: import('ws').WebSocket) => void): Promise { const observedTokens: string[] = [] const observedVoiceTypes: string[] = [] + const observedCredentialModes: string[] = [] + const observedProviderIds: string[] = [] const httpServer = createServer() const wss = new WebSocketServer({ server: httpServer }) @@ -41,6 +50,12 @@ async function startMockServer(handler: (ws: import('ws').WebSocket) => void): P const voiceType = u.searchParams.get('tts_voice_type') if (voiceType != null) observedVoiceTypes.push(voiceType) + const credentialMode = u.searchParams.get('tts_credential_mode') + if (credentialMode != null) + observedCredentialModes.push(credentialMode) + const providerId = u.searchParams.get('tts_provider_id') + if (providerId != null) + observedProviderIds.push(providerId) handler(ws) }) @@ -52,6 +67,8 @@ async function startMockServer(handler: (ws: import('ws').WebSocket) => void): P url: `http://127.0.0.1:${port}`, observedTokens, observedVoiceTypes, + observedCredentialModes, + observedProviderIds, closeUnexpectedly: () => { activeWs?.close(1011, 'simulated_truncation') }, @@ -106,6 +123,7 @@ describe('streamingSynthesize', () => { }) const result = await streamingSynthesize({ + connection: officialConnection, serverUrl: server.url, model: 'volcengine/seed-tts-2.0', voice: 'mock', @@ -124,6 +142,41 @@ describe('streamingSynthesize', () => { expect(server.observedVoiceTypes).toEqual(['official_selected']) }) + it('sends BYOK credentials before the UnSpeech start frame and marks the connection mode', async () => { + const frames: Array> = [] + server = await startMockServer((ws) => { + ws.on('message', (data, isBinary) => { + if (isBinary) + return + const frame = JSON.parse(data.toString()) as Record + frames.push(frame) + if (frame.event === 'finish') + ws.send(JSON.stringify({ event: 'session.finished', payload: { usage: { text_words: 5 } } })) + }) + }) + + await streamingSynthesize({ + connection: { + credentialMode: 'byok', + providerId: 'volcengine-streaming', + apiKey: 'byok-test-key', + }, + serverUrl: server.url, + model: 'volcengine/seed-tts-2.0', + voice: 'mock', + input: 'hello', + }) + + expect(frames.map(frame => frame.event)).toEqual(['credentials', 'start', 'text', 'finish']) + expect(frames[0]).toEqual({ + event: 'credentials', + provider: 'volcengine', + api_key: 'byok-test-key', + }) + expect(server.observedCredentialModes).toEqual(['byok']) + expect(server.observedProviderIds).toEqual(['volcengine-streaming']) + }) + it('rejects when the ws closes before session.finished (codex HIGH #2)', async () => { // Server sends some audio chunks then closes the ws WITHOUT emitting // session.finished. Pre-fix behavior: streamingSynthesize would @@ -150,6 +203,7 @@ describe('streamingSynthesize', () => { }) await expect(streamingSynthesize({ + connection: officialConnection, serverUrl: server.url, model: 'volcengine/seed-tts-2.0', voice: 'mock', @@ -176,6 +230,7 @@ describe('streamingSynthesize', () => { }) await expect(streamingSynthesize({ + connection: officialConnection, serverUrl: server.url, model: 'volcengine/seed-tts-2.0', voice: 'mock', @@ -197,6 +252,7 @@ describe('streamingSynthesize', () => { const ctrl = new AbortController() const promise = streamingSynthesize({ + connection: officialConnection, serverUrl: server.url, model: 'volcengine/seed-tts-2.0', voice: 'mock', diff --git a/packages/stage-ui/src/libs/speech/streaming-session.ts b/packages/stage-ui/src/libs/speech/streaming-session.ts index e64bd71ab..c57c74f0f 100644 --- a/packages/stage-ui/src/libs/speech/streaming-session.ts +++ b/packages/stage-ui/src/libs/speech/streaming-session.ts @@ -1,3 +1,5 @@ +import type { StreamingTtsConnection } from './streaming-connection' + import { getAuthToken } from '../auth' import { SERVER_URL } from '../server' @@ -36,6 +38,8 @@ export interface StreamingTtsSessionResult { } export interface StreamingTtsSessionOptions { + /** Provider-owned credential and billing policy for this session. */ + connection: StreamingTtsConnection /** Server URL override. Defaults to {@link SERVER_URL}. */ serverUrl?: string /** Override the auth token (Bearer). Defaults to {@link getAuthToken}. */ @@ -95,6 +99,7 @@ export async function streamingSynthesize(options: StreamingTtsSessionOptions): ttsTrigger: options.ttsTrigger ?? 'manual', ttsSource: options.ttsSource ?? 'manual_preview', ttsVoiceType: options.ttsVoiceType ?? 'unknown', + connection: options.connection, }) const audioChunks: ArrayBuffer[] = [] @@ -143,6 +148,13 @@ export async function streamingSynthesize(options: StreamingTtsSessionOptions): } ws.addEventListener('open', () => { + if (options.connection.credentialMode === 'byok') { + ws.send(JSON.stringify({ + event: 'credentials', + provider: 'volcengine', + api_key: options.connection.apiKey, + })) + } const startFrame = { event: 'start', model: options.model, @@ -243,6 +255,7 @@ function toWebSocketUrl( ttsTrigger: 'auto' | 'manual' ttsSource: 'chat_auto_tts' | 'manual_preview' | 'settings_test' ttsVoiceType: 'official_default' | 'official_selected' | 'custom_configured' | 'voice_pack' | 'unknown' + connection: StreamingTtsConnection }, ): string { const u = new URL(path, httpBase) @@ -251,6 +264,8 @@ function toWebSocketUrl( u.searchParams.set('tts_trigger', analytics.ttsTrigger) u.searchParams.set('tts_source', analytics.ttsSource) u.searchParams.set('tts_voice_type', analytics.ttsVoiceType) + u.searchParams.set('tts_credential_mode', analytics.connection.credentialMode) + u.searchParams.set('tts_provider_id', analytics.connection.providerId) return u.toString() } diff --git a/packages/stage-ui/src/libs/speech/tts-session.test.ts b/packages/stage-ui/src/libs/speech/tts-session.test.ts index 4fadf051a..deff4407b 100644 --- a/packages/stage-ui/src/libs/speech/tts-session.test.ts +++ b/packages/stage-ui/src/libs/speech/tts-session.test.ts @@ -46,6 +46,10 @@ function makePlaybackManagerStub(): PlaybackManagerSubset< function makeStreamingSnapshot(overrides: Partial = {}): StreamingSessionSnapshot { return { + connection: { + credentialMode: 'official', + providerId: 'official-provider-speech-streaming', + }, model: 'volcengine/seed-tts-2.0', voice: 'mock-voice', voiceType: 'official_selected', diff --git a/packages/stage-ui/src/libs/speech/tts-session.ts b/packages/stage-ui/src/libs/speech/tts-session.ts index e4007ace5..47655f584 100644 --- a/packages/stage-ui/src/libs/speech/tts-session.ts +++ b/packages/stage-ui/src/libs/speech/tts-session.ts @@ -1,5 +1,6 @@ import type { IntentHandle, IntentOptions, PlaybackItem } from '@proj-airi/pipelines-audio' +import type { StreamingTtsConnection } from './streaming-connection' import type { StreamingTtsPipelineOptions } from './streaming-pipeline' import { createStreamingTtsPipeline } from './streaming-pipeline' @@ -69,6 +70,7 @@ function fromIntent(intent: IntentHandleSubset): StageTtsSession { * for cancelling and re-opening. */ export interface StreamingSessionSnapshot { + connection: StreamingTtsConnection model: string voice: string voiceType: 'official_default' | 'official_selected' | 'custom_configured' | 'voice_pack' | 'unknown' @@ -157,6 +159,7 @@ export function createStreamingTtsSession( let terminated = false const handle = pipelineFactory({ + connection: snapshot.connection, model: snapshot.model, voice: snapshot.voice, ttsVoiceType: snapshot.voiceType, diff --git a/packages/stage-ui/src/stores/modules/speech.ts b/packages/stage-ui/src/stores/modules/speech.ts index 36cf66841..ce76925ed 100644 --- a/packages/stage-ui/src/stores/modules/speech.ts +++ b/packages/stage-ui/src/stores/modules/speech.ts @@ -12,7 +12,8 @@ import { useI18n } from 'vue-i18n' import { toXml } from 'xast-util-to-xml' import { x } from 'xastscript' -import { getDefaultSpeechModel, getDefaultStreamingModel, OFFICIAL_SPEECH_PROVIDER_ID, OFFICIAL_SPEECH_STREAMING_PROVIDER_ID, setupOfficialSpeechAutoPick } from '../../libs/providers/providers/official' +import { getDefinedProvider } from '../../libs/providers/providers' +import { getDefaultSpeechModel, OFFICIAL_SPEECH_PROVIDER_ID, OFFICIAL_SPEECH_STREAMING_PROVIDER_ID, setupOfficialSpeechAutoPick } from '../../libs/providers/providers/official' import { useProvidersStore } from '../providers' export function toSignedPercent(value: number): string { @@ -141,18 +142,15 @@ export const useSpeechStore = defineStore('speech', () => { activeSpeechVoice.value = undefined } - // Streaming TTS voices are model-scoped: the server only returns recommended - // voices for an explicit `?model=`. Ensure the active model is a valid - // streaming model id so voice loading gets the right recommendations (parity - // with the HTTP provider's auto-pick). Reseeds the server-curated default - // both when no model is selected AND when `activeSpeechModel` still holds a - // stale id from a previously-active provider (the global model ref is shared - // across providers, and the per-surface reset may not have run yet). No-op - // for non-streaming providers. + // Streaming TTS voices are model-scoped. Resolve the default through the + // active provider capability so official and BYOK transports share the same + // state transition without Stage knowing provider ids. function ensureStreamingDefaultModel() { - if (activeSpeechProvider.value !== OFFICIAL_SPEECH_STREAMING_PROVIDER_ID) + const providerId = activeSpeechProvider.value + const speechCapability = getDefinedProvider(providerId)?.capabilities?.speech + if (speechCapability?.transport !== 'bidirectional-ws') return - const streamingModels = providersStore.getModelsForProvider(OFFICIAL_SPEECH_STREAMING_PROVIDER_ID) + const streamingModels = providersStore.getModelsForProvider(providerId) const hasValidSelection = !!activeSpeechModel.value && streamingModels.some(m => m.id === activeSpeechModel.value) if (hasValidSelection) return @@ -160,7 +158,7 @@ export const useSpeechStore = defineStore('speech', () => { // When no default can be resolved yet (catalog not loaded), clear it to '' // so callers pass `undefined` (server returns the full streaming catalog) // rather than forwarding a stale non-streaming model id as `?model=`. - const nextModel = getDefaultStreamingModel() ?? streamingModels[0]?.id ?? '' + const nextModel = speechCapability.getDefaultModel?.() ?? streamingModels[0]?.id ?? '' if (activeSpeechModel.value === nextModel) return activeSpeechModel.value = nextModel diff --git a/server/apps/api/src/app.ts b/server/apps/api/src/app.ts index 15d17b367..af9ce1681 100644 --- a/server/apps/api/src/app.ts +++ b/server/apps/api/src/app.ts @@ -4,7 +4,7 @@ import type { AuthInstance } from './libs/auth' import type { Database } from './libs/db' import type { Env } from './libs/env' import type { OtelInstance } from './otel' -import type { StreamingTtsVoiceType } from './routes/audio-speech-ws/session' +import type { StreamingTtsProviderId, StreamingTtsVoiceType } from './routes/audio-speech-ws/session' import type { ConfigKVService } from './services/adapters/config-kv' import type { AdminFluxGrantsService } from './services/domain/admin/flux-grants' import type { AdminRouterConfigService } from './services/domain/admin/router-config' @@ -222,6 +222,8 @@ export async function buildApp(deps: AppDeps) { trigger: c.req.query('tts_trigger') === 'auto' ? 'auto' : 'manual', source: parseTtsSource(c.req.query('tts_source'), 'audio.speech.ws'), voiceType: parseTtsVoiceType(c.req.query('tts_voice_type')), + credentialMode: c.req.query('tts_credential_mode') === 'byok' ? 'byok' : 'official', + providerId: parseStreamingTtsProviderId(c.req.query('tts_provider_id')), }) })) @@ -503,6 +505,19 @@ function parseTtsVoiceType( } } +/** + * Bounds client-provided provider identity before it reaches product events. + */ +function parseStreamingTtsProviderId(value: string | undefined): StreamingTtsProviderId { + switch (value) { + case 'official-provider-speech-streaming': + case 'volcengine-streaming': + return value + default: + return 'unknown' + } +} + export type AppType = Awaited>['app'] export async function createApp() { diff --git a/server/apps/api/src/routes/audio-speech-ws/index.ts b/server/apps/api/src/routes/audio-speech-ws/index.ts index 42d6cee75..8f6075dcb 100644 --- a/server/apps/api/src/routes/audio-speech-ws/index.ts +++ b/server/apps/api/src/routes/audio-speech-ws/index.ts @@ -23,8 +23,9 @@ export type { AudioSpeechWsHandlersOptions } from './types' * Expects: * - The route handler has already resolved auth via the `?token=` query * (see app.ts wiring) and passes a verified `userId` in. - * - The client sends a `start` control frame first. The session validates the - * requested streaming model and voice before dialing upstream. + * - Official clients send a `start` control frame first. BYOK clients send an + * AIRI-private `credentials` frame followed by `start`. The session consumes + * credentials locally, then validates model and voice before dialing upstream. * * Returns: * - A function that takes `userId` and returns hono `WSEvents`. Each call diff --git a/server/apps/api/src/routes/audio-speech-ws/route.test.ts b/server/apps/api/src/routes/audio-speech-ws/route.test.ts index 16bfc0429..d21cbd6ee 100644 --- a/server/apps/api/src/routes/audio-speech-ws/route.test.ts +++ b/server/apps/api/src/routes/audio-speech-ws/route.test.ts @@ -312,6 +312,66 @@ describe('audio-speech-ws route', () => { })) }) + it('uses the client BYOK credential without forwarding it as a frame or charging Flux', async () => { + upstream = await startMockUpstream([ + { kind: 'json', payload: { event: 'session.finished', payload: { usage: { text_words: 21 } } } }, + ]) + // A zero balance would reject the official path. BYOK must not run that + // pre-flight or any final Flux accumulation. + const deps = makeFakeDeps({ upstreamURL: upstream.url, restBaseURL: upstream.restBaseURL, fluxBalance: 0 }) + const handlers = createAudioSpeechWsHandlers(deps as any) + const events = handlers('user-byok', { + credentialMode: 'byok', + providerId: 'volcengine-streaming', + voiceType: 'custom_configured', + }) + const client = makeMockClientWs() + + await driveClientSession(events, client, [ + JSON.stringify({ event: 'credentials', provider: 'volcengine', api_key: 'user-owned-x-api-key' }), + JSON.stringify({ event: 'start', model: 'volcengine/seed-tts-2.0', voice: 'mock' }), + JSON.stringify({ event: 'text', text: 'BYOK streaming' }), + JSON.stringify({ event: 'finish' }), + ]) + await new Promise(r => setTimeout(r, 200)) + + expect(upstream.observedAuth).toBe('Bearer user-owned-x-api-key') + expect(upstream.receivedFrames).toHaveLength(3) + expect(upstream.receivedFrames.map(frame => JSON.parse(frame.data as string).event)).toEqual(['start', 'text', 'finish']) + expect(deps.fluxService.getFlux).not.toHaveBeenCalled() + expect(deps.ttsMeter.assertCanAfford).not.toHaveBeenCalled() + expect(deps.ttsMeter.accumulate).not.toHaveBeenCalled() + expect(deps.productEventService.track).toHaveBeenCalledWith(expect.objectContaining({ + action: 'speech_succeeded', + metadata: expect.objectContaining({ + credential_mode: 'byok', + provider_id: 'volcengine-streaming', + flux_consumed: 0, + }), + })) + }) + + it('does not fall back to the operator key when a BYOK session omits credentials', async () => { + upstream = await startMockUpstream([]) + const deps = makeFakeDeps({ upstreamURL: upstream.url, restBaseURL: upstream.restBaseURL, fluxBalance: 100 }) + const handlers = createAudioSpeechWsHandlers(deps as any) + const events = handlers('user-byok', { + credentialMode: 'byok', + providerId: 'volcengine-streaming', + }) + const client = makeMockClientWs() + + await driveClientSession(events, client, [ + JSON.stringify({ event: 'start', model: 'volcengine/seed-tts-2.0', voice: 'mock' }), + ]) + + expect(client.closeCode).toBe(1008) + expect(client.closeReason).toBe('invalid_credentials_frame') + expect(upstream.observedAuth).toBeUndefined() + expect(upstream.receivedFrames).toHaveLength(0) + expect(deps.envelopeCrypto.decryptKey).not.toHaveBeenCalled() + }) + it('refuses the session with insufficient_flux when the user is broke', async () => { upstream = await startMockUpstream([]) const deps = makeFakeDeps({ upstreamURL: upstream.url, restBaseURL: upstream.restBaseURL, fluxBalance: 0 }) diff --git a/server/apps/api/src/routes/audio-speech-ws/session.ts b/server/apps/api/src/routes/audio-speech-ws/session.ts index 790b9987e..f4d0a82c6 100644 --- a/server/apps/api/src/routes/audio-speech-ws/session.ts +++ b/server/apps/api/src/routes/audio-speech-ws/session.ts @@ -44,7 +44,7 @@ const tracer = trace.getTracer('audio-speech-ws') export interface AudioSpeechSessionState { /** Stores the accepted client websocket. */ attachClient: (ws: WSContext) => void - /** Reads config, checks balance, decrypts the upstream key, and dials upstream after the start frame is accepted. */ + /** Resolves the selected credential policy and dials upstream after the start frame is accepted. */ dialUpstream: () => Promise /** Forwards a client frame or queues it while the upstream connection opens. */ handleClientMessage: (message: { data: unknown }, ws: WSContext) => void @@ -55,11 +55,15 @@ export interface AudioSpeechSessionState { export type StreamingTtsTrigger = 'auto' | 'manual' export type StreamingTtsSource = 'audio.speech.ws' | 'chat_auto_tts' | 'manual_preview' | 'settings_test' export type StreamingTtsVoiceType = 'official_default' | 'official_selected' | 'custom_configured' | 'voice_pack' | 'unknown' +export type StreamingTtsCredentialMode = 'official' | 'byok' +export type StreamingTtsProviderId = 'official-provider-speech-streaming' | 'volcengine-streaming' | 'unknown' export interface AudioSpeechSessionAnalytics { trigger?: StreamingTtsTrigger source?: StreamingTtsSource voiceType?: StreamingTtsVoiceType + credentialMode?: StreamingTtsCredentialMode + providerId?: StreamingTtsProviderId } /** @@ -71,7 +75,9 @@ export interface AudioSpeechSessionAnalytics { * are handled at session end. * * Expects: - * - `UNSPEECH_UPSTREAM.streaming` has a base URL and at least one encrypted key. + * - `UNSPEECH_UPSTREAM.streaming` has a base URL. + * - Official sessions have an operator key; BYOK sessions send one private + * credentials frame before the UnSpeech start frame. * * Returns: * - A connection-scoped state object with no global peer registry. @@ -94,7 +100,7 @@ export function createSessionState( let upstreamWs: WebSocket | null = null let upstreamReady = false let closed = false - let billed = false + let completed = false let startFrameAccepted = false let startValidationStarted = false let dialStarted = false @@ -102,6 +108,8 @@ export function createSessionState( let preflightFluxBalance: number | undefined let modelLabel = STREAM_MODEL_LABEL_FALLBACK let voiceLabel: string | undefined + let byokKey: Buffer | null = null + let credentialFrameAccepted = analytics.credentialMode === 'official' /** * Frames the client sent before the upstream finished dialing. Buffered to * avoid silently dropping the `start` frame; flushed in arrival order once @@ -127,7 +135,7 @@ export function createSessionState( model: modelLabel, metadata: { trigger: analytics.trigger, - ...streamingVoiceMetadata(voiceLabel, analytics.voiceType), + ...streamingSessionMetadata(voiceLabel, analytics), }, }) @@ -142,51 +150,63 @@ export function createSessionState( } const upstreamConfig = unspeech?.streaming - if (!upstreamConfig || !upstreamConfig.baseURL || upstreamConfig.keys.length === 0) { + if (!upstreamConfig?.baseURL || (analytics.credentialMode === 'official' && upstreamConfig.keys.length === 0)) { closeWithError(1008, 'streaming_tts_not_configured') return } - // Pre-flight balance check: refuse before dialing if the user cannot - // afford the worst-case session. - try { - const flux = await opts.fluxService.getFlux(userId) - preflightFluxBalance = flux.flux - await opts.ttsMeter.assertCanAfford(userId, STREAMING_PREFLIGHT_CHARS_ESTIMATE, flux.flux) - } - catch (err) { - log.withError(err).withFields({ userId }).warn('pre-flight rejected streaming tts') - // assertCanAfford throws PaymentRequiredError (402) — translate to ws - // policy-violation close. The client can read the close code/reason to - // surface a 'top up' prompt. - if (isPaymentRequiredError(err)) - closeWithBlockedPreflight(1008, 'insufficient_flux') - else - closeWithError(1011, 'flux_preflight_failed') - return + if (analytics.credentialMode === 'official') { + // Official sessions consume AIRI-hosted credentials and Flux. BYOK + // sessions are explicitly excluded so a missing user key can never + // fall back to the operator account or charge the user twice. + try { + const flux = await opts.fluxService.getFlux(userId) + preflightFluxBalance = flux.flux + await opts.ttsMeter.assertCanAfford(userId, STREAMING_PREFLIGHT_CHARS_ESTIMATE, flux.flux) + } + catch (err) { + log.withError(err).withFields({ userId }).warn('pre-flight rejected streaming tts') + // assertCanAfford throws PaymentRequiredError (402) — translate to ws + // policy-violation close. The client can read the close code/reason to + // surface a 'top up' prompt. + if (isPaymentRequiredError(err)) + closeWithBlockedPreflight(1008, 'insufficient_flux') + else + closeWithError(1011, 'flux_preflight_failed') + return + } } - // Decrypt the first key. Streaming surface does not do per-attempt key - // rotation: a live ws cannot transparently switch upstream mid-session - // without breaking audio continuity. Fallback policy belongs at the - // session-retry layer (next client connect), not inline. - const entry = upstreamConfig.keys[0] let keyPlaintext: Buffer - try { - keyPlaintext = opts.envelopeCrypto.decryptKey(entry.ciphertext, { - modelName: STREAM_MODEL_LABEL_FALLBACK, - keyEntryId: entry.id, - }) + if (analytics.credentialMode === 'byok') { + if (!byokKey) { + closeWithError(1008, 'byok_credentials_required') + return + } + keyPlaintext = byokKey + byokKey = null } - catch (err) { - log.withError(err).withFields({ keyEntryId: entry.id }).error('decrypt failed for streaming tts key') - closeWithError(1011, 'decrypt_failed') - return + else { + // Streaming sessions do not rotate keys mid-connection because that + // would break audio continuity. Retry policy belongs to the next client + // connection, never to an in-flight session. + const entry = upstreamConfig.keys[0] + try { + keyPlaintext = opts.envelopeCrypto.decryptKey(entry.ciphertext, { + modelName: STREAM_MODEL_LABEL_FALLBACK, + keyEntryId: entry.id, + }) + } + catch (err) { + log.withError(err).withFields({ keyEntryId: entry.id }).error('decrypt failed for streaming tts key') + closeWithError(1011, 'decrypt_failed') + return + } + span.setAttribute(AIRI_ATTR_GEN_AI_GATEWAY_KEY_ID, entry.id) } const upstreamURL = upstreamConfig.baseURL span.setAttribute(AIRI_ATTR_GEN_AI_GATEWAY_UPSTREAM_URL, upstreamURL) - span.setAttribute(AIRI_ATTR_GEN_AI_GATEWAY_KEY_ID, entry.id) let upstream: WebSocket try { @@ -196,6 +216,11 @@ export function createSessionState( }, }) } + catch (err) { + log.withError(err).withFields({ userId }).warn('failed to dial streaming tts upstream') + closeWithError(1011, 'upstream_dial_failed') + return + } finally { // Wipe plaintext immediately — the ws lib has already serialized the // header into its outgoing handshake buffer. @@ -242,7 +267,7 @@ export function createSessionState( metadata: { duration_ms: Date.now() - startedAt, trigger: analytics.trigger, - ...streamingVoiceMetadata(voiceLabel, analytics.voiceType), + ...streamingSessionMetadata(voiceLabel, analytics), }, }) try { @@ -270,6 +295,26 @@ export function createSessionState( ? Buffer.from(message.data) : Buffer.from(message.data as ArrayBufferLike) + if (!credentialFrameAccepted) { + if (isBinary || typeof payload !== 'string') { + closeWithError(1008, 'invalid_credentials_frame') + return + } + + const credentials = parseCredentialsFrame(payload) + if (!credentials) { + closeWithError(1008, 'invalid_credentials_frame') + return + } + + // This AIRI-private frame is deliberately consumed here and never + // placed in pendingClientFrames, so the upstream UnSpeech protocol can + // never observe or log the plaintext credential as an application frame. + byokKey = Buffer.from(credentials.apiKey, 'utf8') + credentialFrameAccepted = true + return + } + if (!startValidationStarted) { if (isBinary || typeof payload !== 'string') { closeWithError(1008, 'invalid_start_frame') @@ -382,10 +427,7 @@ export function createSessionState( // the client-text-frame estimate accumulated in handleClientMessage. const usageChars = readUsageChars(evt.payload) const billUnits = usageChars ?? totalInputChars - if (billUnits > 0) - void billSession(billUnits, 'session.finished') - else - finalize() + void completeSession(billUnits, 'session.finished') break } case 'error': { @@ -433,7 +475,7 @@ export function createSessionState( } const upstreamConfig = unspeech?.streaming - if (!unspeech?.restBaseURL || !upstreamConfig?.baseURL || upstreamConfig.keys.length === 0) { + if (!unspeech?.restBaseURL || !upstreamConfig?.baseURL || (analytics.credentialMode === 'official' && upstreamConfig.keys.length === 0)) { closeWithError(1008, 'streaming_tts_not_configured') return false } @@ -470,42 +512,44 @@ export function createSessionState( return true } - async function billSession(units: number, reason: string) { - if (billed) + async function completeSession(units: number, reason: string) { + if (completed) return - billed = true + completed = true span.setAttribute(GEN_AI_ATTR_REQUEST_MODEL, modelLabel) - let flux: Awaited> - try { - flux = await opts.fluxService.getFlux(userId) - } - catch (err) { - log.withError(err).withFields({ userId }).warn('flux read failed at session end') - finalize() - return - } - let fluxConsumed = 0 - try { - const result = await otelContext.with(trace.setSpan(otelContext.active(), span), () => - opts.ttsMeter.accumulate({ - userId, - units, - currentBalance: flux.flux, - requestId, - metadata: { model: modelLabel }, - })) - fluxConsumed = result.fluxDebited - span.setAttribute(AIRI_ATTR_BILLING_FLUX_CONSUMED, fluxConsumed) - } - catch (err) { - // Billing failure is surfaced but does not retroactively reject the - // already-delivered audio — the user got the audio, the meter retains - // the debt for the next request to settle (per FluxMeter rollback path). - log.withError(err).withFields({ userId, units, reason }).error('billing accumulate failed for streaming tts') - span.recordException(err as Error) - span.setStatus({ code: SpanStatusCode.ERROR, message: 'billing_failed' }) + if (analytics.credentialMode === 'official' && units > 0) { + let flux: Awaited> + try { + flux = await opts.fluxService.getFlux(userId) + } + catch (err) { + log.withError(err).withFields({ userId }).warn('flux read failed at session end') + finalize() + return + } + + try { + const result = await otelContext.with(trace.setSpan(otelContext.active(), span), () => + opts.ttsMeter.accumulate({ + userId, + units, + currentBalance: flux.flux, + requestId, + metadata: { model: modelLabel }, + })) + fluxConsumed = result.fluxDebited + span.setAttribute(AIRI_ATTR_BILLING_FLUX_CONSUMED, fluxConsumed) + } + catch (err) { + // Billing failure is surfaced but does not retroactively reject the + // already-delivered audio — the user got the audio, the meter retains + // the debt for the next request to settle (per FluxMeter rollback path). + log.withError(err).withFields({ userId, units, reason }).error('billing accumulate failed for streaming tts') + span.recordException(err as Error) + span.setStatus({ code: SpanStatusCode.ERROR, message: 'billing_failed' }) + } } const durationMs = Date.now() - startedAt @@ -534,7 +578,7 @@ export function createSessionState( duration_ms: durationMs, flux_consumed: fluxConsumed, trigger: analytics.trigger, - ...streamingVoiceMetadata(voiceLabel, analytics.voiceType), + ...streamingSessionMetadata(voiceLabel, analytics), }, }) @@ -545,6 +589,7 @@ export function createSessionState( if (closed) return closed = true + wipeByokKey() try { upstreamWs?.close() } @@ -572,7 +617,7 @@ export function createSessionState( close_code: code, duration_ms: Date.now() - startedAt, trigger: analytics.trigger, - ...streamingVoiceMetadata(voiceLabel, analytics.voiceType), + ...streamingSessionMetadata(voiceLabel, analytics), }, }) if (clientWs) { @@ -586,6 +631,7 @@ export function createSessionState( catch {} } closed = true + wipeByokKey() span.end() } @@ -608,7 +654,7 @@ export function createSessionState( close_code: code, duration_ms: Date.now() - startedAt, trigger: analytics.trigger, - ...streamingVoiceMetadata(voiceLabel, analytics.voiceType), + ...streamingSessionMetadata(voiceLabel, analytics), }, }) if (clientWs) { @@ -622,9 +668,15 @@ export function createSessionState( catch {} } closed = true + wipeByokKey() span.end() } + function wipeByokKey() { + byokKey?.fill(0) + byokKey = null + } + return { attachClient, dialUpstream, @@ -638,6 +690,8 @@ function normalizeAnalytics(input: AudioSpeechSessionAnalytics): Required, +): Record { + return { + ...streamingVoiceMetadata(voiceId, analytics.voiceType), + credential_mode: analytics.credentialMode, + provider_id: analytics.providerId, + } +} + +function normalizeProviderId(providerId: AudioSpeechSessionAnalytics['providerId']): StreamingTtsProviderId { + switch (providerId) { + case 'official-provider-speech-streaming': + case 'volcengine-streaming': + return providerId + default: + return 'unknown' + } +} + function isPaymentRequiredError(err: unknown): boolean { if (err instanceof ApiError) return err.statusCode === 402 @@ -697,6 +772,29 @@ interface StreamingTtsStartFrame { voice: string } +interface StreamingTtsCredentialsFrame { + event: 'credentials' + provider: 'volcengine' + apiKey: string +} + +function parseCredentialsFrame(rawText: string): StreamingTtsCredentialsFrame | null { + try { + const parsed = JSON.parse(rawText) as Record + if (parsed.event !== 'credentials' || parsed.provider !== 'volcengine') + return null + if (typeof parsed.api_key !== 'string') + return null + const apiKey = parsed.api_key.trim() + if (apiKey.length === 0 || apiKey.length > 4096) + return null + return { event: 'credentials', provider: 'volcengine', apiKey } + } + catch { + return null + } +} + function parseStartFrame(rawText: string): StreamingTtsStartFrame | null { try { const parsed = JSON.parse(rawText) as Record diff --git a/server/apps/api/src/routes/audio-speech-ws/types.ts b/server/apps/api/src/routes/audio-speech-ws/types.ts index aea679f25..437fab438 100644 --- a/server/apps/api/src/routes/audio-speech-ws/types.ts +++ b/server/apps/api/src/routes/audio-speech-ws/types.ts @@ -9,13 +9,13 @@ import type { EnvelopeCrypto } from '../../utils/envelope-crypto' * Dependencies required by the streaming speech websocket proxy. */ export interface AudioSpeechWsHandlersOptions { - /** Reads upstream websocket URL and encrypted API keys. */ + /** Reads the UnSpeech websocket URL and operator credentials for official sessions. */ configKV: ConfigKVService /** Decrypts the selected upstream API key before the websocket handshake. */ envelopeCrypto: EnvelopeCrypto - /** Reads the user's current Flux balance for pre-flight and final billing. */ + /** Reads the user's current Flux balance for official-session billing. */ fluxService: FluxService - /** Applies pre-flight affordability checks and final streaming TTS billing. */ + /** Applies pre-flight checks and final billing to official sessions only. */ ttsMeter: FluxMeter /** Persists request accounting after a stream finishes. */ requestLogService: RequestLogService