From 5f1c52ec528841831e83faf3302634e54e06251e Mon Sep 17 00:00:00 2001 From: Neko Ayaka Date: Fri, 7 Aug 2026 05:08:38 +0800 Subject: [PATCH] refactor(providers): now simplified --- .../providers/providers/aliyun-nls/index.ts | 107 + .../providers/browser-web-speech-api/index.ts | 77 + .../providers/providers/comet-api/index.ts | 49 +- .../providers/providers/elevenlabs/index.ts | 133 ++ .../google-gemini-audio-speech/index.ts | 172 ++ .../providers/index-tts-vllm/index.ts | 92 + .../src/libs/providers/providers/index.ts | 14 + .../providers/providers/kokoro-local/index.ts | 184 ++ .../providers/providers/local-audio/index.ts | 160 ++ .../providers/providers/mimo-audio/index.ts | 262 ++ .../providers/minimax-speech/index.ts | 152 ++ .../providers/providers/openai-audio/index.ts | 266 +++ .../openrouter-audio-speech/index.ts | 200 ++ .../providers/player2-speech/index.ts | 107 + .../providers/provider-definitions.test.ts | 122 + .../providers/providers/speech-noop/index.ts | 32 + .../providers/providers/unspeech/index.ts | 239 ++ packages/stage-ui/src/stores/providers.ts | 2107 +---------------- 18 files changed, 2373 insertions(+), 2102 deletions(-) create mode 100644 packages/stage-ui/src/libs/providers/providers/aliyun-nls/index.ts create mode 100644 packages/stage-ui/src/libs/providers/providers/browser-web-speech-api/index.ts create mode 100644 packages/stage-ui/src/libs/providers/providers/elevenlabs/index.ts create mode 100644 packages/stage-ui/src/libs/providers/providers/google-gemini-audio-speech/index.ts create mode 100644 packages/stage-ui/src/libs/providers/providers/index-tts-vllm/index.ts create mode 100644 packages/stage-ui/src/libs/providers/providers/kokoro-local/index.ts create mode 100644 packages/stage-ui/src/libs/providers/providers/local-audio/index.ts create mode 100644 packages/stage-ui/src/libs/providers/providers/mimo-audio/index.ts create mode 100644 packages/stage-ui/src/libs/providers/providers/minimax-speech/index.ts create mode 100644 packages/stage-ui/src/libs/providers/providers/openai-audio/index.ts create mode 100644 packages/stage-ui/src/libs/providers/providers/openrouter-audio-speech/index.ts create mode 100644 packages/stage-ui/src/libs/providers/providers/player2-speech/index.ts create mode 100644 packages/stage-ui/src/libs/providers/providers/provider-definitions.test.ts create mode 100644 packages/stage-ui/src/libs/providers/providers/speech-noop/index.ts create mode 100644 packages/stage-ui/src/libs/providers/providers/unspeech/index.ts diff --git a/packages/stage-ui/src/libs/providers/providers/aliyun-nls/index.ts b/packages/stage-ui/src/libs/providers/providers/aliyun-nls/index.ts new file mode 100644 index 000000000..687269f83 --- /dev/null +++ b/packages/stage-ui/src/libs/providers/providers/aliyun-nls/index.ts @@ -0,0 +1,107 @@ +import type { TranscriptionProviderWithExtraOptions } from '@xsai-ext/providers/utils' + +import type { AliyunRealtimeSpeechExtraOptions } from '../../../../stores/providers/aliyun/stream-transcription' + +import { z } from 'zod' + +import { createAliyunNLSProvider } from '../../../../stores/providers/aliyun/stream-transcription' +import { defineProvider } from '../registry' + +const aliyunNlsRegions = [ + 'cn-shanghai', + 'cn-shanghai-internal', + 'cn-beijing', + 'cn-beijing-internal', + 'cn-shenzhen', + 'cn-shenzhen-internal', +] as const + +const aliyunNlsConfigSchema = z.object({ + accessKeyId: z.string(), + accessKeySecret: z.string(), + appKey: z.string(), + region: z.enum(aliyunNlsRegions).default('cn-shanghai'), +}) + +type AliyunNlsConfig = z.input + +export const providerAliyunNlsTranscription = defineProvider({ + id: 'aliyun-nls-transcription', + name: 'Aliyun NLS', + nameLocalize: ({ t }) => t('settings.pages.providers.provider.aliyun-nls.title'), + description: 'nls-console.aliyun.com', + descriptionLocalize: ({ t }) => t('settings.pages.providers.provider.aliyun-nls.description'), + tasks: ['speech-to-text', 'automatic-speech-recognition', 'asr', 'stt', 'streaming-transcription'], + icon: 'i-lobe-icons:alibabacloud', + capabilities: { + transcription: { + protocol: 'websocket', + generateOutput: false, + streamOutput: true, + streamInput: true, + }, + }, + createProviderConfig: () => aliyunNlsConfigSchema, + createProvider(config) { + const accessKeyId = config.accessKeyId.trim() + const accessKeySecret = config.accessKeySecret.trim() + const appKey = config.appKey.trim() + if (!accessKeyId || !accessKeySecret || !appKey) + throw new Error('Aliyun NLS credentials are incomplete.') + + const provider = createAliyunNLSProvider(accessKeyId, accessKeySecret, appKey, { + region: config.region ?? 'cn-shanghai', + }) + + return { + transcription: (model: string, extraOptions?: AliyunRealtimeSpeechExtraOptions) => provider.speech(model, { + ...extraOptions, + sessionOptions: { + format: 'pcm', + sample_rate: 16000, + enable_punctuation_prediction: true, + enable_intermediate_result: true, + enable_words: true, + ...extraOptions?.sessionOptions, + }, + }), + } as TranscriptionProviderWithExtraOptions + }, + validationRequiredWhen: config => Boolean(config.accessKeyId?.trim() && config.accessKeySecret?.trim() && config.appKey?.trim()), + validators: { + validateConfig: [ + ({ t }) => ({ + id: 'aliyun-nls:check-config', + name: t('settings.pages.providers.catalog.edit.validators.openai-compatible.check-config.title'), + validator: async (config) => { + const errors: Array<{ error: unknown }> = [] + if (!config.accessKeyId?.trim()) + errors.push({ error: new Error('Access Key ID is required.') }) + if (!config.accessKeySecret?.trim()) + errors.push({ error: new Error('Access Key Secret is required.') }) + if (!config.appKey?.trim()) + errors.push({ error: new Error('App Key is required.') }) + if (config.region && !aliyunNlsRegions.includes(config.region)) + errors.push({ error: new Error('Region is invalid.') }) + + return { + errors, + reason: errors.map(item => (item.error as Error).message).join(', '), + reasonKey: '', + valid: errors.length === 0, + } + }, + }), + ], + }, + extraMethods: { + listModels: async () => [{ + id: 'aliyun-nls-v1', + name: 'Aliyun NLS Realtime', + provider: 'aliyun-nls-transcription', + description: 'Realtime streaming transcription using Aliyun NLS.', + contextLength: 0, + deprecated: false, + }], + }, +}) diff --git a/packages/stage-ui/src/libs/providers/providers/browser-web-speech-api/index.ts b/packages/stage-ui/src/libs/providers/providers/browser-web-speech-api/index.ts new file mode 100644 index 000000000..d87cb839e --- /dev/null +++ b/packages/stage-ui/src/libs/providers/providers/browser-web-speech-api/index.ts @@ -0,0 +1,77 @@ +import { isStageTamagotchi } from '@proj-airi/stage-shared' +import { z } from 'zod' + +import { createWebSpeechAPIProvider } from '../../../../stores/providers/web-speech-api' +import { defineProvider } from '../registry' + +const webSpeechApiConfigSchema = z.object({ + language: z.string().default('en-US'), + continuous: z.boolean().default(true), + interimResults: z.boolean().default(true), + maxAlternatives: z.number().int().positive().default(1), +}) + +function isWebSpeechApiAvailable() { + if (typeof window === 'undefined') + return false + + return 'webkitSpeechRecognition' in window || 'SpeechRecognition' in window +} + +export const providerBrowserWebSpeechApi = defineProvider({ + id: 'browser-web-speech-api', + name: 'Web Speech API (Browser)', + nameLocalize: ({ t }) => t('settings.pages.providers.provider.browser-web-speech-api.title'), + description: 'Browser-native speech recognition. No API keys.', + descriptionLocalize: ({ t }) => t('settings.pages.providers.provider.browser-web-speech-api.description'), + tasks: ['speech-to-text', 'automatic-speech-recognition', 'asr', 'stt', 'streaming-transcription'], + icon: 'i-solar:microphone-bold-duotone', + requiresCredentials: false, + capabilities: { + transcription: { + protocol: 'http', + generateOutput: false, + streamOutput: true, + streamInput: true, + }, + }, + + // Electron uses Chromium, but it does not include the Google API keys that + // the Web Speech API needs. The provider only works in browser contexts. + isAvailableBy: () => !isStageTamagotchi() && isWebSpeechApiAvailable(), + createProviderConfig: () => webSpeechApiConfigSchema, + createProvider: createWebSpeechAPIProvider, + validationRequiredWhen: () => false, + + extraMethods: { + listModels: async () => [ + { + id: 'web-speech-api', + name: 'Web Speech API', + provider: 'browser-web-speech-api', + description: 'Browser-native speech recognition (no API keys required)', + contextLength: 0, + deprecated: false, + }, + ], + }, + validators: { + validateConfig: [ + ({ t }) => ({ + id: 'browser-web-speech-api:check-availability', + name: t('settings.pages.providers.catalog.edit.validators.openai-compatible.check-config.title'), + validator: async () => { + const valid = isWebSpeechApiAvailable() + return { + errors: valid + ? [] + : [{ error: new Error('Web Speech API is not available. It requires a browser context with SpeechRecognition support (Chrome, Edge, Safari).') }], + reason: valid ? '' : 'Web Speech API is not available in this environment.', + reasonKey: '', + valid, + } + }, + }), + ], + }, +}) diff --git a/packages/stage-ui/src/libs/providers/providers/comet-api/index.ts b/packages/stage-ui/src/libs/providers/providers/comet-api/index.ts index 250fecef9..f1c09e765 100644 --- a/packages/stage-ui/src/libs/providers/providers/comet-api/index.ts +++ b/packages/stage-ui/src/libs/providers/providers/comet-api/index.ts @@ -1,4 +1,4 @@ -import { createChatProvider, createModelProvider, merge } from '@xsai-ext/providers/utils' +import { createChatProvider, createModelProvider, createSpeechProvider, createTranscriptionProvider, merge } from '@xsai-ext/providers/utils' import { z } from 'zod' import { ProviderValidationCheck } from '../../types' @@ -55,3 +55,50 @@ export const providerCometAPI = defineProvider({ }), }, }) + +export const providerCometAPISpeech = defineProvider({ + id: 'comet-api-speech', + name: 'CometAPI Speech', + nameLocalize: ({ t }) => t('settings.pages.providers.provider.comet-api.title'), + description: 'cometapi.com', + descriptionLocalize: ({ t }) => t('settings.pages.providers.provider.comet-api.description'), + tasks: ['text-to-speech'], + icon: 'i-lobe-icons:cometapi', + createProviderConfig: providerCometAPI.createProviderConfig, + createProvider(config) { + return merge( + createModelProvider({ apiKey: config.apiKey, baseURL: config.baseUrl! }), + createSpeechProvider({ apiKey: config.apiKey, baseURL: config.baseUrl! }), + ) + }, + validationRequiredWhen: config => Boolean(config.apiKey?.trim()), + validators: createOpenAICompatibleValidators({ checks: [ProviderValidationCheck.ModelList] }), +}) + +export const providerCometAPITranscription = defineProvider({ + id: 'comet-api-transcription', + name: 'CometAPI Transcription', + nameLocalize: ({ t }) => t('settings.pages.providers.provider.comet-api.title'), + description: 'cometapi.com', + descriptionLocalize: ({ t }) => t('settings.pages.providers.provider.comet-api.description'), + tasks: ['speech-to-text', 'automatic-speech-recognition', 'asr', 'stt'], + icon: 'i-lobe-icons:cometapi', + capabilities: { + transcription: { protocol: 'http', generateOutput: true, streamOutput: false, streamInput: false }, + }, + createProviderConfig: providerCometAPI.createProviderConfig, + createProvider(config) { + const provider = merge( + createModelProvider({ apiKey: config.apiKey, baseURL: config.baseUrl! }), + createTranscriptionProvider({ apiKey: config.apiKey, baseURL: config.baseUrl! }), + ) + const transcription = provider.transcription.bind(provider) + provider.transcription = (model: string, extraOptions?: Record) => ({ + ...transcription(model), + ...extraOptions, + }) + return provider + }, + validationRequiredWhen: config => Boolean(config.apiKey?.trim()), + validators: createOpenAICompatibleValidators({ checks: [ProviderValidationCheck.ModelList] }), +}) diff --git a/packages/stage-ui/src/libs/providers/providers/elevenlabs/index.ts b/packages/stage-ui/src/libs/providers/providers/elevenlabs/index.ts new file mode 100644 index 000000000..928bc1c22 --- /dev/null +++ b/packages/stage-ui/src/libs/providers/providers/elevenlabs/index.ts @@ -0,0 +1,133 @@ +import type { SpeechProviderWithExtraOptions } from '@xsai-ext/providers/utils' +import type { ListVoicesOptions, UnElevenLabsOptions, VoiceProviderWithExtraOptions } from 'unspeech' +import type { ComposerTranslation } from 'vue-i18n' + +import { createUnElevenLabs, listVoices } from 'unspeech' +import { z } from 'zod' + +import { models as elevenLabsModels } from '../../../../stores/providers/elevenlabs/list-models' +import { defineProvider } from '../registry' + +const elevenLabsConfigSchema = z.object({ + apiKey: z.string(), + baseUrl: z.string().default('https://unspeech.hyp3r.link/v1/'), + voiceSettings: z.object({ + similarityBoost: z.number().default(0.75), + stability: z.number().default(0.5), + }).default({ similarityBoost: 0.75, stability: 0.5 }), +}) + +type ElevenLabsConfig = z.input + +function toListVoicesOptions(provider: VoiceProviderWithExtraOptions): ListVoicesOptions { + const { fetch: _fetch, ...voiceOptions } = provider.voice() + return voiceOptions +} + +function createElevenLabsValidators() { + return { + validateConfig: [ + ({ t }: { t: ComposerTranslation }) => ({ + id: 'elevenlabs:check-config', + name: t('settings.pages.providers.catalog.edit.validators.openai-compatible.check-config.title'), + validator: async (config: ElevenLabsConfig) => { + const errors: Array<{ error: unknown }> = [] + const apiKey = config.apiKey?.trim() ?? '' + const baseUrl = config.baseUrl?.trim() ?? '' + + if (!apiKey) + errors.push({ error: new Error('API key is required.') }) + + if (!baseUrl) { + errors.push({ error: new Error('Base URL is required.') }) + } + else { + try { + if (!new URL(baseUrl).host) + errors.push({ error: new Error('Base URL is not absolute. Try to include a scheme (http:// or https://).') }) + else if (!baseUrl.endsWith('/')) + errors.push({ error: new Error('Base URL must end with a trailing slash (/).') }) + } + catch { + errors.push({ error: new Error('Base URL is not absolute. Try to include a scheme (http:// or https://).') }) + } + } + + return { + errors, + reason: errors.map(item => (item.error as Error).message).join(', '), + reasonKey: '', + valid: errors.length === 0, + } + }, + }), + ], + } +} + +export const providerElevenLabs = defineProvider({ + id: 'elevenlabs', + name: 'ElevenLabs', + nameLocalize: ({ t }) => t('settings.pages.providers.provider.elevenlabs.title'), + description: 'elevenlabs.io', + descriptionLocalize: ({ t }) => t('settings.pages.providers.provider.elevenlabs.description'), + tasks: ['text-to-speech'], + icon: 'i-simple-icons:elevenlabs', + + createProviderConfig: ({ t }) => elevenLabsConfigSchema.extend({ + apiKey: elevenLabsConfigSchema.shape.apiKey.meta({ + labelLocalized: t('settings.pages.providers.catalog.edit.config.common.fields.field.api-key.label'), + descriptionLocalized: t('settings.pages.providers.catalog.edit.config.common.fields.field.api-key.description'), + placeholderLocalized: t('settings.pages.providers.catalog.edit.config.common.fields.field.api-key.placeholder'), + type: 'password', + }), + baseUrl: elevenLabsConfigSchema.shape.baseUrl.meta({ + labelLocalized: t('settings.pages.providers.catalog.edit.config.common.fields.field.base-url.label'), + descriptionLocalized: t('settings.pages.providers.catalog.edit.config.common.fields.field.base-url.description'), + placeholderLocalized: t('settings.pages.providers.catalog.edit.config.common.fields.field.base-url.placeholder'), + }), + }), + createProvider(config) { + return createUnElevenLabs(config.apiKey.trim(), config.baseUrl?.trim() ?? 'https://unspeech.hyp3r.link/v1/') as SpeechProviderWithExtraOptions + }, + + validationRequiredWhen: config => Boolean(config.apiKey?.trim() && config.baseUrl?.trim()), + validators: createElevenLabsValidators(), + extraMethods: { + listModels: async () => elevenLabsModels.map(model => ({ + id: model.model_id, + name: model.name, + provider: 'elevenlabs', + description: model.description, + contextLength: 0, + deprecated: false, + })), + listVoices: async (config) => { + const provider = createUnElevenLabs(config.apiKey.trim(), config.baseUrl?.trim() ?? 'https://unspeech.hyp3r.link/v1/') as VoiceProviderWithExtraOptions + const voices = await listVoices(toListVoicesOptions(provider)) + if (!Array.isArray(voices)) + return [] + + // Keep the default ElevenLabs voices together at the end of the list. + const ariaIndex = voices.findIndex(voice => voice.name.includes('Aria')) + const billIndex = voices.findIndex(voice => voice.name.includes('Bill')) + const startIndex = ariaIndex !== -1 ? ariaIndex : 0 + const endIndex = billIndex !== -1 ? billIndex : voices.length - 1 + const lowerIndex = Math.min(startIndex, endIndex) + const higherIndex = Math.max(startIndex, endIndex) + const rearrangedVoices = [ + ...voices.slice(0, lowerIndex), + ...voices.slice(higherIndex + 1), + ...voices.slice(lowerIndex, higherIndex + 1), + ] + + return rearrangedVoices.map(voice => ({ + id: voice.id, + name: voice.name, + provider: 'elevenlabs', + previewURL: voice.preview_audio_url, + languages: voice.languages, + })) + }, + }, +}) diff --git a/packages/stage-ui/src/libs/providers/providers/google-gemini-audio-speech/index.ts b/packages/stage-ui/src/libs/providers/providers/google-gemini-audio-speech/index.ts new file mode 100644 index 000000000..c2bb13260 --- /dev/null +++ b/packages/stage-ui/src/libs/providers/providers/google-gemini-audio-speech/index.ts @@ -0,0 +1,172 @@ +import { toWavFromPCM16 } from '@proj-airi/audio/encoding' +import { z } from 'zod' + +import { defineProvider } from '../registry' + +const DEFAULT_BASE_URL = 'https://generativelanguage.googleapis.com/v1beta/' +const DEFAULT_MODEL = 'gemini-2.5-flash-preview-tts' + +const googleGeminiSpeechConfigSchema = z.object({ + apiKey: z.string(), + baseUrl: z.string().default(DEFAULT_BASE_URL), +}) + +type GoogleGeminiSpeechConfig = z.input + +const googleGeminiTtsModels = [ + 'gemini-2.5-flash-preview-tts', + 'gemini-2.5-pro-preview-tts', + 'gemini-3.1-flash-tts-preview', +] as const + +const googleGeminiTtsVoices: Array<[string, string]> = [ + ['Zephyr', 'Bright'], + ['Puck', 'Upbeat'], + ['Charon', 'Informative'], + ['Kore', 'Firm'], + ['Fenrir', 'Excitable'], + ['Leda', 'Youthful'], + ['Orus', 'Firm'], + ['Aoede', 'Breezy'], + ['Callirrhoe', 'Easy-going'], + ['Autonoe', 'Bright'], + ['Enceladus', 'Breathy'], + ['Iapetus', 'Clear'], + ['Umbriel', 'Easy-going'], + ['Algieba', 'Smooth'], + ['Despina', 'Smooth'], + ['Erinome', 'Clear'], + ['Algenib', 'Gravelly'], + ['Rasalgethi', 'Informative'], + ['Laomedeia', 'Upbeat'], + ['Achernar', 'Soft'], + ['Alnilam', 'Firm'], + ['Schedar', 'Even'], + ['Gacrux', 'Mature'], + ['Pulcherrima', 'Forward'], + ['Achird', 'Friendly'], + ['Zubenelgenubi', 'Casual'], + ['Vindemiatrix', 'Gentle'], + ['Sadachbia', 'Lively'], + ['Sadaltager', 'Knowledgeable'], + ['Sulafat', 'Warm'], +] + +function normalizeBaseUrl(baseUrl: string | undefined) { + const value = baseUrl?.trim() || DEFAULT_BASE_URL + return value.endsWith('/') ? value : `${value}/` +} + +function decodeBase64(base64: string) { + const binary = atob(base64) + const bytes = new Uint8Array(binary.length) + for (let index = 0; index < binary.length; index++) + bytes[index] = binary.charCodeAt(index) + return bytes +} + +function createAudioFetch(apiKey: string, baseUrl: string) { + return async (_input: RequestInfo | URL, init?: RequestInit) => { + if (!init?.body || typeof init.body !== 'string') + throw new Error('Invalid request body') + + const body = JSON.parse(init.body) as { + input?: string + model?: string + voice?: string + temperature?: number + } + if (!body.input) + throw new Error('Missing input text for Gemini TTS') + if (!body.model) + throw new Error('Missing model for Gemini TTS') + + const response = await globalThis.fetch(new URL(`models/${body.model}:generateContent`, baseUrl), { + method: 'POST', + headers: { 'x-goog-api-key': apiKey, 'Content-Type': 'application/json' }, + body: JSON.stringify({ + contents: [{ parts: [{ text: body.input }] }], + generationConfig: { + responseModalities: ['AUDIO'], + speechConfig: { + voiceConfig: { prebuiltVoiceConfig: { voiceName: body.voice || 'Kore' } }, + }, + ...(body.temperature !== undefined ? { temperature: body.temperature } : {}), + }, + }), + }) + if (!response.ok) + throw new Error(`Gemini TTS request failed: ${response.status} ${await response.text().catch(() => '')}`) + + const data = await response.json() as { + candidates?: Array<{ content?: { parts?: Array<{ inlineData?: { data?: string } }> } }> + } + const audio = data.candidates?.[0]?.content?.parts?.find(part => part.inlineData)?.inlineData?.data + if (!audio) + throw new Error('Gemini TTS response missing audio data') + + return new Response(toWavFromPCM16(decodeBase64(audio), 24000), { + status: 200, + headers: { 'Content-Type': 'audio/wav' }, + }) + } +} + +export const providerGoogleGeminiAudioSpeech = defineProvider({ + id: 'google-gemini-audio-speech', + name: 'Google Gemini', + nameLocalize: ({ t }) => t('settings.pages.providers.provider.google-gemini-audio-speech.title'), + description: 'aistudio.google.com', + descriptionLocalize: ({ t }) => t('settings.pages.providers.provider.google-gemini-audio-speech.description'), + tasks: ['text-to-speech', 'tts'], + icon: 'i-lobe-icons:gemini', + iconColor: 'i-lobe-icons:gemini-color', + createProviderConfig: () => googleGeminiSpeechConfigSchema, + createProvider(config) { + const apiKey = config.apiKey.trim() + const baseUrl = normalizeBaseUrl(config.baseUrl) + return { + speech: (model?: string, options?: Record) => ({ + baseURL: baseUrl, + fetch: createAudioFetch(apiKey, baseUrl), + ...options, + model: model || (typeof options?.model === 'string' ? options.model : DEFAULT_MODEL), + }), + } + }, + validationRequiredWhen: config => Boolean(config.apiKey?.trim()), + validators: { + validateConfig: [ + ({ t }) => ({ + id: 'google-gemini-audio-speech:check-config', + name: t('settings.pages.providers.catalog.edit.validators.openai-compatible.check-config.title'), + validator: async (config) => { + const valid = Boolean(config.apiKey?.trim()) + return { + errors: valid ? [] : [{ error: new Error('API Key is required.') }], + reason: valid ? '' : 'API Key is required.', + reasonKey: '', + valid, + } + }, + }), + ], + }, + extraMethods: { + listModels: async () => googleGeminiTtsModels.map(id => ({ + id, + name: id.split('-').map(word => `${word[0].toUpperCase()}${word.slice(1)}`).join(' '), + provider: 'google-gemini-audio-speech', + description: 'Gemini API text-to-speech model', + capabilities: ['text-to-speech'], + })), + listVoices: async () => googleGeminiTtsVoices.map(([id, style]) => ({ + id, + name: id, + provider: 'google-gemini-audio-speech', + description: style, + languages: [{ code: 'auto', title: 'Auto' }], + compatibleModels: [...googleGeminiTtsModels], + })), + }, +}) diff --git a/packages/stage-ui/src/libs/providers/providers/index-tts-vllm/index.ts b/packages/stage-ui/src/libs/providers/providers/index-tts-vllm/index.ts new file mode 100644 index 000000000..2cdccc203 --- /dev/null +++ b/packages/stage-ui/src/libs/providers/providers/index-tts-vllm/index.ts @@ -0,0 +1,92 @@ +import { errorMessageFrom } from '@moeru/std' +import { z } from 'zod' + +import { defineProvider } from '../registry' + +const indexTtsConfigSchema = z.object({ + baseUrl: z.string().default('http://localhost:11996/tts/'), + model: z.string().default('IndexTTS-1.5'), +}) + +type IndexTtsConfig = z.input + +function voicesUrl(config: IndexTtsConfig) { + return `${config.baseUrl ?? 'http://localhost:11996/tts/'}audio/voices` +} + +export const providerIndexTtsVllm = defineProvider({ + id: 'index-tts-vllm', + name: 'Index-TTS by Bilibili', + nameLocalize: ({ t }) => t('settings.pages.providers.provider.index-tts-vllm.title'), + description: 'index-tts.github.io', + descriptionLocalize: ({ t }) => t('settings.pages.providers.provider.index-tts-vllm.description'), + tasks: ['text-to-speech'], + iconColor: 'i-lobe-icons:bilibiliindex', + createProviderConfig: () => indexTtsConfigSchema, + createProvider(config) { + return { + speech: () => ({ + baseURL: config.baseUrl ?? 'http://localhost:11996/tts/', + model: config.model || 'IndexTTS-1.5', + }), + } + }, + validationRequiredWhen: config => Boolean(config.baseUrl?.trim()), + validators: { + validateConfig: [ + ({ t }) => ({ + id: 'index-tts-vllm:check-config', + name: t('settings.pages.providers.catalog.edit.validators.openai-compatible.check-config.title'), + validator: async (config) => { + const baseUrl = config.baseUrl?.trim() ?? '' + if (!baseUrl) { + const reason = 'Base URL is required. Default to http://localhost:11996/tts/ for Index-TTS.' + return { errors: [{ error: new Error(reason) }], reason, reasonKey: '', valid: false } + } + + const controller = new AbortController() + const timeout = setTimeout(() => controller.abort(), 5000) + try { + const response = await fetch(voicesUrl(config), { signal: controller.signal }) + if (!response.ok) { + const reason = `IndexTTS unreachable: HTTP ${response.status} ${response.statusText}` + return { errors: [{ error: new Error(reason) }], reason, reasonKey: '', valid: false } + } + } + catch (error) { + const reason = `IndexTTS connection failed: ${errorMessageFrom(error) ?? 'Unknown error'}` + return { errors: [{ error }], reason, reasonKey: '', valid: false } + } + finally { + clearTimeout(timeout) + } + + return { errors: [], reason: '', reasonKey: '', valid: true } + }, + }), + ], + }, + extraMethods: { + listModels: async () => [{ + id: 'IndexTTS-1.5', + name: 'IndexTTS-1.5', + provider: 'index-tts-vllm', + description: 'Default model for Index-TTS vLLM deployment', + contextLength: 0, + deprecated: false, + }], + listVoices: async (config) => { + const response = await fetch(voicesUrl(config)) + if (!response.ok) + throw new Error(`Failed to fetch voices: ${response.statusText}`) + + const voices = await response.json() as Record + return Object.keys(voices).map(voice => ({ + id: voice, + name: voice, + provider: 'index-tts-vllm', + languages: [{ code: 'cn', title: 'Chinese' }, { code: 'en', title: 'English' }], + })) + }, + }, +}) diff --git a/packages/stage-ui/src/libs/providers/providers/index.ts b/packages/stage-ui/src/libs/providers/providers/index.ts index bcebf2b79..a30a8728e 100644 --- a/packages/stage-ui/src/libs/providers/providers/index.ts +++ b/packages/stage-ui/src/libs/providers/providers/index.ts @@ -1,21 +1,30 @@ import './amazon-bedrock' import './openai' +import './openai-audio' import './aihubmix' +import './aliyun-nls' import './lm-studio' +import './local-audio' +import './index-tts-vllm' +import './kokoro-local' import './azure-openai' import './openai-compatible' import './atlascloud' import './volcengine-coding-plan' import './byteplus' import './byteplus-coding-plan' +import './browser-web-speech-api' import './n1n' import './openpaths' import './openrouter-ai' +import './openrouter-audio-speech' import './nvidia' import './groq' import './anthropic' import './google-generative-ai' +import './google-gemini-audio-speech' import './deepseek' +import './elevenlabs' import './302-ai' import './cerebras-ai' import './together-ai' @@ -26,15 +35,20 @@ import './fireworks-ai' import './featherless-ai' import './comet-api' import './perplexity-ai' +import './player2-speech' import './minimax' +import './minimax-speech' import './mistral-ai' import './moonshot-ai' import './modelscope' import './ollama' import './mimo' +import './mimo-audio' import './cloudflare-workers-ai' import './azure-ai-foundry' import './official' +import './speech-noop' +import './unspeech' export { getDefaultStreamingModel, diff --git a/packages/stage-ui/src/libs/providers/providers/kokoro-local/index.ts b/packages/stage-ui/src/libs/providers/providers/kokoro-local/index.ts new file mode 100644 index 000000000..951e85e4f --- /dev/null +++ b/packages/stage-ui/src/libs/providers/providers/kokoro-local/index.ts @@ -0,0 +1,184 @@ +import type { ProgressInfo } from '@xsai-transformers/shared/types' + +import type { VoiceKey } from '../../../../workers/kokoro/types' + +import { getCachedWebGPUCapabilities } from '@proj-airi/stage-shared/webgpu' +import { z } from 'zod' + +import { getDefaultKokoroModel, KOKORO_MODELS, kokoroModelsToModelInfo } from '../../../../workers/kokoro/constants' +import { getKokoroAdapter } from '../../../inference/adapters/kokoro' +import { defineProvider } from '../registry' + +interface KokoroVoice { + language: string + name: string + gender: string +} + +const languageByCode: Record = { + 'en-us': { code: 'en-US', title: 'English (US)' }, + 'en-gb': { code: 'en-GB', title: 'English (UK)' }, + 'ja': { code: 'ja', title: 'Japanese' }, + 'zh-cn': { code: 'zh-CN', title: 'Chinese (Mandarin)' }, + 'es': { code: 'es', title: 'Spanish' }, + 'fr': { code: 'fr', title: 'French' }, + 'hi': { code: 'hi', title: 'Hindi' }, + 'it': { code: 'it', title: 'Italian' }, + 'pt-br': { code: 'pt-BR', title: 'Portuguese (Brazil)' }, +} + +function getWebGpuState() { + const capabilities = getCachedWebGPUCapabilities() + return { + supported: capabilities?.supported ?? (typeof navigator !== 'undefined' && Boolean(navigator.gpu)), + fp16Supported: capabilities?.fp16Supported ?? false, + } +} + +function getModel(modelId: string) { + return KOKORO_MODELS.find(model => model.id === modelId) +} + +function assertModelSupported(modelId: string) { + const model = getModel(modelId) + if (!model) + throw new Error(`Invalid model: ${modelId}. Must be one of: ${KOKORO_MODELS.map(item => item.id).join(', ')}`) + + if (model.platform === 'webgpu' && !getWebGpuState().supported) + throw new Error('WebGPU is required for this model but is not available in your browser') + + return model +} + +function progressInfo(progress: { file?: string, percent: number, loaded?: number, total?: number }): ProgressInfo { + return { + name: progress.file ?? '', + file: progress.file ?? '', + progress: progress.percent >= 0 ? progress.percent : 0, + status: 'progress', + loaded: progress.loaded ?? 0, + total: progress.total ?? 0, + } +} + +let lastLoadedModelId: string | null = null + +export const providerKokoroLocal = defineProvider({ + id: 'kokoro-local', + name: 'Kokoro TTS', + nameLocalize: ({ t }) => t('settings.pages.providers.provider.kokoro-local.title'), + description: 'Local text-to-speech using Kokoro-82M.', + descriptionLocalize: ({ t }) => t('settings.pages.providers.provider.kokoro-local.description'), + tasks: ['text-to-speech'], + icon: 'i-lobe-icons:speaker', + requiresCredentials: false, + createProviderConfig: () => { + const capabilities = getWebGpuState() + return z.object({ + model: z.string().default(getDefaultKokoroModel(capabilities.supported, capabilities.fp16Supported)), + voiceId: z.string().default(''), + }) + }, + createProvider() { + const adapterPromise = getKokoroAdapter() + return { + speech: () => ({ + baseURL: 'http://kokoro-local/v1/', + model: 'kokoro-82m', + fetch: async (_input: RequestInfo | URL, init?: RequestInit) => { + if (!init?.body || typeof init.body !== 'string') + throw new Error('Invalid request body') + + const body = JSON.parse(init.body) as { input?: string, voice?: string } + if (!body.voice) + throw new Error('Voice parameter is required') + + try { + const adapter = await adapterPromise + if (!(body.voice in adapter.getVoices())) + throw new Error(`Unknown Kokoro voice: ${body.voice}`) + const buffer = await adapter.generate(body.input ?? '', body.voice as VoiceKey) + return new Response(buffer, { + status: 200, + headers: { 'Content-Type': 'audio/wav' }, + }) + } + catch (error) { + console.error('Kokoro TTS generation failed:', error) + throw error + } + }, + }), + } + }, + validationRequiredWhen: () => false, + validators: { + validateConfig: [ + ({ t }) => ({ + id: 'kokoro-local:check-config', + name: t('settings.pages.providers.catalog.edit.validators.openai-compatible.check-config.title'), + validator: async (config) => { + if (!config.model) { + return { + errors: [{ error: new Error('No model selected') }], + reason: 'Please select a model from the dropdown menu', + reasonKey: '', + valid: false, + } + } + if (!getModel(config.model)) { + const reason = `Invalid model. Must be one of: ${KOKORO_MODELS.map(model => model.id).join(', ')}` + return { errors: [{ error: new Error(`Invalid model: ${config.model}`) }], reason, reasonKey: '', valid: false } + } + return { errors: [], reason: '', reasonKey: '', valid: true } + }, + }), + ], + }, + extraMethods: { + listModels: async () => { + const capabilities = getWebGpuState() + return kokoroModelsToModelInfo(capabilities.supported, undefined, capabilities.fp16Supported) + }, + loadModel: async (config, _provider, hooks) => { + const model = assertModelSupported(config.model) + try { + const adapter = await getKokoroAdapter() + await adapter.loadModel(model.quantization, model.platform, { + onProgress: hooks?.onProgress ? progress => hooks.onProgress?.(progressInfo(progress)) : undefined, + }) + } + catch (error) { + console.error('Failed to load Kokoro model:', error) + throw error + } + }, + listVoices: async (config) => { + try { + const adapter = await getKokoroAdapter() + if (adapter.state !== 'ready' || config.model !== lastLoadedModelId) { + const model = assertModelSupported(config.model) + await adapter.loadModel(model.quantization, model.platform) + lastLoadedModelId = config.model + } + + return Object.entries(adapter.getVoices() as Record).map(([id, voice]) => { + const languageCode = voice.language.toLowerCase() + const language = languageByCode[languageCode] || { code: languageCode, title: voice.language } + return { + id, + name: `${voice.name} (${voice.gender}, ${language.title.split('(')[0].trim()})`, + provider: 'kokoro-local', + languages: [language], + gender: voice.gender.toLowerCase(), + } + }) + } + catch (error) { + console.error('Failed to fetch Kokoro voices:', error) + // Voice discovery can run before model loading. An empty list is safe. + return [] + } + }, + }, +}) diff --git a/packages/stage-ui/src/libs/providers/providers/local-audio/index.ts b/packages/stage-ui/src/libs/providers/providers/local-audio/index.ts new file mode 100644 index 000000000..42c425e34 --- /dev/null +++ b/packages/stage-ui/src/libs/providers/providers/local-audio/index.ts @@ -0,0 +1,160 @@ +import type { ComposerTranslation } from 'vue-i18n' + +import { isStageTamagotchi } from '@proj-airi/stage-shared' +import { isWebGPUSupported } from '@proj-airi/stage-shared/webgpu' +import { createOpenAI } from '@xsai-ext/providers/create' +import { z } from 'zod' + +import { defineProvider } from '../registry' + +const localAudioConfigSchema = z.object({ + apiKey: z.string().optional(), + baseUrl: z.string().optional().default(''), +}) + +type LocalAudioConfig = z.input + +function createLocalAudioConfigSchema(t: ComposerTranslation) { + return localAudioConfigSchema.extend({ + apiKey: localAudioConfigSchema.shape.apiKey.meta({ + labelLocalized: t('settings.pages.providers.catalog.edit.config.common.fields.field.api-key.label'), + descriptionLocalized: t('settings.pages.providers.catalog.edit.config.common.fields.field.api-key.description'), + placeholderLocalized: t('settings.pages.providers.catalog.edit.config.common.fields.field.api-key.placeholder'), + type: 'password', + }), + baseUrl: localAudioConfigSchema.shape.baseUrl.meta({ + labelLocalized: t('settings.pages.providers.catalog.edit.config.common.fields.field.base-url.label'), + descriptionLocalized: t('settings.pages.providers.catalog.edit.config.common.fields.field.base-url.description'), + placeholderLocalized: t('settings.pages.providers.catalog.edit.config.common.fields.field.base-url.placeholder'), + }), + }) +} + +function normalizeBaseUrl(baseUrl: string | undefined) { + const normalized = baseUrl?.trim() ?? '' + return normalized && !normalized.endsWith('/') ? `${normalized}/` : normalized +} + +function createLocalAudioProvider(config: LocalAudioConfig) { + return createOpenAI(config.apiKey?.trim() ?? '', normalizeBaseUrl(config.baseUrl)) +} + +function createLocalTranscriptionProvider(config: LocalAudioConfig) { + const provider = createLocalAudioProvider(config) + const transcription = provider.transcription.bind(provider) + provider.transcription = (model: string, extraOptions?: Record) => ({ + ...transcription(model), + ...extraOptions, + }) + return provider +} + +function createLocalAudioValidators() { + return { + validateConfig: [ + ({ t }: { t: ComposerTranslation }) => ({ + id: 'local-audio:check-config', + name: t('settings.pages.providers.catalog.edit.validators.openai-compatible.check-config.title'), + validator: async (config: LocalAudioConfig) => { + const valid = Boolean(config.baseUrl) + const reason = valid + ? '' + : 'Base URL is required. This is likely a bug, report to developers on https://github.com/moeru-ai/airi/issues.' + return { + errors: valid ? [] : [{ error: new Error('Base URL is required.') }], + reason, + reasonKey: '', + valid, + } + }, + }), + ], + } +} + +async function isBrowserAndMemoryEnough() { + if (isStageTamagotchi()) + return false + + if (await isWebGPUSupported()) + return true + + if ('navigator' in globalThis && globalThis.navigator != null && 'deviceMemory' in globalThis.navigator && typeof globalThis.navigator.deviceMemory === 'number') { + // The browser model needs at least 8 GB of system memory without WebGPU. + return globalThis.navigator.deviceMemory >= 8 + } + + return false +} + +export const providerAppLocalAudioSpeech = defineProvider({ + id: 'app-local-audio-speech', + name: 'App (Local)', + nameLocalize: ({ t }) => t('settings.pages.providers.provider.app-local-audio-speech.title'), + description: 'https://github.com/huggingface/candle', + descriptionLocalize: ({ t }) => t('settings.pages.providers.provider.app-local-audio-speech.description'), + tasks: ['text-to-speech', 'tts'], + icon: 'i-lobe-icons:huggingface', + isAvailableBy: isStageTamagotchi, + createProviderConfig: ({ t }) => createLocalAudioConfigSchema(t), + createProvider: createLocalAudioProvider, + validators: createLocalAudioValidators(), +}) + +export const providerAppLocalAudioTranscription = defineProvider({ + id: 'app-local-audio-transcription', + name: 'App (Local)', + nameLocalize: ({ t }) => t('settings.pages.providers.provider.app-local-audio-transcription.title'), + description: 'https://github.com/huggingface/candle', + descriptionLocalize: ({ t }) => t('settings.pages.providers.provider.app-local-audio-transcription.description'), + tasks: ['speech-to-text', 'automatic-speech-recognition', 'asr', 'stt'], + icon: 'i-lobe-icons:huggingface', + isAvailableBy: isStageTamagotchi, + capabilities: { + transcription: { + protocol: 'http', + generateOutput: true, + streamOutput: false, + streamInput: false, + }, + }, + createProviderConfig: ({ t }) => createLocalAudioConfigSchema(t), + createProvider: createLocalTranscriptionProvider, + validators: createLocalAudioValidators(), +}) + +export const providerBrowserLocalAudioSpeech = defineProvider({ + id: 'browser-local-audio-speech', + name: 'Browser (Local)', + nameLocalize: ({ t }) => t('settings.pages.providers.provider.browser-local-audio-speech.title'), + description: 'https://github.com/moeru-ai/xsai-transformers', + descriptionLocalize: ({ t }) => t('settings.pages.providers.provider.browser-local-audio-speech.description'), + tasks: ['text-to-speech', 'tts'], + icon: 'i-lobe-icons:huggingface', + isAvailableBy: isBrowserAndMemoryEnough, + createProviderConfig: ({ t }) => createLocalAudioConfigSchema(t), + createProvider: createLocalAudioProvider, + validators: createLocalAudioValidators(), +}) + +export const providerBrowserLocalAudioTranscription = defineProvider({ + id: 'browser-local-audio-transcription', + name: 'Browser (Local)', + nameLocalize: ({ t }) => t('settings.pages.providers.provider.browser-local-audio-transcription.title'), + description: 'https://github.com/moeru-ai/xsai-transformers', + descriptionLocalize: ({ t }) => t('settings.pages.providers.provider.browser-local-audio-transcription.description'), + tasks: ['speech-to-text', 'automatic-speech-recognition', 'asr', 'stt'], + icon: 'i-lobe-icons:huggingface', + isAvailableBy: isBrowserAndMemoryEnough, + capabilities: { + transcription: { + protocol: 'http', + generateOutput: true, + streamOutput: false, + streamInput: false, + }, + }, + createProviderConfig: ({ t }) => createLocalAudioConfigSchema(t), + createProvider: createLocalTranscriptionProvider, + validators: createLocalAudioValidators(), +}) diff --git a/packages/stage-ui/src/libs/providers/providers/mimo-audio/index.ts b/packages/stage-ui/src/libs/providers/providers/mimo-audio/index.ts new file mode 100644 index 000000000..b7b0b0685 --- /dev/null +++ b/packages/stage-ui/src/libs/providers/providers/mimo-audio/index.ts @@ -0,0 +1,262 @@ +import { z } from 'zod' + +import { defineProvider } from '../registry' + +const mimoSpeechConfigSchema = z.object({ + apiKey: z.string(), + baseUrl: z.string().default('https://api.xiaomimimo.com/v1/'), + model: z.string().default('mimo-v2.5-tts'), + voice: z.string().default('mimo_default'), + format: z.string().default('wav'), + stylePrompt: z.string().optional(), + voiceSample: z.string().optional(), +}) + +const mimoTranscriptionConfigSchema = z.object({ + apiKey: z.string(), + baseUrl: z.string().default('https://api.xiaomimimo.com/v1/'), + model: z.string().default('mimo-v2-omni'), +}) + +type MimoSpeechConfig = z.input +type MimoTranscriptionConfig = z.input +type MimoConfig = MimoSpeechConfig | MimoTranscriptionConfig + +function normalizeBaseUrl(baseUrl: string | undefined) { + return `${(baseUrl || 'https://api.xiaomimimo.com/v1/').replace(/\/+$/, '')}/` +} + +function createMimoValidators(id: string) { + return { + validateConfig: [ + ({ t }: { t: (key: string) => string }) => ({ + id: `${id}:check-config`, + name: t('settings.pages.providers.catalog.edit.validators.openai-compatible.check-config.title'), + validator: async (config: TConfig) => { + const errors: Array<{ error: unknown }> = [] + if (!config.apiKey?.trim()) + errors.push({ error: new Error('API key is required.') }) + if (!config.baseUrl?.trim()) + errors.push({ error: new Error('Base URL is required.') }) + + return { + errors, + reason: errors.map(item => (item.error as Error).message).join(', '), + reasonKey: '', + valid: errors.length === 0, + } + }, + }), + ], + } +} + +function createMimoSpeechProvider(config: MimoSpeechConfig) { + const apiKey = config.apiKey?.trim() ?? '' + const baseUrl = normalizeBaseUrl(config.baseUrl) + const defaultModel = config.model || 'mimo-v2.5-tts' + const defaultVoice = config.voice || 'mimo_default' + const defaultFormat = config.format || 'wav' + + return { + speech: () => ({ + baseURL: baseUrl, + model: defaultModel, + fetch: async (_input: RequestInfo | URL, init?: RequestInit) => { + if (!init?.body || typeof init.body !== 'string') + throw new Error('Invalid request body') + + const body = JSON.parse(init.body) as { + input?: string + model?: string + response_format?: string + style_prompt?: string + voice_sample?: string + voice?: string + } + const model = body.model || defaultModel + const format = body.response_format || defaultFormat + const stylePrompt = body.style_prompt?.trim() || config.stylePrompt?.trim() || '' + const voiceSample = body.voice_sample?.trim() || config.voiceSample?.trim() || '' + const userPrompt = model === 'mimo-v2.5-tts-voiceclone' + ? stylePrompt + : stylePrompt || 'Use a natural, clear speaking style.' + + const audio: Record = { format } + if (model === 'mimo-v2.5-tts-voiceclone') { + if (!voiceSample) + throw new Error('MiMo voice clone requires a base64 audio sample in data URI format.') + audio.voice = voiceSample + } + else if (model === 'mimo-v2.5-tts') { + audio.voice = body.voice || defaultVoice + } + + if (model === 'mimo-v2.5-tts-voicedesign' && !stylePrompt) + throw new Error('MiMo voice design requires a style prompt in the user message.') + + const response = await fetch(new URL('chat/completions', baseUrl), { + method: 'POST', + headers: { 'Content-Type': 'application/json', 'api-key': apiKey }, + body: JSON.stringify({ + model, + messages: [ + { role: 'user', content: userPrompt }, + { role: 'assistant', content: body.input ?? '' }, + ], + audio, + }), + }) + if (!response.ok || !response.body) + throw new Error(`MiMo TTS request failed: ${response.status} ${response.statusText}`) + + const data = await response.json() as { + choices?: Array<{ message?: { audio?: { data?: string } } }> + } + const audioBase64 = data.choices?.[0]?.message?.audio?.data + if (!audioBase64) + throw new Error('MiMo TTS response missing audio data') + + const binary = atob(audioBase64) + const bytes = new Uint8Array(binary.length) + for (let index = 0; index < binary.length; index++) + bytes[index] = binary.charCodeAt(index) + + let contentType = `audio/${format}` + if (format === 'wav') + contentType = 'audio/wav' + else if (format === 'mp3') + contentType = 'audio/mpeg' + + return new Response(bytes.buffer, { + status: 200, + headers: { 'Content-Type': contentType }, + }) + }, + }), + } +} + +function audioFormatFromDataUri(dataUri: string) { + const mimeType = dataUri.split(';')[0]?.split(':')[1] || 'audio/wav' + const format = mimeType.split('/')[1] || 'wav' + if (format === 'webm' || format === 'mp4') + return format + if (format === 'mpeg' || format === 'mp3') + return 'mp3' + return 'wav' +} + +function readBlobAsDataUri(file: Blob) { + return new Promise((resolve, reject) => { + const reader = new FileReader() + reader.onload = () => resolve(String(reader.result)) + reader.onerror = () => reject(new Error('Failed to read audio file')) + reader.readAsDataURL(file) + }) +} + +function createMimoTranscriptionProvider(config: MimoTranscriptionConfig) { + const apiKey = config.apiKey?.trim() ?? '' + const baseUrl = normalizeBaseUrl(config.baseUrl) + const defaultModel = config.model || 'mimo-v2-omni' + + return { + transcription: (model: string) => ({ + baseURL: baseUrl, + model: model || defaultModel, + headers: {}, + fetch: async (_input: RequestInfo | URL, init?: RequestInit) => { + if (!(init?.body instanceof FormData)) + throw new Error('No audio file provided for transcription.') + + const file = init.body.get('file') + if (!(file instanceof Blob)) + throw new Error('No audio file provided for transcription.') + + const modelName = String(init.body.get('model') || defaultModel) + const dataUri = await readBlobAsDataUri(file) + const base64Data = dataUri.split(',')[1] + const response = await fetch(new URL('chat/completions', baseUrl), { + method: 'POST', + headers: { 'Content-Type': 'application/json', 'api-key': apiKey }, + body: JSON.stringify({ + model: modelName, + messages: [{ + role: 'user', + content: [ + { type: 'text', text: 'Transcribe the audio content.' }, + { type: 'input_audio', input_audio: { data: base64Data, format: audioFormatFromDataUri(dataUri) } }, + ], + }], + }), + }) + if (!response.ok) { + const errorBody = await response.text().catch(() => '') + throw new Error(`MiMo transcription failed: ${response.status} ${response.statusText}${errorBody ? ` — ${errorBody}` : ''}`) + } + + const data = await response.json() as { choices?: Array<{ message?: { content?: string } }> } + return new Response(JSON.stringify({ text: data.choices?.[0]?.message?.content || '' }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }) + }, + }), + } +} + +export const providerMimoAudioSpeech = defineProvider({ + id: 'mimo-audio-speech', + name: 'Xiaomi MiMo', + nameLocalize: ({ t }) => t('settings.pages.providers.provider.mimo.title'), + description: 'api.xiaomimimo.com', + descriptionLocalize: ({ t }) => t('settings.pages.providers.provider.mimo.description'), + tasks: ['text-to-speech'], + icon: 'i-simple-icons:xiaomi', + createProviderConfig: () => mimoSpeechConfigSchema, + createProvider: createMimoSpeechProvider, + validationRequiredWhen: config => Boolean(config.apiKey?.trim() && config.baseUrl?.trim()), + validators: createMimoValidators('mimo-audio-speech'), + extraMethods: { + listModels: async () => [ + { id: 'mimo-v2.5-tts', name: 'MiMo v2.5 TTS', provider: 'mimo-audio-speech', description: 'Preset voice synthesis with the built-in MiMo voice list', deprecated: false }, + { id: 'mimo-v2.5-tts-voicedesign', name: 'MiMo v2.5 TTS Voice Design', provider: 'mimo-audio-speech', description: 'Design a new voice from a natural language description', deprecated: false }, + { id: 'mimo-v2.5-tts-voiceclone', name: 'MiMo v2.5 TTS Voice Clone', provider: 'mimo-audio-speech', description: 'Clone a voice from a base64-encoded audio sample', deprecated: false }, + ], + listVoices: async () => [ + { id: 'mimo_default', name: 'MiMo-默认', provider: 'mimo-audio-speech', gender: 'female', languages: [{ code: 'en', title: 'English' }, { code: 'zh', title: 'Chinese' }] }, + { id: '冰糖', name: '冰糖', provider: 'mimo-audio-speech', gender: 'female', languages: [{ code: 'zh', title: 'Chinese' }] }, + { id: '茉莉', name: '茉莉', provider: 'mimo-audio-speech', gender: 'female', languages: [{ code: 'zh', title: 'Chinese' }] }, + { id: '苏打', name: '苏打', provider: 'mimo-audio-speech', gender: 'male', languages: [{ code: 'zh', title: 'Chinese' }] }, + { id: '白桦', name: '白桦', provider: 'mimo-audio-speech', gender: 'male', languages: [{ code: 'zh', title: 'Chinese' }] }, + { id: 'Mia', name: 'Mia', provider: 'mimo-audio-speech', gender: 'female', languages: [{ code: 'en', title: 'English' }] }, + { id: 'Chloe', name: 'Chloe', provider: 'mimo-audio-speech', gender: 'female', languages: [{ code: 'en', title: 'English' }] }, + { id: 'Milo', name: 'Milo', provider: 'mimo-audio-speech', gender: 'male', languages: [{ code: 'en', title: 'English' }] }, + { id: 'Dean', name: 'Dean', provider: 'mimo-audio-speech', gender: 'male', languages: [{ code: 'en', title: 'English' }] }, + ], + }, +}) + +export const providerMimoAudioTranscription = defineProvider({ + id: 'mimo-audio-transcription', + name: 'Xiaomi MiMo', + nameLocalize: ({ t }) => t('settings.pages.providers.provider.mimo.title'), + description: 'api.xiaomimimo.com', + descriptionLocalize: ({ t }) => t('settings.pages.providers.provider.mimo.description'), + tasks: ['speech-to-text', 'automatic-speech-recognition', 'asr', 'stt'], + icon: 'i-simple-icons:xiaomi', + capabilities: { + transcription: { protocol: 'http', generateOutput: true, streamOutput: false, streamInput: false }, + }, + createProviderConfig: () => mimoTranscriptionConfigSchema, + createProvider: createMimoTranscriptionProvider, + validationRequiredWhen: config => Boolean(config.apiKey?.trim() && config.baseUrl?.trim()), + validators: createMimoValidators('mimo-audio-transcription'), + extraMethods: { + listModels: async () => [ + { id: 'mimo-v2-omni', name: 'MiMo V2 Omni', provider: 'mimo-audio-transcription', description: 'Omni-modal model with native audio understanding and speech-to-text', contextLength: 256000, deprecated: false }, + { id: 'mimo-v2.5', name: 'MiMo V2.5', provider: 'mimo-audio-transcription', description: 'Latest omni-modal model with audio understanding, 1M context', contextLength: 1_000_000, deprecated: false }, + ], + }, +}) diff --git a/packages/stage-ui/src/libs/providers/providers/minimax-speech/index.ts b/packages/stage-ui/src/libs/providers/providers/minimax-speech/index.ts new file mode 100644 index 000000000..f15062c4f --- /dev/null +++ b/packages/stage-ui/src/libs/providers/providers/minimax-speech/index.ts @@ -0,0 +1,152 @@ +import { z } from 'zod' + +import { defineProvider } from '../registry' + +const minimaxSpeechConfigSchema = z.object({ + apiKey: z.string(), + baseUrl: z.string().default('https://api.minimax.io'), +}) + +type MinimaxSpeechConfig = z.input + +export const providerMinimaxSpeech = defineProvider({ + id: 'minimax-speech', + name: 'MiniMax Speech', + nameLocalize: ({ t }) => t('settings.pages.providers.provider.minimax-speech.title'), + description: 'minimax.io', + descriptionLocalize: ({ t }) => t('settings.pages.providers.provider.minimax-speech.description'), + tasks: ['text-to-speech'], + icon: 'i-lobe-icons:minimax', + iconColor: 'i-lobe-icons:minimax-color', + createProviderConfig: () => minimaxSpeechConfigSchema, + createProvider(config) { + const apiKey = config.apiKey.trim() + const baseUrl = (config.baseUrl || 'https://api.minimax.io').replace(/\/$/, '') + + return { + speech: () => ({ + baseURL: `${baseUrl}/v1/`, + model: 'speech-2.8-hd', + fetch: async (_input: RequestInfo | URL, init?: RequestInit) => { + if (!init?.body || typeof init.body !== 'string') + throw new Error('Invalid request body') + + const body = JSON.parse(init.body) as { input?: string, voice?: string, model?: string } + const response = await fetch(`${baseUrl}/v1/t2a_v2`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Authorization': `Bearer ${apiKey}`, + }, + body: JSON.stringify({ + model: body.model || 'speech-2.8-hd', + text: body.input ?? '', + stream: true, + voice_setting: { + voice_id: body.voice || 'English_Graceful_Lady', + speed: 1, + vol: 1, + pitch: 0, + }, + audio_setting: { + sample_rate: 32000, + bitrate: 128000, + format: 'mp3', + channel: 1, + }, + }), + }) + + if (!response.ok || !response.body) + throw new Error(`MiniMax TTS request failed: ${response.status} ${response.statusText}`) + + // MiniMax streams SSE events that contain hex-encoded audio chunks. + const reader = response.body.getReader() + const decoder = new TextDecoder() + const audioChunks: Uint8Array[] = [] + let buffer = '' + + while (true) { + const { done, value } = await reader.read() + if (done) + break + + buffer += decoder.decode(value, { stream: true }) + const lines = buffer.split('\n') + buffer = lines.pop() || '' + for (const line of lines) { + if (!line.startsWith('data:')) + continue + + const json = line.slice(5).trim() + if (!json || json === '[DONE]') + continue + + try { + const event = JSON.parse(json) as { data?: { audio?: string, status?: number } } + // Status 2 is the final summary. Its audio duplicates prior chunks. + if (event.data?.audio && event.data.status !== 2) { + const bytes = new Uint8Array(event.data.audio.length / 2) + for (let index = 0; index < event.data.audio.length; index += 2) + bytes[index / 2] = Number.parseInt(event.data.audio.slice(index, index + 2), 16) + audioChunks.push(bytes) + } + } + catch { + // A malformed SSE event does not invalidate earlier audio chunks. + } + } + } + + const combined = new Uint8Array(audioChunks.reduce((sum, chunk) => sum + chunk.length, 0)) + let offset = 0 + for (const chunk of audioChunks) { + combined.set(chunk, offset) + offset += chunk.length + } + + return new Response(combined.buffer, { + status: 200, + headers: { 'Content-Type': 'audio/mpeg' }, + }) + }, + }), + } + }, + validationRequiredWhen: config => Boolean(config.apiKey?.trim()), + validators: { + validateConfig: [ + ({ t }) => ({ + id: 'minimax-speech:check-config', + name: t('settings.pages.providers.catalog.edit.validators.openai-compatible.check-config.title'), + validator: async (config) => { + const valid = Boolean(config.apiKey?.trim()) + return { + errors: valid ? [] : [{ error: new Error('API key is required.') }], + reason: valid ? '' : 'API key is required.', + reasonKey: '', + valid, + } + }, + }), + ], + }, + extraMethods: { + listModels: async () => [ + { id: 'speech-2.8-hd', name: 'Speech 2.8 HD', provider: 'minimax-speech', description: 'High-definition TTS model with natural prosody', deprecated: false }, + { id: 'speech-2.8-turbo', name: 'Speech 2.8 Turbo', provider: 'minimax-speech', description: 'Fast TTS model for low-latency scenarios', deprecated: false }, + ], + listVoices: async () => [ + { id: 'English_Graceful_Lady', name: 'Graceful Lady', provider: 'minimax-speech', gender: 'female', languages: [{ code: 'en', title: 'English' }] }, + { id: 'English_Insightful_Speaker', name: 'Insightful Speaker', provider: 'minimax-speech', gender: 'male', languages: [{ code: 'en', title: 'English' }] }, + { id: 'English_radiant_girl', name: 'Radiant Girl', provider: 'minimax-speech', gender: 'female', languages: [{ code: 'en', title: 'English' }] }, + { id: 'English_Persuasive_Man', name: 'Persuasive Man', provider: 'minimax-speech', gender: 'male', languages: [{ code: 'en', title: 'English' }] }, + { id: 'English_Lucky_Robot', name: 'Lucky Robot', provider: 'minimax-speech', gender: 'neutral', languages: [{ code: 'en', title: 'English' }] }, + { id: 'English_expressive_narrator', name: 'Expressive Narrator', provider: 'minimax-speech', gender: 'neutral', languages: [{ code: 'en', title: 'English' }] }, + { id: 'Mandarin_Gentle_Woman', name: 'Gentle Woman', provider: 'minimax-speech', gender: 'female', languages: [{ code: 'zh', title: 'Chinese' }] }, + { id: 'Mandarin_Steadfast_Man', name: 'Steadfast Man', provider: 'minimax-speech', gender: 'male', languages: [{ code: 'zh', title: 'Chinese' }] }, + { id: 'Mandarin_Sweet_Girl', name: 'Sweet Girl', provider: 'minimax-speech', gender: 'female', languages: [{ code: 'zh', title: 'Chinese' }] }, + { id: 'Mandarin_Magnetic_Gentleman', name: 'Magnetic Gentleman', provider: 'minimax-speech', gender: 'male', languages: [{ code: 'zh', title: 'Chinese' }] }, + ], + }, +}) diff --git a/packages/stage-ui/src/libs/providers/providers/openai-audio/index.ts b/packages/stage-ui/src/libs/providers/providers/openai-audio/index.ts new file mode 100644 index 000000000..c5ec9a76d --- /dev/null +++ b/packages/stage-ui/src/libs/providers/providers/openai-audio/index.ts @@ -0,0 +1,266 @@ +import type { ComposerTranslation } from 'vue-i18n' + +import { createOpenAI } from '@xsai-ext/providers/create' +import { listModels } from '@xsai/model' +import { z } from 'zod' + +import { defineProvider } from '../registry' + +const OPENAI_BASE_URL = 'https://api.openai.com/v1/' + +const openAIAudioConfigSchema = z.object({ + apiKey: z.string(), + baseUrl: z.string().default(OPENAI_BASE_URL), +}) + +const openAICompatibleAudioConfigSchema = z.object({ + apiKey: z.string(), + baseUrl: z.string().default(''), +}) + +type OpenAIAudioConfig = z.input +type OpenAICompatibleAudioConfig = z.input +type AudioConfig = OpenAIAudioConfig | OpenAICompatibleAudioConfig + +function createAudioConfigSchema(schema: T, t: ComposerTranslation) { + return schema.extend({ + apiKey: schema.shape.apiKey.meta({ + labelLocalized: t('settings.pages.providers.catalog.edit.config.common.fields.field.api-key.label'), + descriptionLocalized: t('settings.pages.providers.catalog.edit.config.common.fields.field.api-key.description'), + placeholderLocalized: t('settings.pages.providers.catalog.edit.config.common.fields.field.api-key.placeholder'), + type: 'password', + }), + baseUrl: schema.shape.baseUrl.meta({ + labelLocalized: t('settings.pages.providers.catalog.edit.config.common.fields.field.base-url.label'), + descriptionLocalized: t('settings.pages.providers.catalog.edit.config.common.fields.field.base-url.description'), + placeholderLocalized: t('settings.pages.providers.catalog.edit.config.common.fields.field.base-url.placeholder'), + }), + }) +} + +function normalizeBaseUrl(baseUrl: string | undefined) { + const value = baseUrl?.trim() ?? '' + return value && !value.endsWith('/') ? `${value}/` : value +} + +function createAudioProvider(config: AudioConfig) { + return createOpenAI(config.apiKey.trim(), normalizeBaseUrl(config.baseUrl)) +} + +function createTranscriptionProvider(config: AudioConfig) { + const provider = createAudioProvider(config) + const transcription = provider.transcription.bind(provider) + provider.transcription = (model: string, extraOptions?: Record) => ({ + ...transcription(model), + ...extraOptions, + }) + return provider +} + +function createAudioValidators() { + return { + validateConfig: [ + ({ t }: { t: ComposerTranslation }) => ({ + id: 'openai-audio:check-config', + name: t('settings.pages.providers.catalog.edit.validators.openai-compatible.check-config.title'), + validator: async (config: TConfig) => { + const errors: Array<{ error: unknown }> = [] + const apiKey = config.apiKey?.trim() ?? '' + const baseUrl = config.baseUrl?.trim() ?? '' + + if (!apiKey) + errors.push({ error: new Error('API Key is required') }) + if (!baseUrl) + errors.push({ error: new Error('Base URL is required.') }) + + if (baseUrl) { + try { + const url = new URL(baseUrl) + if (!url.host) + errors.push({ error: new Error('Base URL is not absolute. Try to include a scheme (http:// or https://).') }) + else if (!baseUrl.endsWith('/')) + errors.push({ error: new Error('Base URL must end with a trailing slash (/).') }) + } + catch { + errors.push({ error: new Error('Base URL is not absolute. Try to include a scheme (http:// or https://).') }) + } + } + + return { + errors, + reason: errors.map(item => (item.error as Error).message).join(', '), + reasonKey: '', + valid: errors.length === 0, + } + }, + }), + ], + } +} + +const openAISpeechModels = [ + { + id: 'tts-1', + name: 'TTS-1', + provider: 'openai-audio-speech', + description: 'Optimized for real-time text-to-speech tasks', + contextLength: 0, + deprecated: false, + }, + { + id: 'tts-1-hd', + name: 'TTS-1-HD', + provider: 'openai-audio-speech', + description: 'Higher fidelity audio output', + contextLength: 0, + deprecated: false, + }, + { + id: 'gpt-4o-mini-tts', + name: 'GPT-4o Mini TTS', + provider: 'openai-audio-speech', + description: 'GPT-4o Mini optimized for text-to-speech', + contextLength: 0, + deprecated: false, + }, + { + id: 'gpt-4o-mini-tts-2025-12-15', + name: 'GPT-4o Mini TTS (2025-12-15)', + provider: 'openai-audio-speech', + description: 'GPT-4o Mini TTS snapshot from 2025-12-15', + contextLength: 0, + deprecated: false, + }, +] + +// OpenAI does not provide a voice-list endpoint. This list follows the +// create-speech API. The TTS-1 models support nine voices. GPT-4o Mini TTS +// also supports ballad, verse, marin, and cedar. +const openAISpeechVoices = [ + { id: 'alloy', models: ['tts-1', 'tts-1-hd', 'gpt-4o-mini-tts', 'gpt-4o-mini-tts-2025-12-15'] }, + { id: 'ash', models: ['tts-1', 'tts-1-hd', 'gpt-4o-mini-tts', 'gpt-4o-mini-tts-2025-12-15'] }, + { id: 'ballad', models: ['gpt-4o-mini-tts', 'gpt-4o-mini-tts-2025-12-15'] }, + { id: 'coral', models: ['tts-1', 'tts-1-hd', 'gpt-4o-mini-tts', 'gpt-4o-mini-tts-2025-12-15'] }, + { id: 'echo', models: ['tts-1', 'tts-1-hd', 'gpt-4o-mini-tts', 'gpt-4o-mini-tts-2025-12-15'] }, + { id: 'fable', models: ['tts-1', 'tts-1-hd', 'gpt-4o-mini-tts', 'gpt-4o-mini-tts-2025-12-15'] }, + { id: 'onyx', models: ['tts-1', 'tts-1-hd', 'gpt-4o-mini-tts', 'gpt-4o-mini-tts-2025-12-15'] }, + { id: 'nova', models: ['tts-1', 'tts-1-hd', 'gpt-4o-mini-tts', 'gpt-4o-mini-tts-2025-12-15'] }, + { id: 'sage', models: ['tts-1', 'tts-1-hd', 'gpt-4o-mini-tts', 'gpt-4o-mini-tts-2025-12-15'] }, + { id: 'shimmer', models: ['tts-1', 'tts-1-hd', 'gpt-4o-mini-tts', 'gpt-4o-mini-tts-2025-12-15'] }, + { id: 'verse', models: ['gpt-4o-mini-tts', 'gpt-4o-mini-tts-2025-12-15'] }, + { id: 'marin', models: ['gpt-4o-mini-tts', 'gpt-4o-mini-tts-2025-12-15'] }, + { id: 'cedar', models: ['gpt-4o-mini-tts', 'gpt-4o-mini-tts-2025-12-15'] }, +].map(voice => ({ + id: voice.id, + name: `${voice.id[0].toUpperCase()}${voice.id.slice(1)}`, + provider: 'openai-audio-speech', + languages: [], + compatibleModels: voice.models, +})) + +const openAITranscriptionModels = [ + ['gpt-4o-transcribe', 'GPT-4o Transcribe', 'High-quality transcription model'], + ['gpt-4o-mini-transcribe', 'GPT-4o Mini Transcribe', 'Faster, cost-effective transcription model'], + ['gpt-4o-mini-transcribe-2025-12-15', 'GPT-4o Mini Transcribe (2025-12-15)', 'GPT-4o Mini Transcribe snapshot from 2025-12-15'], + ['whisper-1', 'Whisper-1', 'Powered by our open source Whisper V2 model'], + ['gpt-4o-transcribe-diarize', 'GPT-4o Transcribe Diarize', 'Transcription with speaker diarization'], +].map(([id, name, description]) => ({ + id, + name, + provider: 'openai-audio-transcription', + description, + contextLength: 0, + deprecated: false, +})) + +export const providerOpenAIAudioSpeech = defineProvider({ + id: 'openai-audio-speech', + name: 'OpenAI', + nameLocalize: ({ t }) => t('settings.pages.providers.provider.openai.title'), + description: 'openai.com', + descriptionLocalize: ({ t }) => t('settings.pages.providers.provider.openai.description'), + tasks: ['text-to-speech'], + icon: 'i-lobe-icons:openai', + createProviderConfig: ({ t }) => createAudioConfigSchema(openAIAudioConfigSchema, t), + createProvider: createAudioProvider, + validationRequiredWhen: config => Boolean(config.apiKey?.trim() && config.baseUrl?.trim()), + validators: createAudioValidators(), + extraMethods: { + listModels: async () => openAISpeechModels, + listVoices: async () => openAISpeechVoices, + }, +}) + +export const providerOpenAICompatibleAudioSpeech = defineProvider({ + id: 'openai-compatible-audio-speech', + name: 'OpenAI Compatible', + nameLocalize: ({ t }) => t('settings.pages.providers.provider.openai-compatible.title'), + description: 'Connect to any API that follows the OpenAI specification.', + descriptionLocalize: ({ t }) => t('settings.pages.providers.provider.openai-compatible.description'), + tasks: ['text-to-speech'], + icon: 'i-lobe-icons:openai', + createProviderConfig: ({ t }) => createAudioConfigSchema(openAICompatibleAudioConfigSchema, t), + createProvider: createAudioProvider, + validationRequiredWhen: config => Boolean(config.apiKey?.trim() && config.baseUrl?.trim()), + validators: createAudioValidators(), + extraMethods: { + listVoices: async () => [], + listModels: async (config) => { + const apiKey = config.apiKey?.trim() ?? '' + const baseUrl = normalizeBaseUrl(config.baseUrl) + if (!apiKey || !baseUrl) + return [] + + const models = await listModels({ apiKey, baseURL: baseUrl }) + return models + .filter(model => model.id.toLowerCase().includes('tts')) + .map(model => ({ + id: model.id, + name: model.id, + provider: 'openai-compatible-audio-speech', + description: '', + contextLength: 0, + deprecated: false, + })) + }, + }, +}) + +export const providerOpenAIAudioTranscription = defineProvider({ + id: 'openai-audio-transcription', + name: 'OpenAI', + nameLocalize: ({ t }) => t('settings.pages.providers.provider.openai.title'), + description: 'openai.com', + descriptionLocalize: ({ t }) => t('settings.pages.providers.provider.openai.description'), + tasks: ['speech-to-text', 'automatic-speech-recognition', 'asr', 'stt'], + icon: 'i-lobe-icons:openai', + capabilities: { + transcription: { protocol: 'http', generateOutput: true, streamOutput: false, streamInput: false }, + }, + createProviderConfig: ({ t }) => createAudioConfigSchema(openAIAudioConfigSchema, t), + createProvider: createTranscriptionProvider, + validationRequiredWhen: config => Boolean(config.apiKey?.trim() && config.baseUrl?.trim()), + validators: createAudioValidators(), + extraMethods: { + listModels: async () => openAITranscriptionModels, + }, +}) + +export const providerOpenAICompatibleAudioTranscription = defineProvider({ + id: 'openai-compatible-audio-transcription', + name: 'OpenAI Compatible', + nameLocalize: ({ t }) => t('settings.pages.providers.provider.openai-compatible.title'), + description: 'Connect to any API that follows the OpenAI specification.', + descriptionLocalize: ({ t }) => t('settings.pages.providers.provider.openai-compatible.description'), + tasks: ['speech-to-text', 'automatic-speech-recognition', 'asr', 'stt'], + icon: 'i-lobe-icons:openai', + capabilities: { + transcription: { protocol: 'http', generateOutput: true, streamOutput: false, streamInput: false }, + }, + createProviderConfig: ({ t }) => createAudioConfigSchema(openAICompatibleAudioConfigSchema, t), + createProvider: createTranscriptionProvider, + validationRequiredWhen: config => Boolean(config.apiKey?.trim() && config.baseUrl?.trim()), + validators: createAudioValidators(), + // Transcription model names are not reliably available from /v1/models. + extraMethods: { listModels: async () => [] }, +}) diff --git a/packages/stage-ui/src/libs/providers/providers/openrouter-audio-speech/index.ts b/packages/stage-ui/src/libs/providers/providers/openrouter-audio-speech/index.ts new file mode 100644 index 000000000..e0fad4796 --- /dev/null +++ b/packages/stage-ui/src/libs/providers/providers/openrouter-audio-speech/index.ts @@ -0,0 +1,200 @@ +import { toWavFromPCM16 } from '@proj-airi/audio/encoding' +import { z } from 'zod' + +import { OPENROUTER_ATTRIBUTION_HEADERS } from '../openrouter-ai' +import { defineProvider } from '../registry' + +const DEFAULT_BASE_URL = 'https://openrouter.ai/api/v1/' +const DEFAULT_MODEL = 'openai/gpt-audio-mini' + +const openRouterAudioConfigSchema = z.object({ + apiKey: z.string(), + baseUrl: z.string().default(DEFAULT_BASE_URL), +}) + +type OpenRouterAudioConfig = z.input + +const openAIVoices = [ + 'alloy', + 'ash', + 'ballad', + 'coral', + 'echo', + 'fable', + 'onyx', + 'nova', + 'sage', + 'shimmer', + 'verse', + 'marin', + 'cedar', +] as const + +function normalizeBaseUrl(baseUrl: string | undefined) { + const value = baseUrl?.trim() || DEFAULT_BASE_URL + return value.endsWith('/') ? value : `${value}/` +} + +function ttsPrompt(input: string) { + return `Read this text aloud exactly as written, without any commentary or extra words:\n\n${input}` +} + +async function collectAudioChunks(body: ReadableStream) { + const reader = body.getReader() + const decoder = new TextDecoder() + const chunks: string[] = [] + let buffer = '' + let done = false + + while (!done) { + const result = await reader.read() + if (result.done) + break + + buffer += decoder.decode(result.value, { stream: true }) + const lines = buffer.split('\n') + buffer = lines.pop() || '' + for (const line of lines) { + if (!line.startsWith('data: ')) + continue + + const data = line.slice('data: '.length).trim() + if (data === '[DONE]') { + done = true + break + } + + try { + const event = JSON.parse(data) as { + choices?: Array<{ delta?: { audio?: { data?: string } } }> + } + const audio = event.choices?.[0]?.delta?.audio?.data + if (audio) + chunks.push(audio) + } + catch (error) { + console.warn('Skipping malformed SSE chunk from OpenRouter audio stream:', data, error) + } + } + } + + return chunks +} + +function decodeBase64Pcm(chunks: string[]) { + const binary = atob(chunks.join('')) + const bytes = new Uint8Array(binary.length) + for (let index = 0; index < binary.length; index++) + bytes[index] = binary.charCodeAt(index) + return bytes +} + +function createAudioFetch(apiKey: string, baseUrl: string, model: string) { + return async (_input: RequestInfo | URL, init?: RequestInit) => { + if (!init?.body || typeof init.body !== 'string') + throw new Error('Invalid request body') + + const body = JSON.parse(init.body) as { input?: string, voice?: string } + const response = await globalThis.fetch(new URL('chat/completions', baseUrl), { + method: 'POST', + headers: { + 'Authorization': `Bearer ${apiKey}`, + 'Content-Type': 'application/json', + ...OPENROUTER_ATTRIBUTION_HEADERS, + }, + body: JSON.stringify({ + model, + messages: [{ role: 'user', content: ttsPrompt(body.input ?? '') }], + modalities: ['text', 'audio'], + audio: { voice: body.voice, format: 'pcm16' }, + stream: true, + }), + }) + if (!response.ok) + throw new Error(`OpenRouter audio request failed: ${response.status} ${await response.text()}`) + if (!response.body) + throw new Error('OpenRouter audio response has no body') + + const wav = toWavFromPCM16(decodeBase64Pcm(await collectAudioChunks(response.body)), 24000) + return new Response(new Blob([wav], { type: 'audio/wav' }), { + status: 200, + headers: { 'Content-Type': 'audio/wav' }, + }) + } +} + +export const providerOpenRouterAudioSpeech = defineProvider({ + id: 'openrouter-audio-speech', + name: 'OpenRouter', + nameLocalize: ({ t }) => t('settings.pages.providers.provider.openrouter-audio-speech.title'), + description: 'openrouter.ai', + descriptionLocalize: ({ t }) => t('settings.pages.providers.provider.openrouter-audio-speech.description'), + tasks: ['text-to-speech'], + icon: 'i-lobe-icons:openrouter', + createProviderConfig: () => openRouterAudioConfigSchema, + createProvider(config) { + const apiKey = config.apiKey.trim() + const baseUrl = normalizeBaseUrl(config.baseUrl) + return { + speech: (model?: string) => { + const resolvedModel = model || DEFAULT_MODEL + return { + baseURL: baseUrl, + model: resolvedModel, + fetch: createAudioFetch(apiKey, baseUrl, resolvedModel), + } + }, + } + }, + validationRequiredWhen: config => Boolean(config.apiKey?.trim()), + validators: { + validateConfig: [ + ({ t }) => ({ + id: 'openrouter-audio-speech:check-config', + name: t('settings.pages.providers.catalog.edit.validators.openai-compatible.check-config.title'), + validator: async (config) => { + const valid = Boolean(config.apiKey?.trim()) + return { + errors: valid ? [] : [{ error: new Error('API Key is required.') }], + reason: valid ? '' : 'API Key is required.', + reasonKey: '', + valid, + } + }, + }), + ], + }, + extraMethods: { + listModels: async (config) => { + try { + const response = await fetch(new URL('models?output_modality=audio', normalizeBaseUrl(config.baseUrl)), { + headers: OPENROUTER_ATTRIBUTION_HEADERS, + }) + if (!response.ok) + return [] + + const data = await response.json() as { + data?: Array<{ id: string, name?: string, description?: string, context_length?: number }> + } + return (data.data ?? []).map(model => ({ + id: model.id, + name: model.name || model.id, + provider: 'openrouter-audio-speech', + description: model.description || '', + contextLength: model.context_length || 0, + deprecated: false, + })) + } + catch (error) { + console.error('Failed to fetch OpenRouter audio models:', error) + return [] + } + }, + listVoices: async () => openAIVoices.map(id => ({ + id, + name: `${id[0].toUpperCase()}${id.slice(1)}`, + provider: 'openrouter-audio-speech', + languages: [], + })), + }, +}) diff --git a/packages/stage-ui/src/libs/providers/providers/player2-speech/index.ts b/packages/stage-ui/src/libs/providers/providers/player2-speech/index.ts new file mode 100644 index 000000000..3973cd370 --- /dev/null +++ b/packages/stage-ui/src/libs/providers/providers/player2-speech/index.ts @@ -0,0 +1,107 @@ +import { errorMessageFrom } from '@moeru/std' +import { createPlayer2 } from '@xsai-ext/providers/special/create' +import { z } from 'zod' + +import { defineProvider } from '../registry' + +const player2ConfigSchema = z.object({ + baseUrl: z.string().default('http://localhost:4315/v1/'), +}) + +type Player2Config = z.input +type Player2VoiceLanguage = keyof typeof player2VoiceLanguages + +const player2VoiceLanguages = { + american_english: { code: 'en', title: 'English' }, + british_english: { code: 'en', title: 'English' }, + japanese: { code: 'ja', title: 'Japanese' }, + mandarin_chinese: { code: 'zh', title: 'Chinese' }, + spanish: { code: 'es', title: 'Spanish' }, + french: { code: 'fr', title: 'French' }, + hindi: { code: 'hi', title: 'Hindi' }, + italian: { code: 'it', title: 'Italian' }, + brazilian_portuguese: { code: 'pt', title: 'Portuguese' }, +} as const + +function normalizeBaseUrl(baseUrl: string | undefined) { + const value = baseUrl?.trim() ?? 'http://localhost:4315/v1/' + return value.endsWith('/') ? value : `${value}/` +} + +export const providerPlayer2Speech = defineProvider({ + id: 'player2-speech', + name: 'Player2 Speech', + nameLocalize: ({ t }) => t('settings.pages.providers.provider.player2.title'), + description: 'player2.game', + descriptionLocalize: ({ t }) => t('settings.pages.providers.provider.player2.description'), + tasks: ['text-to-speech'], + icon: 'i-lobe-icons:player2', + createProviderConfig: () => player2ConfigSchema, + createProvider: config => createPlayer2(normalizeBaseUrl(config.baseUrl), 'airi'), + validationRequiredWhen: config => Boolean(config.baseUrl?.trim()), + validators: { + validateConfig: [ + ({ t }) => ({ + id: 'player2-speech:check-config', + name: t('settings.pages.providers.catalog.edit.validators.openai-compatible.check-config.title'), + validator: async (config) => { + const valid = Boolean(config.baseUrl?.trim()) + const reason = valid ? '' : 'Base URL is required. Default to http://localhost:4315/v1/' + return { errors: valid ? [] : [{ error: new Error(reason) }], reason, reasonKey: '', valid } + }, + }), + ], + validateProvider: [ + ({ t }) => ({ + id: 'player2-speech:check-connectivity', + name: t('settings.pages.providers.catalog.edit.validators.openai-compatible.check-connectivity.title'), + validator: async (config) => { + const controller = new AbortController() + const timeout = setTimeout(() => controller.abort(), 5000) + try { + const response = await fetch(new URL('health', normalizeBaseUrl(config.baseUrl)), { + headers: { 'player2-game-key': 'airi' }, + signal: controller.signal, + }) + if (!response.ok) { + const reason = `Player2 speech unreachable: HTTP ${response.status} ${response.statusText}` + return { errors: [{ error: new Error(reason) }], reason, reasonKey: '', valid: false } + } + } + catch (error) { + const reason = `Player2 speech connection failed: ${errorMessageFrom(error) ?? 'Unknown error'}` + return { errors: [{ error }], reason, reasonKey: '', valid: false } + } + finally { + clearTimeout(timeout) + } + + return { errors: [], reason: '', reasonKey: '', valid: true } + }, + }), + ], + }, + extraMethods: { + listModels: async () => [{ + id: 'player2-tts', + name: 'Player2 Speech', + provider: 'player2-speech', + description: 'Default model for Player2 speech endpoint', + contextLength: 0, + deprecated: false, + }], + listVoices: async (config) => { + const response = await fetch(new URL('tts/voices', normalizeBaseUrl(config.baseUrl))) + const data = await response.json() as { + voices?: Array<{ id: string, language: Player2VoiceLanguage, name: string, gender: string }> + } + return (data.voices ?? []).map(voice => ({ + id: voice.id, + name: voice.name, + provider: 'player2-speech', + gender: voice.gender, + languages: [player2VoiceLanguages[voice.language]], + })) + }, + }, +}) diff --git a/packages/stage-ui/src/libs/providers/providers/provider-definitions.test.ts b/packages/stage-ui/src/libs/providers/providers/provider-definitions.test.ts new file mode 100644 index 000000000..66364a07d --- /dev/null +++ b/packages/stage-ui/src/libs/providers/providers/provider-definitions.test.ts @@ -0,0 +1,122 @@ +import type { ComposerTranslation } from 'vue-i18n' + +import { describe, expect, it } from 'vitest' +import { z } from 'zod' + +import { providerBrowserWebSpeechApi } from './browser-web-speech-api' +import { providerElevenLabs } from './elevenlabs' +import { + providerAppLocalAudioSpeech, + providerAppLocalAudioTranscription, + providerBrowserLocalAudioSpeech, + providerBrowserLocalAudioTranscription, +} from './local-audio' +import { getDefinedProvider } from './registry' +import { providerSpeechNoop } from './speech-noop' + +import './index' + +const translate = ((key: string) => key) as unknown as ComposerTranslation + +describe('migrated provider definitions', () => { + it('registers every provider that moved out of the legacy store', () => { + const providerIds = [ + 'speech-noop', + 'app-local-audio-speech', + 'app-local-audio-transcription', + 'browser-local-audio-speech', + 'browser-local-audio-transcription', + 'openai-audio-speech', + 'openai-compatible-audio-speech', + 'openai-audio-transcription', + 'openai-compatible-audio-transcription', + 'aliyun-nls-transcription', + 'browser-web-speech-api', + 'elevenlabs', + 'deepgram-tts', + 'microsoft-speech', + 'index-tts-vllm', + 'alibaba-cloud-model-studio', + 'volcengine', + 'minimax-speech', + 'openrouter-audio-speech', + 'mimo-audio-speech', + 'comet-api-speech', + 'comet-api-transcription', + 'mimo-audio-transcription', + 'player2-speech', + 'kokoro-local', + 'google-gemini-audio-speech', + ] + + for (const providerId of providerIds) + expect(getDefinedProvider(providerId), providerId).toBeDefined() + }) + + it('creates the no-op speech provider through ProviderDefinition', () => { + const provider = providerSpeechNoop.createProvider({}) + + expect(provider).toHaveProperty('speech') + expect('speech' in provider && provider.speech('unused')).toMatchObject({ + baseURL: 'http://speech-noop.invalid/v1/', + model: 'noop', + }) + }) + + it('keeps local audio providers split by inference task', () => { + expect(providerAppLocalAudioSpeech.tasks).toContain('text-to-speech') + expect(providerBrowserLocalAudioSpeech.tasks).toContain('text-to-speech') + expect(providerAppLocalAudioTranscription.tasks).toContain('speech-to-text') + expect(providerBrowserLocalAudioTranscription.tasks).toContain('speech-to-text') + expect(providerAppLocalAudioTranscription.capabilities?.transcription).toEqual({ + protocol: 'http', + generateOutput: true, + streamOutput: false, + streamInput: false, + }) + }) + + it('keeps the local audio base URL validation in the definition', async () => { + const validator = providerAppLocalAudioSpeech.validators?.validateConfig?.[0]({ t: translate }) + + const missing = await validator?.validator({}, { t: translate }) + const configured = await validator?.validator({ baseUrl: 'http://localhost:1234/v1/' }, { t: translate }) + + expect(missing?.valid).toBe(false) + expect(missing?.reason).toContain('Base URL is required.') + expect(configured?.valid).toBe(true) + }) + + it('describes Web Speech API streaming support without runtime state', async () => { + const defaults = z.parse(providerBrowserWebSpeechApi.createProviderConfig({ t: translate }), {}) + + expect(defaults).toEqual({ + language: 'en-US', + continuous: true, + interimResults: true, + maxAlternatives: 1, + }) + expect(providerBrowserWebSpeechApi.capabilities?.transcription).toEqual({ + protocol: 'http', + generateOutput: false, + streamOutput: true, + streamInput: true, + }) + expect(await providerBrowserWebSpeechApi.isAvailableBy?.()).toBe(false) + }) + + it('keeps ElevenLabs configuration and model discovery in the definition', async () => { + const defaults = z.parse(providerElevenLabs.createProviderConfig({ t: translate }), { apiKey: 'test' }) + const models = await providerElevenLabs.extraMethods?.listModels?.(defaults, providerElevenLabs.createProvider(defaults)) + + expect(defaults).toMatchObject({ + baseUrl: 'https://unspeech.hyp3r.link/v1/', + voiceSettings: { + similarityBoost: 0.75, + stability: 0.5, + }, + }) + expect(models?.length).toBeGreaterThan(0) + expect(models?.every(model => model.provider === 'elevenlabs')).toBe(true) + }) +}) diff --git a/packages/stage-ui/src/libs/providers/providers/speech-noop/index.ts b/packages/stage-ui/src/libs/providers/providers/speech-noop/index.ts new file mode 100644 index 000000000..7cf7c2506 --- /dev/null +++ b/packages/stage-ui/src/libs/providers/providers/speech-noop/index.ts @@ -0,0 +1,32 @@ +import { z } from 'zod' + +import { defineProvider } from '../registry' + +const speechNoopConfigSchema = z.object({}) + +export const providerSpeechNoop = defineProvider({ + id: 'speech-noop', + name: 'None', + nameLocalize: ({ t }) => t('settings.pages.providers.provider.speech-noop.title'), + description: 'No speech output.', + descriptionLocalize: ({ t }) => t('settings.pages.providers.provider.speech-noop.description'), + tasks: ['text-to-speech', 'tts'], + icon: 'i-solar:volume-cross-bold-duotone', + requiresCredentials: false, + + createProviderConfig: () => speechNoopConfigSchema, + createProvider() { + return { + speech: () => ({ + baseURL: 'http://speech-noop.invalid/v1/', + model: 'noop', + }), + } + }, + + validationRequiredWhen: () => false, + extraMethods: { + listModels: async () => [], + listVoices: async () => [], + }, +}) diff --git a/packages/stage-ui/src/libs/providers/providers/unspeech/index.ts b/packages/stage-ui/src/libs/providers/providers/unspeech/index.ts new file mode 100644 index 000000000..16c17d026 --- /dev/null +++ b/packages/stage-ui/src/libs/providers/providers/unspeech/index.ts @@ -0,0 +1,239 @@ +import type { + ListVoicesOptions, + UnAlibabaCloudOptions, + UnDeepgramOptions, + UnMicrosoftOptions, + UnVolcengineOptions, + VoiceProviderWithExtraOptions, +} from 'unspeech' +import type { ComposerTranslation } from 'vue-i18n' + +import { + createUnAlibabaCloud, + createUnDeepgram, + createUnMicrosoft, + createUnVolcengine, + listVoices, +} from 'unspeech' +import { z } from 'zod' + +import { defineProvider } from '../registry' + +const unspeechConfigSchema = z.object({ + apiKey: z.string(), + baseUrl: z.string().default('https://unspeech.hyp3r.link/v1/'), +}) + +const microsoftSpeechConfigSchema = unspeechConfigSchema.extend({ + region: z.string().optional(), +}) + +const volcengineSpeechConfigSchema = unspeechConfigSchema.extend({ + app: z.object({ appId: z.string() }), +}) + +type UnspeechConfig = z.input +type MicrosoftSpeechConfig = z.input +type VolcengineSpeechConfig = z.input + +function createUnspeechConfigSchema(schema: typeof volcengineSpeechConfigSchema, t: ComposerTranslation): typeof volcengineSpeechConfigSchema +function createUnspeechConfigSchema(schema: typeof microsoftSpeechConfigSchema, t: ComposerTranslation): typeof microsoftSpeechConfigSchema +function createUnspeechConfigSchema(schema: typeof unspeechConfigSchema, t: ComposerTranslation): typeof unspeechConfigSchema +function createUnspeechConfigSchema( + schema: typeof unspeechConfigSchema | typeof microsoftSpeechConfigSchema | typeof volcengineSpeechConfigSchema, + t: ComposerTranslation, +) { + return schema.extend({ + apiKey: schema.shape.apiKey.meta({ + labelLocalized: t('settings.pages.providers.catalog.edit.config.common.fields.field.api-key.label'), + descriptionLocalized: t('settings.pages.providers.catalog.edit.config.common.fields.field.api-key.description'), + placeholderLocalized: t('settings.pages.providers.catalog.edit.config.common.fields.field.api-key.placeholder'), + type: 'password', + }), + baseUrl: schema.shape.baseUrl.meta({ + labelLocalized: t('settings.pages.providers.catalog.edit.config.common.fields.field.base-url.label'), + descriptionLocalized: t('settings.pages.providers.catalog.edit.config.common.fields.field.base-url.description'), + placeholderLocalized: t('settings.pages.providers.catalog.edit.config.common.fields.field.base-url.placeholder'), + }), + }) +} + +function toListVoicesOptions(provider: VoiceProviderWithExtraOptions, options?: T): ListVoicesOptions { + const { fetch: _fetch, ...voiceOptions } = provider.voice(options) + return voiceOptions +} + +function validateUnspeechConfig(config: UnspeechConfig, requireAppId = false) { + const errors: Array<{ error: unknown }> = [] + const apiKey = config.apiKey?.trim() ?? '' + const baseUrl = config.baseUrl?.trim() ?? '' + + if (!apiKey) + errors.push({ error: new Error('API key is required.') }) + if (!baseUrl) + errors.push({ error: new Error('Base URL is required.') }) + + if (baseUrl) { + try { + const url = new URL(baseUrl) + if (!url.host) + errors.push({ error: new Error('Base URL is not absolute. Try to include a scheme (http:// or https://).') }) + else if (!baseUrl.endsWith('/')) + errors.push({ error: new Error('Base URL must end with a trailing slash (/).') }) + } + catch { + errors.push({ error: new Error('Base URL is not absolute. Try to include a scheme (http:// or https://).') }) + } + } + + if (requireAppId) { + const appId = 'app' in config && config.app && typeof config.app === 'object' && 'appId' in config.app + ? String(config.app.appId).trim() + : '' + if (!appId) + errors.push({ error: new Error('App ID is required.') }) + } + + return { + errors, + reason: errors.map(item => (item.error as Error).message).join(', '), + reasonKey: '', + valid: errors.length === 0, + } +} + +function createUnspeechValidators(id: string, requireAppId = false) { + return { + validateConfig: [ + ({ t }: { t: ComposerTranslation }) => ({ + id: `${id}:check-config`, + name: t('settings.pages.providers.catalog.edit.validators.openai-compatible.check-config.title'), + validator: async (config: TConfig) => validateUnspeechConfig(config, requireAppId), + }), + ], + } +} + +export const providerDeepgramTts = defineProvider({ + id: 'deepgram-tts', + name: 'Deepgram', + nameLocalize: ({ t }) => t('settings.pages.providers.provider.deepgram-tts.title'), + description: 'deepgram.com', + descriptionLocalize: ({ t }) => t('settings.pages.providers.provider.deepgram-tts.description'), + tasks: ['text-to-speech'], + icon: 'i-simple-icons:deepgram', + createProviderConfig: ({ t }) => createUnspeechConfigSchema(unspeechConfigSchema, t), + createProvider: config => createUnDeepgram(config.apiKey.trim(), config.baseUrl?.trim() ?? ''), + validationRequiredWhen: config => Boolean(config.apiKey?.trim() && config.baseUrl?.trim()), + validators: createUnspeechValidators('deepgram-tts'), + extraMethods: { + listModels: async () => [ + { id: 'aura-2', name: 'Aura 2', provider: 'deepgram-tts', description: 'Latest generation Aura model', deprecated: false }, + { id: 'aura-1', name: 'Aura 1', provider: 'deepgram-tts', description: 'First generation Aura model', deprecated: false }, + { id: 'aura', name: 'Aura (Legacy)', provider: 'deepgram-tts', description: 'Original Aura model', deprecated: true }, + ], + listVoices: async (config) => { + const provider = createUnDeepgram(config.apiKey.trim(), config.baseUrl?.trim() ?? '') as VoiceProviderWithExtraOptions + const voices = await listVoices(toListVoicesOptions(provider)) + return voices.map(voice => ({ + id: voice.id, + name: voice.name, + provider: 'deepgram-tts', + description: voice.description, + languages: voice.languages, + gender: voice.labels?.gender, + })) + }, + }, +}) + +export const providerMicrosoftSpeech = defineProvider({ + id: 'microsoft-speech', + name: 'Microsoft / Azure Speech', + nameLocalize: ({ t }) => t('settings.pages.providers.provider.microsoft-speech.title'), + description: 'speech.microsoft.com', + descriptionLocalize: ({ t }) => t('settings.pages.providers.provider.microsoft-speech.description'), + tasks: ['text-to-speech'], + iconColor: 'i-lobe-icons:microsoft', + createProviderConfig: ({ t }) => createUnspeechConfigSchema(microsoftSpeechConfigSchema, t), + createProvider: config => createUnMicrosoft(config.apiKey.trim(), config.baseUrl?.trim() ?? ''), + validationRequiredWhen: config => Boolean(config.apiKey?.trim() && config.baseUrl?.trim()), + validators: createUnspeechValidators('microsoft-speech'), + extraMethods: { + listModels: async () => [{ id: 'v1', name: 'v1', provider: 'microsoft-speech', description: '', deprecated: false }], + listVoices: async (config) => { + const provider = createUnMicrosoft(config.apiKey.trim(), config.baseUrl?.trim() ?? '') as VoiceProviderWithExtraOptions + const voices = await listVoices(toListVoicesOptions(provider, { region: config.region ?? '' })) + return voices.map(voice => ({ + id: voice.id, + name: voice.name, + provider: 'microsoft-speech', + previewURL: voice.preview_audio_url, + languages: voice.languages, + gender: voice.labels?.gender, + })) + }, + }, +}) + +export const providerAlibabaCloudModelStudio = defineProvider({ + id: 'alibaba-cloud-model-studio', + name: 'Alibaba Cloud Model Studio', + nameLocalize: ({ t }) => t('settings.pages.providers.provider.alibaba-cloud-model-studio.title'), + description: 'bailian.console.aliyun.com', + descriptionLocalize: ({ t }) => t('settings.pages.providers.provider.alibaba-cloud-model-studio.description'), + tasks: ['text-to-speech'], + iconColor: 'i-lobe-icons:alibabacloud', + createProviderConfig: ({ t }) => createUnspeechConfigSchema(unspeechConfigSchema, t), + createProvider: config => createUnAlibabaCloud(config.apiKey.trim(), config.baseUrl?.trim() ?? ''), + validationRequiredWhen: config => Boolean(config.apiKey?.trim() && config.baseUrl?.trim()), + validators: createUnspeechValidators('alibaba-cloud-model-studio'), + extraMethods: { + listModels: async () => [ + { id: 'cosyvoice-v1', name: 'CosyVoice', provider: 'alibaba-cloud-model-studio', description: '', deprecated: false }, + { id: 'cosyvoice-v2', name: 'CosyVoice (New)', provider: 'alibaba-cloud-model-studio', description: '', deprecated: false }, + ], + listVoices: async (config) => { + const provider = createUnAlibabaCloud(config.apiKey.trim(), config.baseUrl?.trim() ?? '') as VoiceProviderWithExtraOptions + const voices = await listVoices(toListVoicesOptions(provider)) + return voices.map(voice => ({ + id: voice.id, + name: voice.name, + provider: 'alibaba-cloud-model-studio', + compatibleModels: voice.compatible_models, + previewURL: voice.preview_audio_url, + languages: voice.languages, + gender: voice.labels?.gender, + })) + }, + }, +}) + +export const providerVolcengineSpeech = defineProvider({ + id: 'volcengine', + name: 'Volcengine', + nameLocalize: ({ t }) => t('settings.pages.providers.provider.volcengine.title'), + description: 'volcengine.com', + descriptionLocalize: ({ t }) => t('settings.pages.providers.provider.volcengine.description'), + tasks: ['text-to-speech'], + iconColor: 'i-lobe-icons:volcengine', + createProviderConfig: ({ t }) => createUnspeechConfigSchema(volcengineSpeechConfigSchema, t), + createProvider: config => createUnVolcengine(config.apiKey.trim(), config.baseUrl?.trim() ?? ''), + validationRequiredWhen: config => Boolean(config.apiKey?.trim() && config.baseUrl?.trim() && config.app?.appId.trim()), + validators: createUnspeechValidators('volcengine', true), + extraMethods: { + listModels: async () => [{ id: 'v1', name: 'v1', provider: 'volcano-engine', description: '', deprecated: false }], + listVoices: async (config) => { + const provider = createUnVolcengine(config.apiKey.trim(), config.baseUrl?.trim() ?? '') as VoiceProviderWithExtraOptions + const voices = await listVoices(toListVoicesOptions(provider)) + return voices.map(voice => ({ + id: voice.id, + name: voice.name, + provider: 'volcano-engine', + previewURL: voice.preview_audio_url, + languages: voice.languages, + gender: voice.labels?.gender, + })) + }, + }, +}) diff --git a/packages/stage-ui/src/stores/providers.ts b/packages/stage-ui/src/stores/providers.ts index 046215f07..cca81c7ce 100644 --- a/packages/stage-ui/src/stores/providers.ts +++ b/packages/stage-ui/src/stores/providers.ts @@ -9,79 +9,24 @@ import type { TranscriptionProviderWithExtraOptions, } from '@xsai-ext/providers/utils' import type { ProgressInfo } from '@xsai-transformers/shared/types' -import type { - ListVoicesOptions, - UnAlibabaCloudOptions, - UnDeepgramOptions, - UnElevenLabsOptions, - UnMicrosoftOptions, - UnVolcengineOptions, - VoiceProviderWithExtraOptions, -} from 'unspeech' import type { ProviderSourceDeployment, ProviderSourcePricing } from '../libs/providers/source-metadata' import type { ProviderOnboardingField } from '../libs/providers/types' -import type { AliyunRealtimeSpeechExtraOptions } from './providers/aliyun/stream-transcription' import { errorMessageFrom } from '@moeru/std' -import { isCustomProvidersDisabled, isStageCapacitor, isStageTamagotchi, isUrl } from '@proj-airi/stage-shared' -import { getCachedWebGPUCapabilities, isWebGPUSupported } from '@proj-airi/stage-shared/webgpu' +import { isCustomProvidersDisabled, isStageCapacitor, isStageTamagotchi } from '@proj-airi/stage-shared' import { computedAsync, useIntervalFn, useLocalStorage } from '@vueuse/core' -import { - createOpenAI, -} from '@xsai-ext/providers/create' -import { createPlayer2 } from '@xsai-ext/providers/special/create' -import { - createModelProvider, - createSpeechProvider, - createTranscriptionProvider, - merge, -} from '@xsai-ext/providers/utils' -import { listModels } from '@xsai/model' import { uniqBy } from 'es-toolkit' import { defineStore } from 'pinia' -import { - createUnAlibabaCloud, - createUnDeepgram, - createUnElevenLabs, - createUnMicrosoft, - createUnVolcengine, - listVoices, -} from 'unspeech' import { computed, ref, watch } from 'vue' import { useI18n } from 'vue-i18n' -import { getKokoroAdapter } from '../libs/inference/adapters/kokoro' -import { getProviderValidationIntervalMs, listProviders as listDefinedProviders, ProviderValidationCheck } from '../libs/providers' -import { resolveProviderSourceMetadata } from '../libs/providers/source-metadata' -import { getDefaultKokoroModel, KOKORO_MODELS, kokoroModelsToModelInfo } from '../workers/kokoro/constants' +import { getProviderValidationIntervalMs, listProviders as listDefinedProviders } from '../libs/providers' import { captureAnalyticsEvent, ensureAnalyticsInitialized, isAnalyticsAvailableInBuild } from './analytics/client' import { useAuthStore } from './auth' -import { createAliyunNLSProvider as createAliyunNlsStreamProvider } from './providers/aliyun/stream-transcription' import { convertProviderDefinitionsToMetadata } from './providers/converters' -import { models as elevenLabsModels } from './providers/elevenlabs/list-models' -import { buildGoogleGeminiSpeechProvider } from './providers/google-gemini-speech' -import { buildOpenAICompatibleProvider } from './providers/openai-compatible-builder' -import { buildOpenRouterAudioSpeechProvider } from './providers/openrouter/audio-speech' -import { createWebSpeechAPIProvider } from './providers/web-speech-api' import { useSettingsAnalytics } from './settings/analytics' -const ALIYUN_NLS_REGIONS = [ - 'cn-shanghai', - 'cn-shanghai-internal', - 'cn-beijing', - 'cn-beijing-internal', - 'cn-shenzhen', - 'cn-shenzhen-internal', -] as const - -type AliyunNlsRegion = typeof ALIYUN_NLS_REGIONS[number] - -function toListVoicesOptions(provider: VoiceProviderWithExtraOptions, options?: T): ListVoicesOptions { - const { fetch: _fetch, ...voiceOptions } = provider.voice(options) - return voiceOptions -} - /** * Classifies provider ids into bounded analytics buckets. */ @@ -303,2020 +248,8 @@ export const useProvidersStore = defineStore('providers', () => { const addedProviders = useLocalStorage>('settings/providers/added', {}) const providerInstanceCache = ref>({}) const { t } = useI18n() - const baseUrlValidator = computed(() => (baseUrl: unknown) => { - let msg = '' - if (!baseUrl) { - msg = 'Base URL is required.' - } - else if (typeof baseUrl !== 'string') { - msg = 'Base URL must be a string.' - } - else if (!isUrl(baseUrl) || new URL(baseUrl).host.length === 0) { - msg = 'Base URL is not absolute. Try to include a scheme (http:// or https://).' - } - else if (!baseUrl.endsWith('/')) { - msg = 'Base URL must end with a trailing slash (/).' - } - if (msg) { - return { - errors: [new Error(msg)], - reason: msg, - valid: false, - } - } - return null - }) - async function isBrowserAndMemoryEnough() { - if (isStageTamagotchi()) - return false - - const webGPUAvailable = await isWebGPUSupported() - if (webGPUAvailable) { - return true - } - - if ('navigator' in globalThis && globalThis.navigator != null && 'deviceMemory' in globalThis.navigator && typeof globalThis.navigator.deviceMemory === 'number') { - const memory = globalThis.navigator.deviceMemory - // Check if the device has at least 8GB of RAM - if (memory >= 8) { - return true - } - } - - return false - } - - // Centralized provider metadata with provider factory functions const authState = useAuthStore() - const providerMetadata: Record = { - 'speech-noop': { - id: 'speech-noop', - category: 'speech', - tasks: ['text-to-speech', 'tts'], - nameKey: 'settings.pages.providers.provider.speech-noop.title', - name: 'None', - descriptionKey: 'settings.pages.providers.provider.speech-noop.description', - description: 'No speech output.', - icon: 'i-solar:volume-cross-bold-duotone', - requiresCredentials: false, - defaultOptions: () => ({}), - createProvider: async () => ({ - speech: () => ({ - baseURL: 'http://speech-noop.invalid/v1/', - model: 'noop', - }), - }), - capabilities: { - listModels: async () => [], - listVoices: async () => [], - }, - validators: { - chatPingCheckAvailable: false, - validateProviderConfig: () => ({ - errors: [], - reason: '', - valid: true, - }), - }, - }, - 'app-local-audio-speech': buildOpenAICompatibleProvider({ - id: 'app-local-audio-speech', - name: 'App (Local)', - nameKey: 'settings.pages.providers.provider.app-local-audio-speech.title', - descriptionKey: 'settings.pages.providers.provider.app-local-audio-speech.description', - icon: 'i-lobe-icons:huggingface', - description: 'https://github.com/huggingface/candle', - category: 'speech', - tasks: ['text-to-speech', 'tts'], - isAvailableBy: isStageTamagotchi, - creator: createOpenAI, - validation: [], - validators: { - chatPingCheckAvailable: false, - validateProviderConfig: (config) => { - if (!config.baseUrl) { - return { - errors: [new Error('Base URL is required.')], - reason: 'Base URL is required. This is likely a bug, report to developers on https://github.com/moeru-ai/airi/issues.', - valid: false, - } - } - - return { - errors: [], - reason: '', - valid: true, - } - }, - }, - }), - 'app-local-audio-transcription': buildOpenAICompatibleProvider({ - id: 'app-local-audio-transcription', - name: 'App (Local)', - nameKey: 'settings.pages.providers.provider.app-local-audio-transcription.title', - descriptionKey: 'settings.pages.providers.provider.app-local-audio-transcription.description', - icon: 'i-lobe-icons:huggingface', - description: 'https://github.com/huggingface/candle', - category: 'transcription', - tasks: ['speech-to-text', 'automatic-speech-recognition', 'asr', 'stt'], - isAvailableBy: isStageTamagotchi, - creator: createOpenAI, - validation: [], - validators: { - chatPingCheckAvailable: false, - validateProviderConfig: (config) => { - if (!config.baseUrl) { - return { - errors: [new Error('Base URL is required.')], - reason: 'Base URL is required. This is likely a bug, report to developers on https://github.com/moeru-ai/airi/issues.', - valid: false, - } - } - - return { - errors: [], - reason: '', - valid: true, - } - }, - }, - }), - 'browser-local-audio-speech': buildOpenAICompatibleProvider({ - id: 'browser-local-audio-speech', - name: 'Browser (Local)', - nameKey: 'settings.pages.providers.provider.browser-local-audio-speech.title', - descriptionKey: 'settings.pages.providers.provider.browser-local-audio-speech.description', - icon: 'i-lobe-icons:huggingface', - description: 'https://github.com/moeru-ai/xsai-transformers', - category: 'speech', - tasks: ['text-to-speech', 'tts'], - isAvailableBy: isBrowserAndMemoryEnough, - creator: createOpenAI, - validation: [], - validators: { - chatPingCheckAvailable: false, - validateProviderConfig: (config) => { - if (!config.baseUrl) { - return { - errors: [new Error('Base URL is required.')], - reason: 'Base URL is required. This is likely a bug, report to developers on https://github.com/moeru-ai/airi/issues.', - valid: false, - } - } - - return { - errors: [], - reason: '', - valid: true, - } - }, - }, - }), - 'browser-local-audio-transcription': buildOpenAICompatibleProvider({ - id: 'browser-local-audio-transcription', - name: 'Browser (Local)', - nameKey: 'settings.pages.providers.provider.browser-local-audio-transcription.title', - descriptionKey: 'settings.pages.providers.provider.browser-local-audio-transcription.description', - icon: 'i-lobe-icons:huggingface', - description: 'https://github.com/moeru-ai/xsai-transformers', - category: 'transcription', - tasks: ['speech-to-text', 'automatic-speech-recognition', 'asr', 'stt'], - isAvailableBy: isBrowserAndMemoryEnough, - creator: createOpenAI, - validation: [], - validators: { - chatPingCheckAvailable: false, - validateProviderConfig: (config) => { - if (!config.baseUrl) { - return { - errors: [new Error('Base URL is required.')], - reason: 'Base URL is required. This is likely a bug, report to developers on https://github.com/moeru-ai/airi/issues.', - valid: false, - } - } - - return { - errors: [], - reason: '', - valid: true, - } - }, - }, - }), - 'openai-audio-speech': buildOpenAICompatibleProvider({ - id: 'openai-audio-speech', - name: 'OpenAI', - nameKey: 'settings.pages.providers.provider.openai.title', - descriptionKey: 'settings.pages.providers.provider.openai.description', - icon: 'i-lobe-icons:openai', - description: 'openai.com', - category: 'speech', - tasks: ['text-to-speech'], - defaultBaseUrl: 'https://api.openai.com/v1/', - creator: createOpenAI, - validation: [ProviderValidationCheck.Health], - capabilities: { - // NOTE: OpenAI does not provide an API endpoint to retrieve available voices. - // Voices are hardcoded here - this is a provider limitation, not an application limitation. - // Voice compatibility per https://platform.openai.com/docs/api-reference/audio/createSpeech: - // - tts-1 and tts-1-hd support: alloy, ash, coral, echo, fable, onyx, nova, sage, shimmer (9 voices) - // - gpt-4o-mini-tts supports all 13 voices: alloy, ash, ballad, coral, echo, fable, nova, onyx, sage, shimmer, verse, marin, cedar - listVoices: async (_config: Record) => { - return [ - { - id: 'alloy', - name: 'Alloy', - provider: 'openai-audio-speech', - languages: [], - compatibleModels: ['tts-1', 'tts-1-hd', 'gpt-4o-mini-tts', 'gpt-4o-mini-tts-2025-12-15'], - }, - { - id: 'ash', - name: 'Ash', - provider: 'openai-audio-speech', - languages: [], - compatibleModels: ['tts-1', 'tts-1-hd', 'gpt-4o-mini-tts', 'gpt-4o-mini-tts-2025-12-15'], - }, - { - id: 'ballad', - name: 'Ballad', - provider: 'openai-audio-speech', - languages: [], - compatibleModels: ['gpt-4o-mini-tts', 'gpt-4o-mini-tts-2025-12-15'], - }, - { - id: 'coral', - name: 'Coral', - provider: 'openai-audio-speech', - languages: [], - compatibleModels: ['tts-1', 'tts-1-hd', 'gpt-4o-mini-tts', 'gpt-4o-mini-tts-2025-12-15'], - }, - { - id: 'echo', - name: 'Echo', - provider: 'openai-audio-speech', - languages: [], - compatibleModels: ['tts-1', 'tts-1-hd', 'gpt-4o-mini-tts', 'gpt-4o-mini-tts-2025-12-15'], - }, - { - id: 'fable', - name: 'Fable', - provider: 'openai-audio-speech', - languages: [], - compatibleModels: ['tts-1', 'tts-1-hd', 'gpt-4o-mini-tts', 'gpt-4o-mini-tts-2025-12-15'], - }, - { - id: 'onyx', - name: 'Onyx', - provider: 'openai-audio-speech', - languages: [], - compatibleModels: ['tts-1', 'tts-1-hd', 'gpt-4o-mini-tts', 'gpt-4o-mini-tts-2025-12-15'], - }, - { - id: 'nova', - name: 'Nova', - provider: 'openai-audio-speech', - languages: [], - compatibleModels: ['tts-1', 'tts-1-hd', 'gpt-4o-mini-tts', 'gpt-4o-mini-tts-2025-12-15'], - }, - { - id: 'sage', - name: 'Sage', - provider: 'openai-audio-speech', - languages: [], - compatibleModels: ['tts-1', 'tts-1-hd', 'gpt-4o-mini-tts', 'gpt-4o-mini-tts-2025-12-15'], - }, - { - id: 'shimmer', - name: 'Shimmer', - provider: 'openai-audio-speech', - languages: [], - compatibleModels: ['tts-1', 'tts-1-hd', 'gpt-4o-mini-tts', 'gpt-4o-mini-tts-2025-12-15'], - }, - { - id: 'verse', - name: 'Verse', - provider: 'openai-audio-speech', - languages: [], - compatibleModels: ['gpt-4o-mini-tts', 'gpt-4o-mini-tts-2025-12-15'], - }, - { - id: 'marin', - name: 'Marin', - provider: 'openai-audio-speech', - languages: [], - compatibleModels: ['gpt-4o-mini-tts', 'gpt-4o-mini-tts-2025-12-15'], - }, - { - id: 'cedar', - name: 'Cedar', - provider: 'openai-audio-speech', - languages: [], - compatibleModels: ['gpt-4o-mini-tts', 'gpt-4o-mini-tts-2025-12-15'], - }, - ] satisfies VoiceInfo[] - }, - listModels: async () => { - // TESTING NOTES: All 4 models tested and confirmed working with fable voice: - // - tts-1: {model: "tts-1", input: "test", voice: "fable"} ✓ - // - tts-1-hd: {model: "tts-1-hd", input: "test", voice: "fable"} ✓ - // - gpt-4o-mini-tts: {model: "gpt-4o-mini-tts", input: "test", voice: "fable"} ✓ - // - gpt-4o-mini-tts-2025-12-15: {model: "gpt-4o-mini-tts-2025-12-15", input: "test", voice: "fable"} ✓ - return [ - { - id: 'tts-1', - name: 'TTS-1', - provider: 'openai-audio-speech', - description: 'Optimized for real-time text-to-speech tasks', - contextLength: 0, - deprecated: false, - }, - { - id: 'tts-1-hd', - name: 'TTS-1-HD', - provider: 'openai-audio-speech', - description: 'Higher fidelity audio output', - contextLength: 0, - deprecated: false, - }, - { - id: 'gpt-4o-mini-tts', - name: 'GPT-4o Mini TTS', - provider: 'openai-audio-speech', - description: 'GPT-4o Mini optimized for text-to-speech', - contextLength: 0, - deprecated: false, - }, - { - id: 'gpt-4o-mini-tts-2025-12-15', - name: 'GPT-4o Mini TTS (2025-12-15)', - provider: 'openai-audio-speech', - description: 'GPT-4o Mini TTS snapshot from 2025-12-15', - contextLength: 0, - deprecated: false, - }, - ] - }, - }, - validators: { - chatPingCheckAvailable: false, - validateProviderConfig: (config) => { - const errors = [ - !config.apiKey && new Error('API Key is required'), - !config.baseUrl && new Error('Base URL is required. Default to https://api.openai.com/v1/ for official OpenAI API.'), - ].filter(Boolean) - - const res = baseUrlValidator.value(config.baseUrl) - if (res) { - return res - } - - return { - errors, - reason: errors.filter(e => e).map(e => String(e)).join(', ') || '', - valid: !!config.apiKey && !!config.baseUrl, - } - }, - }, - }), - 'openai-compatible-audio-speech': buildOpenAICompatibleProvider({ - id: 'openai-compatible-audio-speech', - name: 'OpenAI Compatible', - nameKey: 'settings.pages.providers.provider.openai-compatible.title', - descriptionKey: 'settings.pages.providers.provider.openai-compatible.description', - icon: 'i-lobe-icons:openai', - description: 'Connect to any API that follows the OpenAI specification.', - category: 'speech', - tasks: ['text-to-speech'], - capabilities: { - listVoices: async () => { - return [] - }, - listModels: async (config: Record) => { - // Filter models to only include TTS models - const apiKey = typeof config.apiKey === 'string' ? config.apiKey.trim() : '' - let baseUrl = typeof config.baseUrl === 'string' ? config.baseUrl.trim() : '' - - if (!baseUrl.endsWith('/')) - baseUrl += '/' - - if (!apiKey || !baseUrl) { - return [] - } - - const provider = await createOpenAI(apiKey, baseUrl) - if (!provider || typeof provider.model !== 'function') { - return [] - } - - const models = await listModels({ - apiKey, - baseURL: baseUrl, - }) - - // Filter for TTS models - look for models with "tts" in the ID - return models - .filter((model: any) => { - const modelId = model.id.toLowerCase() - // Include models that contain "tts" in their ID - return modelId.includes('tts') - }) - .map((model: any) => { - return { - id: model.id, - name: model.name || model.display_name || model.id, - provider: 'openai-compatible-audio-speech', - description: model.description || '', - contextLength: model.context_length || 0, - deprecated: false, - } satisfies ModelInfo - }) - }, - }, - creator: createOpenAI, - }), - 'openai-audio-transcription': buildOpenAICompatibleProvider({ - id: 'openai-audio-transcription', - name: 'OpenAI', - nameKey: 'settings.pages.providers.provider.openai.title', - descriptionKey: 'settings.pages.providers.provider.openai.description', - icon: 'i-lobe-icons:openai', - description: 'openai.com', - category: 'transcription', - tasks: ['speech-to-text', 'automatic-speech-recognition', 'asr', 'stt'], - defaultBaseUrl: 'https://api.openai.com/v1/', - creator: createOpenAI, - validation: [ProviderValidationCheck.Health], - capabilities: { - listModels: async () => { - // OpenAI transcription models are hardcoded (no API endpoint to list them) - return [ - { - id: 'gpt-4o-transcribe', - name: 'GPT-4o Transcribe', - provider: 'openai-audio-transcription', - description: 'High-quality transcription model', - contextLength: 0, - deprecated: false, - }, - { - id: 'gpt-4o-mini-transcribe', - name: 'GPT-4o Mini Transcribe', - provider: 'openai-audio-transcription', - description: 'Faster, cost-effective transcription model', - contextLength: 0, - deprecated: false, - }, - { - id: 'gpt-4o-mini-transcribe-2025-12-15', - name: 'GPT-4o Mini Transcribe (2025-12-15)', - provider: 'openai-audio-transcription', - description: 'GPT-4o Mini Transcribe snapshot from 2025-12-15', - contextLength: 0, - deprecated: false, - }, - { - id: 'whisper-1', - name: 'Whisper-1', - provider: 'openai-audio-transcription', - description: 'Powered by our open source Whisper V2 model', - contextLength: 0, - deprecated: false, - }, - { - id: 'gpt-4o-transcribe-diarize', - name: 'GPT-4o Transcribe Diarize', - provider: 'openai-audio-transcription', - description: 'Transcription with speaker diarization', - contextLength: 0, - deprecated: false, - }, - ] satisfies ModelInfo[] - }, - }, - validators: { - chatPingCheckAvailable: false, - validateProviderConfig: (config) => { - const errors = [ - !config.apiKey && new Error('API Key is required'), - !config.baseUrl && new Error('Base URL is required. Default to https://api.openai.com/v1/ for official OpenAI API.'), - ].filter(Boolean) - - const res = baseUrlValidator.value(config.baseUrl) - if (res) { - return res - } - - return { - errors, - reason: errors.filter(e => e).map(e => String(e)).join(', ') || '', - valid: !!config.apiKey && !!config.baseUrl, - } - }, - }, - }), - 'openai-compatible-audio-transcription': buildOpenAICompatibleProvider({ - id: 'openai-compatible-audio-transcription', - name: 'OpenAI Compatible', - nameKey: 'settings.pages.providers.provider.openai-compatible.title', - descriptionKey: 'settings.pages.providers.provider.openai-compatible.description', - icon: 'i-lobe-icons:openai', - description: 'Connect to any API that follows the OpenAI specification.', - category: 'transcription', - tasks: ['speech-to-text', 'automatic-speech-recognition', 'asr', 'stt'], - creator: createOpenAI, - capabilities: { - // Override listModels to return empty array - transcription models cannot be fetched from /v1/models - // Users must manually enter transcription model names (e.g., whisper-1, gpt-4o-transcribe) - // The /v1/models endpoint only returns chat models, not transcription models - listModels: async () => { - return [] - }, - }, - }), - 'aliyun-nls-transcription': { - id: 'aliyun-nls-transcription', - category: 'transcription', - tasks: ['speech-to-text', 'automatic-speech-recognition', 'asr', 'stt', 'streaming-transcription'], - nameKey: 'settings.pages.providers.provider.aliyun-nls.title', - name: 'Aliyun NLS', - descriptionKey: 'settings.pages.providers.provider.aliyun-nls.description', - description: 'nls-console.aliyun.com', - icon: 'i-lobe-icons:alibabacloud', - defaultOptions: () => ({ - accessKeyId: '', - accessKeySecret: '', - appKey: '', - region: 'cn-shanghai', - }), - transcriptionFeatures: { - supportsGenerate: false, - supportsStreamOutput: true, - supportsStreamInput: true, - }, - createProvider: async (config) => { - const toString = (value: unknown) => typeof value === 'string' ? value.trim() : '' - - const accessKeyId = toString(config.accessKeyId) - const accessKeySecret = toString(config.accessKeySecret) - const appKey = toString(config.appKey) - const region = toString(config.region) - const resolvedRegion = ALIYUN_NLS_REGIONS.includes(region as AliyunNlsRegion) ? region as AliyunNlsRegion : 'cn-shanghai' - - if (!accessKeyId || !accessKeySecret || !appKey) - throw new Error('Aliyun NLS credentials are incomplete.') - - const provider = createAliyunNlsStreamProvider(accessKeyId, accessKeySecret, appKey, { region: resolvedRegion }) - - return { - transcription: (model: string, extraOptions?: AliyunRealtimeSpeechExtraOptions) => provider.speech(model, { - ...extraOptions, - sessionOptions: { - format: 'pcm', - sample_rate: 16000, - enable_punctuation_prediction: true, - enable_intermediate_result: true, - enable_words: true, - ...extraOptions?.sessionOptions, - }, - }), - } as TranscriptionProviderWithExtraOptions - }, - capabilities: { - listModels: async () => { - return [ - { - id: 'aliyun-nls-v1', - name: 'Aliyun NLS Realtime', - provider: 'aliyun-nls-transcription', - description: 'Realtime streaming transcription using Aliyun NLS.', - contextLength: 0, - deprecated: false, - }, - ] - }, - }, - validators: { - chatPingCheckAvailable: false, - validateProviderConfig: (config) => { - const errors: Error[] = [] - const toString = (value: unknown) => typeof value === 'string' ? value.trim() : '' - - const accessKeyId = toString(config.accessKeyId) - const accessKeySecret = toString(config.accessKeySecret) - const appKey = toString(config.appKey) - const region = toString(config.region) - - if (!accessKeyId) - errors.push(new Error('Access Key ID is required.')) - if (!accessKeySecret) - errors.push(new Error('Access Key Secret is required.')) - if (!appKey) - errors.push(new Error('App Key is required.')) - if (region && !ALIYUN_NLS_REGIONS.includes(region as AliyunNlsRegion)) - errors.push(new Error('Region is invalid.')) - - return { - errors, - reason: errors.length > 0 ? errors.map(error => error.message).join(', ') : '', - valid: errors.length === 0, - } - }, - }, - }, - 'browser-web-speech-api': { - id: 'browser-web-speech-api', - category: 'transcription', - tasks: ['speech-to-text', 'automatic-speech-recognition', 'asr', 'stt', 'streaming-transcription'], - nameKey: 'settings.pages.providers.provider.browser-web-speech-api.title', - name: 'Web Speech API (Browser)', - descriptionKey: 'settings.pages.providers.provider.browser-web-speech-api.description', - description: 'Browser-native speech recognition. No API keys.', - icon: 'i-solar:microphone-bold-duotone', - requiresCredentials: false, - defaultOptions: () => ({ - language: 'en-US', - continuous: true, - interimResults: true, - maxAlternatives: 1, - }), - transcriptionFeatures: { - supportsGenerate: false, - supportsStreamOutput: true, - supportsStreamInput: true, - }, - isAvailableBy: async () => { - // Web Speech API is only available in browser contexts, NOT in Electron - // Even though Electron uses Chromium, Web Speech API requires Google's embedded API keys - // which are not available in Electron, causing it to fail at runtime - if (typeof window === 'undefined') - return false - - // Explicitly exclude Electron - Web Speech API doesn't work there - if (isStageTamagotchi()) - return false - - // Check if API is available in browser - return 'webkitSpeechRecognition' in window || 'SpeechRecognition' in window - }, - createProvider: async (_config) => { - // Web Speech API doesn't need config, but we accept it for consistency - return createWebSpeechAPIProvider() - }, - capabilities: { - listModels: async () => { - return [ - { - id: 'web-speech-api', - name: 'Web Speech API', - provider: 'browser-web-speech-api', - description: 'Browser-native speech recognition (no API keys required)', - contextLength: 0, - deprecated: false, - }, - ] - }, - }, - validators: { - chatPingCheckAvailable: false, - validateProviderConfig: () => { - // Web Speech API requires no configuration, just browser support - // Always return valid if browser supports it, so it auto-configures - const isAvailable = typeof window !== 'undefined' - && ('webkitSpeechRecognition' in window || 'SpeechRecognition' in window) - - if (!isAvailable) { - return { - errors: [new Error('Web Speech API is not available. It requires a browser context with SpeechRecognition support (Chrome, Edge, Safari).')], - reason: 'Web Speech API is not available in this environment.', - valid: false, - } - } - - // Auto-configure if available (no credentials needed) - return { - errors: [], - reason: '', - valid: true, - } - }, - }, - }, - 'elevenlabs': { - id: 'elevenlabs', - category: 'speech', - tasks: ['text-to-speech'], - nameKey: 'settings.pages.providers.provider.elevenlabs.title', - name: 'ElevenLabs', - descriptionKey: 'settings.pages.providers.provider.elevenlabs.description', - description: 'elevenlabs.io', - icon: 'i-simple-icons:elevenlabs', - defaultOptions: () => ({ - baseUrl: 'https://unspeech.hyp3r.link/v1/', - voiceSettings: { - similarityBoost: 0.75, - stability: 0.5, - }, - }), - createProvider: async config => createUnElevenLabs((config.apiKey as string).trim(), (config.baseUrl as string).trim()) as SpeechProviderWithExtraOptions, - capabilities: { - listModels: async () => { - return elevenLabsModels.map((model) => { - return { - id: model.model_id, - name: model.name, - provider: 'elevenlabs', - description: model.description, - contextLength: 0, - deprecated: false, - } satisfies ModelInfo - }) - }, - listVoices: async (config) => { - const provider = createUnElevenLabs((config.apiKey as string).trim(), (config.baseUrl as string).trim()) as VoiceProviderWithExtraOptions - - const voices = await listVoices(toListVoicesOptions(provider)) - - if (!voices || !Array.isArray(voices)) { - return [] - } - - // Find indices of Aria and Bill - const ariaIndex = voices.findIndex(voice => voice.name.includes('Aria')) - const billIndex = voices.findIndex(voice => voice.name.includes('Bill')) - - // Determine the range to move (ensure valid indices and proper order) - const startIndex = ariaIndex !== -1 ? ariaIndex : 0 - const endIndex = billIndex !== -1 ? billIndex : voices.length - 1 - const lowerIndex = Math.min(startIndex, endIndex) - const higherIndex = Math.max(startIndex, endIndex) - - // Rearrange voices: voices outside the range first, then voices within the range - const rearrangedVoices = [ - ...voices.slice(0, lowerIndex), - ...voices.slice(higherIndex + 1), - ...voices.slice(lowerIndex, higherIndex + 1), - ] - - return rearrangedVoices.map((voice) => { - return { - id: voice.id, - name: voice.name, - provider: 'elevenlabs', - previewURL: voice.preview_audio_url, - languages: voice.languages, - } - }) - }, - }, - validators: { - chatPingCheckAvailable: false, - validateProviderConfig: (config) => { - const errors = [ - !config.apiKey && new Error('API key is required.'), - !config.baseUrl && new Error('Base URL is required.'), - ].filter(Boolean) - - const res = baseUrlValidator.value(config.baseUrl) - if (res) { - return res - } - - return { - errors, - reason: errors.filter(e => e).map(e => String(e)).join(', ') || '', - valid: !!config.apiKey && !!config.baseUrl, - } - }, - }, - }, - 'deepgram-tts': { - id: 'deepgram-tts', - category: 'speech', - tasks: ['text-to-speech'], - nameKey: 'settings.pages.providers.provider.deepgram-tts.title', - name: 'Deepgram', - descriptionKey: 'settings.pages.providers.provider.deepgram-tts.description', - description: 'deepgram.com', - icon: 'i-simple-icons:deepgram', - defaultOptions: () => ({ - baseUrl: 'https://unspeech.hyp3r.link/v1/', - }), - createProvider: async (config) => { - const provider = createUnDeepgram((config.apiKey as string).trim(), (config.baseUrl as string).trim()) as SpeechProviderWithExtraOptions - return provider - }, - capabilities: { - listModels: async () => { - return [ - { - id: 'aura-2', - name: 'Aura 2', - provider: 'deepgram-tts', - description: 'Latest generation Aura model', - contextLength: 0, - deprecated: false, - }, - { - id: 'aura-1', - name: 'Aura 1', - provider: 'deepgram-tts', - description: 'First generation Aura model', - contextLength: 0, - deprecated: false, - }, - { - id: 'aura', - name: 'Aura (Legacy)', - provider: 'deepgram-tts', - description: 'Original Aura model', - contextLength: 0, - deprecated: true, - }, - ] - }, - listVoices: async (config) => { - const provider = createUnDeepgram((config.apiKey as string).trim(), (config.baseUrl as string).trim()) as VoiceProviderWithExtraOptions - - const voices = await listVoices(toListVoicesOptions(provider)) - - return voices.map((voice) => { - return { - id: voice.id, - name: voice.name, - provider: 'deepgram-tts', - description: voice.description, - languages: voice.languages, - gender: voice.labels?.gender, - } - }) - }, - }, - validators: { - chatPingCheckAvailable: false, - validateProviderConfig: (config) => { - const errors: Error[] = [] - if (!config.apiKey) { - errors.push(new Error('API key is required.')) - } - - const baseUrlValidationResult = baseUrlValidator.value(config.baseUrl) - if (baseUrlValidationResult) { - errors.push(...(baseUrlValidationResult.errors as Error[])) - } - - return { - errors, - reason: errors.map(e => e.message).join(', '), - valid: errors.length === 0, - } - }, - }, - }, - 'microsoft-speech': { - id: 'microsoft-speech', - category: 'speech', - tasks: ['text-to-speech'], - nameKey: 'settings.pages.providers.provider.microsoft-speech.title', - name: 'Microsoft / Azure Speech', - descriptionKey: 'settings.pages.providers.provider.microsoft-speech.description', - description: 'speech.microsoft.com', - iconColor: 'i-lobe-icons:microsoft', - defaultOptions: () => ({ - baseUrl: 'https://unspeech.hyp3r.link/v1/', - }), - createProvider: async config => createUnMicrosoft((config.apiKey as string).trim(), (config.baseUrl as string).trim()) as SpeechProviderWithExtraOptions, - capabilities: { - listModels: async () => { - return [ - { - id: 'v1', - name: 'v1', - provider: 'microsoft-speech', - description: '', - contextLength: 0, - deprecated: false, - }, - ] - }, - listVoices: async (config) => { - const provider = createUnMicrosoft((config.apiKey as string).trim(), (config.baseUrl as string).trim()) as VoiceProviderWithExtraOptions - - const voices = await listVoices(toListVoicesOptions(provider, { region: config.region as string })) - - return voices.map((voice) => { - return { - id: voice.id, - name: voice.name, - provider: 'microsoft-speech', - previewURL: voice.preview_audio_url, - languages: voice.languages, - gender: voice.labels?.gender, - } - }) - }, - }, - validators: { - chatPingCheckAvailable: false, - validateProviderConfig: (config) => { - const errors = [ - !config.apiKey && new Error('API key is required.'), - !config.baseUrl && new Error('Base URL is required.'), - ].filter(Boolean) - - const res = baseUrlValidator.value(config.baseUrl) - if (res) { - return res - } - - return { - errors, - reason: errors.filter(e => e).map(e => String(e)).join(', ') || '', - valid: !!config.apiKey && !!config.baseUrl, - } - }, - }, - }, - 'index-tts-vllm': { - id: 'index-tts-vllm', - category: 'speech', - tasks: ['text-to-speech'], - nameKey: 'settings.pages.providers.provider.index-tts-vllm.title', - name: 'Index-TTS by Bilibili', - descriptionKey: 'settings.pages.providers.provider.index-tts-vllm.description', - description: 'index-tts.github.io', - iconColor: 'i-lobe-icons:bilibiliindex', - defaultOptions: () => ({ - baseUrl: 'http://localhost:11996/tts/', - model: 'IndexTTS-1.5', - }), - createProvider: async (config) => { - const provider: SpeechProvider = { - speech: () => { - const req = { - baseURL: config.baseUrl as string, - model: (config.model as string) || 'IndexTTS-1.5', - } - return req - }, - } - return provider - }, - capabilities: { - listModels: async () => { - return [ - { - id: 'IndexTTS-1.5', - name: 'IndexTTS-1.5', - provider: 'index-tts-vllm', - description: 'Default model for Index-TTS vLLM deployment', - contextLength: 0, - deprecated: false, - }, - ] - }, - listVoices: async (config) => { - const voicesUrl = config.baseUrl as string - const response = await fetch(`${voicesUrl}audio/voices`) - if (!response.ok) { - throw new Error(`Failed to fetch voices: ${response.statusText}`) - } - const voices = await response.json() - return Object.keys(voices).map((voice: any) => { - return { - id: voice, - name: voice, - provider: 'index-tts-vllm', - // previewURL: voice.preview_audio_url, - languages: [{ code: 'cn', title: 'Chinese' }, { code: 'en', title: 'English' }], - } - }) - }, - }, - validators: { - chatPingCheckAvailable: false, - validateProviderConfig: async (config) => { - const errors = [ - !config.baseUrl && new Error('Base URL is required. Default to http://localhost:11996/tts/ for Index-TTS.'), - ].filter(Boolean) - - const res = baseUrlValidator.value(config.baseUrl) - if (res) { - return res - } - - try { - const controller = new AbortController() - const timeout = setTimeout(() => controller.abort(), 5000) - const response = await fetch(`${config.baseUrl as string}audio/voices`, { signal: controller.signal }) - clearTimeout(timeout) - - if (!response.ok) { - const reason = `IndexTTS unreachable: HTTP ${response.status} ${response.statusText}` - return { errors: [new Error(reason)], reason, valid: false } - } - } - catch (err) { - const reason = `IndexTTS connection failed: ${String(err)}` - return { errors: [err as Error], reason, valid: false } - } - - return { - errors, - reason: errors.filter(e => e).map(e => String(e)).join(', ') || '', - valid: errors.length === 0, - } - }, - }, - }, - 'alibaba-cloud-model-studio': { - id: 'alibaba-cloud-model-studio', - category: 'speech', - tasks: ['text-to-speech'], - nameKey: 'settings.pages.providers.provider.alibaba-cloud-model-studio.title', - name: 'Alibaba Cloud Model Studio', - descriptionKey: 'settings.pages.providers.provider.alibaba-cloud-model-studio.description', - description: 'bailian.console.aliyun.com', - iconColor: 'i-lobe-icons:alibabacloud', - defaultOptions: () => ({ - baseUrl: 'https://unspeech.hyp3r.link/v1/', - }), - createProvider: async config => createUnAlibabaCloud((config.apiKey as string).trim(), (config.baseUrl as string).trim()), - capabilities: { - listVoices: async (config) => { - const provider = createUnAlibabaCloud((config.apiKey as string).trim(), (config.baseUrl as string).trim()) as VoiceProviderWithExtraOptions - - const voices = await listVoices(toListVoicesOptions(provider)) - - return voices.map((voice) => { - return { - id: voice.id, - name: voice.name, - provider: 'alibaba-cloud-model-studio', - compatibleModels: voice.compatible_models, - previewURL: voice.preview_audio_url, - languages: voice.languages, - gender: voice.labels?.gender, - } - }) - }, - listModels: async () => { - return [ - { - id: 'cosyvoice-v1', - name: 'CosyVoice', - provider: 'alibaba-cloud-model-studio', - description: '', - contextLength: 0, - deprecated: false, - }, - { - id: 'cosyvoice-v2', - name: 'CosyVoice (New)', - provider: 'alibaba-cloud-model-studio', - description: '', - contextLength: 0, - deprecated: false, - }, - ] - }, - }, - validators: { - chatPingCheckAvailable: false, - validateProviderConfig: (config) => { - const errors = [ - !config.apiKey && new Error('API key is required.'), - !config.baseUrl && new Error('Base URL is required.'), - ].filter(Boolean) - - const res = baseUrlValidator.value(config.baseUrl) - if (res) { - return res - } - - return { - errors, - reason: errors.filter(e => e).map(e => String(e)).join(', ') || '', - valid: !!config.apiKey && !!config.baseUrl, - } - }, - }, - }, - 'volcengine': { - id: 'volcengine', - category: 'speech', - tasks: ['text-to-speech'], - nameKey: 'settings.pages.providers.provider.volcengine.title', - name: 'settings.pages.providers.provider.volcengine.title', - descriptionKey: 'settings.pages.providers.provider.volcengine.description', - description: 'volcengine.com', - iconColor: 'i-lobe-icons:volcengine', - defaultOptions: () => ({ - baseUrl: 'https://unspeech.hyp3r.link/v1/', - }), - createProvider: async config => createUnVolcengine((config.apiKey as string).trim(), (config.baseUrl as string).trim()), - capabilities: { - listVoices: async (config) => { - const provider = createUnVolcengine((config.apiKey as string).trim(), (config.baseUrl as string).trim()) as VoiceProviderWithExtraOptions - - const voices = await listVoices(toListVoicesOptions(provider)) - - return voices.map((voice) => { - return { - id: voice.id, - name: voice.name, - provider: 'volcano-engine', - previewURL: voice.preview_audio_url, - languages: voice.languages, - gender: voice.labels?.gender, - } - }) - }, - listModels: async () => { - return [ - { - id: 'v1', - name: 'v1', - provider: 'volcano-engine', - description: '', - contextLength: 0, - deprecated: false, - }, - ] - }, - }, - validators: { - chatPingCheckAvailable: false, - validateProviderConfig: (config) => { - const errors = [ - !config.apiKey && new Error('API key is required.'), - !config.baseUrl && new Error('Base URL is required.'), - !((config.app as any)?.appId) && new Error('App ID is required.'), - ].filter(Boolean) - - const res = baseUrlValidator.value(config.baseUrl) - if (res) { - return res - } - - return { - errors, - reason: errors.filter(e => e).map(e => String(e)).join(', ') || '', - valid: !!config.apiKey && !!config.baseUrl && !!config.app && !!(config.app as any).appId, - } - }, - }, - }, - 'minimax-speech': { - id: 'minimax-speech', - category: 'speech', - tasks: ['text-to-speech'], - nameKey: 'settings.pages.providers.provider.minimax-speech.title', - name: 'MiniMax Speech', - descriptionKey: 'settings.pages.providers.provider.minimax-speech.description', - description: 'minimax.io', - icon: 'i-lobe-icons:minimax', - iconColor: 'i-lobe-icons:minimax-color', - defaultOptions: () => ({ - apiKey: '', - baseUrl: 'https://api.minimax.io', - }), - createProvider: async (config) => { - const apiKey = (config.apiKey as string).trim() - const baseUrl = ((config.baseUrl as string) || 'https://api.minimax.io').replace(/\/$/, '') - - const provider: SpeechProvider = { - speech: () => ({ - baseURL: `${baseUrl}/v1/`, - model: 'speech-2.8-hd', - fetch: async (_input: RequestInfo | URL, init?: RequestInit) => { - if (!init?.body || typeof init.body !== 'string') { - throw new Error('Invalid request body') - } - - const body = JSON.parse(init.body) - const text = body.input as string - const voiceId = (body.voice as string) || 'English_Graceful_Lady' - const model = (body.model as string) || 'speech-2.8-hd' - - const response = await fetch(`${baseUrl}/v1/t2a_v2`, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'Authorization': `Bearer ${apiKey}`, - }, - body: JSON.stringify({ - model, - text, - stream: true, - voice_setting: { - voice_id: voiceId, - speed: 1, - vol: 1, - pitch: 0, - }, - audio_setting: { - sample_rate: 32000, - bitrate: 128000, - format: 'mp3', - channel: 1, - }, - }), - }) - - if (!response.ok || !response.body) { - throw new Error(`MiniMax TTS request failed: ${response.status} ${response.statusText}`) - } - - // Parse SSE stream and collect hex-encoded audio chunks - const reader = response.body.getReader() - const decoder = new TextDecoder() - const audioChunks: Uint8Array[] = [] - let buffer = '' - - while (true) { - const { done, value } = await reader.read() - if (done) - break - buffer += decoder.decode(value, { stream: true }) - const lines = buffer.split('\n') - buffer = lines.pop() || '' - for (const line of lines) { - if (!line.startsWith('data:')) - continue - const jsonStr = line.slice(5).trim() - if (!jsonStr || jsonStr === '[DONE]') - continue - try { - const eventData = JSON.parse(jsonStr) - const audio = eventData?.data?.audio - // status 2 is the final summary chunk; skip it to avoid duplication - if (audio && eventData?.data?.status !== 2) { - const hexStr = audio as string - const bytes = new Uint8Array(hexStr.length / 2) - for (let i = 0; i < hexStr.length; i += 2) { - bytes[i / 2] = Number.parseInt(hexStr.slice(i, i + 2), 16) - } - audioChunks.push(bytes) - } - } - catch { - // ignore malformed SSE events - } - } - } - - const totalLength = audioChunks.reduce((sum, chunk) => sum + chunk.length, 0) - const combined = new Uint8Array(totalLength) - let offset = 0 - for (const chunk of audioChunks) { - combined.set(chunk, offset) - offset += chunk.length - } - - return new Response(combined.buffer, { - status: 200, - headers: { 'Content-Type': 'audio/mpeg' }, - }) - }, - }), - } - return provider - }, - capabilities: { - listModels: async () => [ - { - id: 'speech-2.8-hd', - name: 'Speech 2.8 HD', - provider: 'minimax-speech', - description: 'High-definition TTS model with natural prosody', - contextLength: 0, - deprecated: false, - }, - { - id: 'speech-2.8-turbo', - name: 'Speech 2.8 Turbo', - provider: 'minimax-speech', - description: 'Fast TTS model for low-latency scenarios', - contextLength: 0, - deprecated: false, - }, - ], - listVoices: async () => [ - { id: 'English_Graceful_Lady', name: 'Graceful Lady', provider: 'minimax-speech', gender: 'female', languages: [{ code: 'en', title: 'English' }] }, - { id: 'English_Insightful_Speaker', name: 'Insightful Speaker', provider: 'minimax-speech', gender: 'male', languages: [{ code: 'en', title: 'English' }] }, - { id: 'English_radiant_girl', name: 'Radiant Girl', provider: 'minimax-speech', gender: 'female', languages: [{ code: 'en', title: 'English' }] }, - { id: 'English_Persuasive_Man', name: 'Persuasive Man', provider: 'minimax-speech', gender: 'male', languages: [{ code: 'en', title: 'English' }] }, - { id: 'English_Lucky_Robot', name: 'Lucky Robot', provider: 'minimax-speech', gender: 'neutral', languages: [{ code: 'en', title: 'English' }] }, - { id: 'English_expressive_narrator', name: 'Expressive Narrator', provider: 'minimax-speech', gender: 'neutral', languages: [{ code: 'en', title: 'English' }] }, - { id: 'Mandarin_Gentle_Woman', name: 'Gentle Woman', provider: 'minimax-speech', gender: 'female', languages: [{ code: 'zh', title: 'Chinese' }] }, - { id: 'Mandarin_Steadfast_Man', name: 'Steadfast Man', provider: 'minimax-speech', gender: 'male', languages: [{ code: 'zh', title: 'Chinese' }] }, - { id: 'Mandarin_Sweet_Girl', name: 'Sweet Girl', provider: 'minimax-speech', gender: 'female', languages: [{ code: 'zh', title: 'Chinese' }] }, - { id: 'Mandarin_Magnetic_Gentleman', name: 'Magnetic Gentleman', provider: 'minimax-speech', gender: 'male', languages: [{ code: 'zh', title: 'Chinese' }] }, - ], - }, - validators: { - chatPingCheckAvailable: false, - validateProviderConfig: (config) => { - const errors = [ - !config.apiKey && new Error('API key is required.'), - ].filter(Boolean) - - return { - errors, - reason: errors.filter(e => e).map(e => String(e)).join(', ') || '', - valid: !!config.apiKey, - } - }, - }, - }, - 'openrouter-audio-speech': buildOpenRouterAudioSpeechProvider(v => baseUrlValidator.value(v)), - 'mimo-audio-speech': { - id: 'mimo-audio-speech', - category: 'speech', - tasks: ['text-to-speech'], - nameKey: 'settings.pages.providers.provider.mimo.title', - name: 'Xiaomi MiMo', - descriptionKey: 'settings.pages.providers.provider.mimo.description', - description: 'api.xiaomimimo.com', - icon: 'i-simple-icons:xiaomi', - defaultOptions: () => ({ - baseUrl: 'https://api.xiaomimimo.com/v1/', - model: 'mimo-v2.5-tts', - voice: 'mimo_default', - format: 'wav', - }), - createProvider: async (config) => { - const apiKey = (config.apiKey as string)?.trim() ?? '' - const baseUrl = ((config.baseUrl as string) || 'https://api.xiaomimimo.com/v1/').replace(/\/+$/, '') - const defaultModel = (config.model as string) || 'mimo-v2.5-tts' - const defaultVoice = (config.voice as string) || 'mimo_default' - const defaultFormat = (config.format as string) || 'wav' - - const provider: SpeechProvider = { - speech: () => ({ - baseURL: `${baseUrl}/`, - model: defaultModel, - fetch: async (_input: RequestInfo | URL, init?: RequestInit) => { - if (!init?.body || typeof init.body !== 'string') { - throw new Error('Invalid request body') - } - - const body = JSON.parse(init.body) - const text = body.input as string - const modelId = (body.model as string) || defaultModel - const format = (body.response_format as string) || defaultFormat - const stylePrompt = typeof body.style_prompt === 'string' - ? body.style_prompt.trim() - : typeof config.stylePrompt === 'string' - ? config.stylePrompt.trim() - : '' - const voiceSample = typeof body.voice_sample === 'string' - ? body.voice_sample.trim() - : typeof config.voiceSample === 'string' - ? config.voiceSample.trim() - : '' - - const userPrompt = modelId === 'mimo-v2.5-tts-voiceclone' - ? stylePrompt - : stylePrompt || 'Use a natural, clear speaking style.' - - const audio: Record = { format } - if (modelId === 'mimo-v2.5-tts-voiceclone') { - if (!voiceSample) { - throw new Error('MiMo voice clone requires a base64 audio sample in data URI format.') - } - audio.voice = voiceSample - } - else if (modelId === 'mimo-v2.5-tts') { - audio.voice = (body.voice as string) || defaultVoice - } - - if (modelId === 'mimo-v2.5-tts-voicedesign' && !stylePrompt) { - throw new Error('MiMo voice design requires a style prompt in the user message.') - } - - const response = await fetch(`${baseUrl}/chat/completions`, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'api-key': apiKey, - }, - body: JSON.stringify({ - model: modelId, - messages: [ - { role: 'user', content: userPrompt }, - { role: 'assistant', content: text }, - ], - audio, - }), - }) - - if (!response.ok || !response.body) { - throw new Error(`MiMo TTS request failed: ${response.status} ${response.statusText}`) - } - - const data = await response.json() - const audioBase64 = data?.choices?.[0]?.message?.audio?.data - if (!audioBase64) { - throw new Error('MiMo TTS response missing audio data') - } - - const binaryString = atob(audioBase64) - const bytes = new Uint8Array(binaryString.length) - for (let i = 0; i < binaryString.length; i++) { - bytes[i] = binaryString.charCodeAt(i) - } - - const contentType = format === 'wav' ? 'audio/wav' : format === 'mp3' ? 'audio/mpeg' : `audio/${format}` - return new Response(bytes.buffer, { - status: 200, - headers: { 'Content-Type': contentType }, - }) - }, - }), - } - return provider - }, - capabilities: { - listModels: async () => [ - { - id: 'mimo-v2.5-tts', - name: 'MiMo v2.5 TTS', - provider: 'mimo-audio-speech', - description: 'Preset voice synthesis with the built-in MiMo voice list', - contextLength: 0, - deprecated: false, - }, - { - id: 'mimo-v2.5-tts-voicedesign', - name: 'MiMo v2.5 TTS Voice Design', - provider: 'mimo-audio-speech', - description: 'Design a new voice from a natural language description', - contextLength: 0, - deprecated: false, - }, - { - id: 'mimo-v2.5-tts-voiceclone', - name: 'MiMo v2.5 TTS Voice Clone', - provider: 'mimo-audio-speech', - description: 'Clone a voice from a base64-encoded audio sample', - contextLength: 0, - deprecated: false, - }, - ], - listVoices: async () => [ - { id: 'mimo_default', name: 'MiMo-默认', provider: 'mimo-audio-speech', gender: 'female', languages: [{ code: 'en', title: 'English' }, { code: 'zh', title: 'Chinese' }] }, - { id: '冰糖', name: '冰糖', provider: 'mimo-audio-speech', gender: 'female', languages: [{ code: 'zh', title: 'Chinese' }] }, - { id: '茉莉', name: '茉莉', provider: 'mimo-audio-speech', gender: 'female', languages: [{ code: 'zh', title: 'Chinese' }] }, - { id: '苏打', name: '苏打', provider: 'mimo-audio-speech', gender: 'male', languages: [{ code: 'zh', title: 'Chinese' }] }, - { id: '白桦', name: '白桦', provider: 'mimo-audio-speech', gender: 'male', languages: [{ code: 'zh', title: 'Chinese' }] }, - { id: 'Mia', name: 'Mia', provider: 'mimo-audio-speech', gender: 'female', languages: [{ code: 'en', title: 'English' }] }, - { id: 'Chloe', name: 'Chloe', provider: 'mimo-audio-speech', gender: 'female', languages: [{ code: 'en', title: 'English' }] }, - { id: 'Milo', name: 'Milo', provider: 'mimo-audio-speech', gender: 'male', languages: [{ code: 'en', title: 'English' }] }, - { id: 'Dean', name: 'Dean', provider: 'mimo-audio-speech', gender: 'male', languages: [{ code: 'en', title: 'English' }] }, - ], - }, - validators: { - chatPingCheckAvailable: false, - validateProviderConfig: (config) => { - const errors = [ - !config.apiKey && new Error('API key is required.'), - !config.baseUrl && new Error('Base URL is required.'), - ].filter(Boolean) - - const res = baseUrlValidator.value(config.baseUrl) - if (res) { - return res - } - - return { - errors, - reason: errors.map(e => (e as Error).message).join(', ') || '', - valid: !!config.apiKey && !!config.baseUrl, - } - }, - }, - }, - 'comet-api-speech': buildOpenAICompatibleProvider({ - id: 'comet-api-speech', - name: 'CometAPI Speech', - nameKey: 'settings.pages.providers.provider.comet-api.title', - descriptionKey: 'settings.pages.providers.provider.comet-api.description', - icon: 'i-lobe-icons:cometapi', - description: 'cometapi.com', - category: 'speech', - tasks: ['text-to-speech'], - defaultBaseUrl: 'https://api.cometapi.com/v1/', - creator: (apiKey, baseURL = 'https://api.cometapi.com/v1/') => merge( - createModelProvider({ apiKey, baseURL }), - createSpeechProvider({ apiKey, baseURL }), - ), - validation: [ProviderValidationCheck.ModelList], - }), - 'comet-api-transcription': buildOpenAICompatibleProvider({ - id: 'comet-api-transcription', - name: 'CometAPI Transcription', - nameKey: 'settings.pages.providers.provider.comet-api.title', - descriptionKey: 'settings.pages.providers.provider.comet-api.description', - icon: 'i-lobe-icons:cometapi', - description: 'cometapi.com', - category: 'transcription', - tasks: ['speech-to-text', 'automatic-speech-recognition', 'asr', 'stt'], - defaultBaseUrl: 'https://api.cometapi.com/v1/', - creator: (apiKey, baseURL = 'https://api.cometapi.com/v1/') => merge( - createModelProvider({ apiKey, baseURL }), - createTranscriptionProvider({ apiKey, baseURL }), - ), - validation: [ProviderValidationCheck.ModelList], - }), - 'mimo-audio-transcription': { - id: 'mimo-audio-transcription', - category: 'transcription', - tasks: ['speech-to-text', 'automatic-speech-recognition', 'asr', 'stt'], - nameKey: 'settings.pages.providers.provider.mimo.title', - name: 'Xiaomi MiMo', - descriptionKey: 'settings.pages.providers.provider.mimo.description', - description: 'api.xiaomimimo.com', - icon: 'i-simple-icons:xiaomi', - defaultOptions: () => ({ - baseUrl: 'https://api.xiaomimimo.com/v1/', - model: 'mimo-v2-omni', - }), - createProvider: async (config) => { - const apiKey = (config.apiKey as string)?.trim() ?? '' - const rawBaseUrl = `${((config.baseUrl as string) || 'https://api.xiaomimimo.com/v1/').replace(/\/+$/, '')}/` - const defaultModel = (config.model as string) || 'mimo-v2-omni' - - const provider: TranscriptionProvider = { - transcription: model => ({ - baseURL: rawBaseUrl, - model: model || defaultModel, - headers: {}, - fetch: async (_input: RequestInfo | URL, init?: RequestInit) => { - const formData = init?.body as FormData - const file = formData?.get('file') as Blob | null - const modelName = (formData?.get('model') as string) || defaultModel - - if (!file) { - throw new Error('No audio file provided for transcription.') - } - - // Read the file as base64 data URI (works with both Blob and File) - const base64DataUri: string = await new Promise((resolve, reject) => { - const reader = new FileReader() - reader.onload = () => resolve(reader.result as string) - reader.onerror = () => reject(new Error('Failed to read audio file')) - reader.readAsDataURL(file) - }) - - // Extract format and base64 data from data URI - // data:audio/wav;base64,UklGR... - const mimeType = base64DataUri.split(';')[0]?.split(':')[1] || 'audio/wav' - const formatFromMime = mimeType.split('/')[1] || 'wav' - const base64Data = base64DataUri.split(',')[1] - - // Map MIME sub-type to MiMo supported audio format - const audioFormat = formatFromMime === 'webm' - ? 'webm' - : formatFromMime === 'mp4' - ? 'mp4' - : formatFromMime === 'mpeg' || formatFromMime === 'mp3' - ? 'mp3' - : 'wav' - - // MiMo audio understanding uses chat completions with input_audio, - // not a dedicated transcription endpoint - const response = await fetch(`${rawBaseUrl}chat/completions`, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'api-key': apiKey, - }, - body: JSON.stringify({ - model: modelName, - messages: [ - { - role: 'user', - content: [ - { type: 'text', text: 'Transcribe the audio content.' }, - { - type: 'input_audio', - input_audio: { - data: base64Data, - format: audioFormat, - }, - }, - ], - }, - ], - }), - }) - - if (!response.ok) { - const errorBody = await response.text().catch(() => '') - throw new Error( - `MiMo transcription failed: ${response.status} ${response.statusText}${errorBody ? ` — ${errorBody}` : ''}`, - ) - } - - const data = await response.json() - const text = data?.choices?.[0]?.message?.content || '' - - // Return in OpenAI transcription response format { text: "..." } - return new Response(JSON.stringify({ text }), { - status: 200, - headers: { 'Content-Type': 'application/json' }, - }) - }, - }), - } - - return provider - }, - capabilities: { - listModels: async () => [ - { - id: 'mimo-v2-omni', - name: 'MiMo V2 Omni', - provider: 'mimo-audio-transcription', - description: 'Omni-modal model with native audio understanding and speech-to-text', - contextLength: 256000, - deprecated: false, - }, - { - id: 'mimo-v2.5', - name: 'MiMo V2.5', - provider: 'mimo-audio-transcription', - description: 'Latest omni-modal model with audio understanding, 1M context', - contextLength: 1_000_000, - deprecated: false, - }, - ], - }, - transcriptionFeatures: { - supportsGenerate: true, - supportsStreamOutput: false, - supportsStreamInput: false, - }, - validators: { - chatPingCheckAvailable: false, - validateProviderConfig: (config) => { - const errors = [ - !config.apiKey && new Error('API key is required.'), - !config.baseUrl && new Error('Base URL is required.'), - ].filter(Boolean) - - const res = baseUrlValidator.value(config.baseUrl) - if (res) { - return res - } - - return { - errors, - reason: errors.map(e => (e as Error).message).join(', ') || '', - valid: !!config.apiKey && !!config.baseUrl, - } - }, - }, - }, - 'player2-speech': { - id: 'player2-speech', - category: 'speech', - tasks: ['text-to-speech'], - nameKey: 'settings.pages.providers.provider.player2.title', - name: 'Player2 Speech', - descriptionKey: 'settings.pages.providers.provider.player2.description', - description: 'player2.game', - icon: 'i-lobe-icons:player2', - defaultOptions: () => ({ - baseUrl: 'http://localhost:4315/v1/', - }), - createProvider: async config => createPlayer2((config.baseUrl as string).trim(), 'airi'), - capabilities: { - listModels: async () => { - return [ - { - id: 'player2-tts', - name: 'Player2 Speech', - provider: 'player2-speech', - description: 'Default model for Player2 speech endpoint', - contextLength: 0, - deprecated: false, - }, - ] - }, - listVoices: async (config) => { - const baseUrl = (config.baseUrl as string).endsWith('/') ? (config.baseUrl as string).slice(0, -1) : config.baseUrl as string - return await fetch(`${baseUrl}/tts/voices`).then(res => res.json()).then(({ voices }) => (voices as { id: string, language: 'american_english' | 'british_english' | 'japanese' | 'mandarin_chinese' | 'spanish' | 'french' | 'hindi' | 'italian' | 'brazilian_portuguese', name: string, gender: string }[]).map(({ id, language, name, gender }) => ( - { - - id, - name, - provider: 'player2-speech', - gender, - languages: [{ - american_english: { - code: 'en', - title: 'English', - }, - british_english: { - code: 'en', - title: 'English', - }, - japanese: { - code: 'ja', - title: 'Japanese', - }, - mandarin_chinese: { - code: 'zh', - title: 'Chinese', - }, - spanish: { - code: 'es', - title: 'Spanish', - }, - french: { - code: 'fr', - title: 'French', - }, - hindi: { - code: 'hi', - title: 'Hindi', - }, - - italian: { - code: 'it', - title: 'Italian', - }, - brazilian_portuguese: - { - code: 'pt', - title: 'Portuguese', - }, - - }[language]], - } - ))) - }, - }, - validators: { - chatPingCheckAvailable: false, - validateProviderConfig: async (config) => { - const errors = [ - !config.baseUrl && new Error('Base URL is required. Default to http://localhost:4315/v1/'), - ].filter(Boolean) - - const res = baseUrlValidator.value(config.baseUrl) - if (res) - return res - - try { - const controller = new AbortController() - const timeout = setTimeout(() => controller.abort(), 5000) - const response = await fetch(`${config.baseUrl as string}health`, { - method: 'GET', - headers: { - 'player2-game-key': 'airi', - }, - signal: controller.signal, - }) - clearTimeout(timeout) - - if (!response.ok) { - const reason = `Player2 speech unreachable: HTTP ${response.status} ${response.statusText}` - return { errors: [new Error(reason)], reason, valid: false } - } - } - catch (err) { - const reason = `Player2 speech connection failed: ${String(err)}` - return { errors: [err as Error], reason, valid: false } - } - - return { - errors, - reason: errors.filter(e => e).map(e => String(e)).join(', ') || '', - valid: errors.length === 0, - } - }, - }, - }, - 'kokoro-local': { - id: 'kokoro-local', - category: 'speech', - tasks: ['text-to-speech'], - nameKey: 'settings.pages.providers.provider.kokoro-local.title', - name: 'Kokoro TTS', - descriptionKey: 'settings.pages.providers.provider.kokoro-local.description', - description: 'Local text-to-speech using Kokoro-82M.', - icon: 'i-lobe-icons:speaker', - requiresCredentials: false, - - defaultOptions: () => { - const capabilities = getCachedWebGPUCapabilities() - const hasWebGPU = capabilities?.supported ?? (typeof navigator !== 'undefined' && !!navigator.gpu) - const fp16Supported = capabilities?.fp16Supported ?? false - const model = getDefaultKokoroModel(hasWebGPU, fp16Supported) - return { - model, - voiceId: '', - } - }, - - createProvider: async (_config) => { - // Import the worker manager - const workerManagerPromise = getKokoroAdapter() - - const provider: SpeechProvider = { - speech: () => { - return { - baseURL: 'http://kokoro-local/v1/', - model: 'kokoro-82m', - fetch: async (_input: RequestInfo | URL, init?: RequestInit) => { - try { - // Parse OpenAI-compatible request body - if (!init?.body || typeof init.body !== 'string') { - throw new Error('Invalid request body') - } - const body = JSON.parse(init.body) - const text = body.input - const voice = body.voice - - if (!voice) { - throw new Error('Voice parameter is required') - } - - // Generate audio in the worker thread - const buffer = await (await workerManagerPromise).generate(text, voice) - - return new Response(buffer, { - status: 200, - headers: { - 'Content-Type': 'audio/wav', - }, - }) - } - catch (error) { - console.error('Kokoro TTS generation failed:', error) - throw error - } - }, - } - }, - } - - return provider - }, - - capabilities: { - listModels: async (_config: Record) => { - const caps = getCachedWebGPUCapabilities() - const hasWebGPU = caps?.supported ?? (typeof navigator !== 'undefined' && !!navigator.gpu) - const fp16Supported = caps?.fp16Supported ?? false - return kokoroModelsToModelInfo(hasWebGPU, t, fp16Supported) - }, - - loadModel: async (config: Record, _hooks?: { onProgress?: (progress: ProgressInfo) => Promise | void }) => { - const modelId = config.model as string - - if (!modelId) { - throw new Error('No model specified') - } - - const modelDef = KOKORO_MODELS.find(m => m.id === modelId) - if (!modelDef) { - throw new Error(`Invalid model: ${modelId}. Must be one of: ${KOKORO_MODELS.map(m => m.id).join(', ')}`) - } - - // Validate platform requirements - if (modelDef.platform === 'webgpu') { - const hasWebGPU = getCachedWebGPUCapabilities()?.supported ?? (typeof navigator !== 'undefined' && !!navigator.gpu) - if (!hasWebGPU) { - throw new Error('WebGPU is required for this model but is not available in your browser') - } - } - - try { - const workerManager = await getKokoroAdapter() - await workerManager.loadModel(modelDef.quantization, modelDef.platform, { - onProgress: _hooks?.onProgress - ? (p) => { - // Map unified ProgressPayload back to ProgressInfo shape - // that the provider hooks expect (HuggingFace transformers format) - _hooks.onProgress!({ - name: p.file ?? '', - file: p.file ?? '', - progress: p.percent >= 0 ? p.percent : 0, - status: 'progress', - loaded: p.loaded ?? 0, - total: p.total ?? 0, - } as ProgressInfo) - } - : undefined, - }) - } - catch (error) { - console.error('Failed to load Kokoro model:', error) - throw error - } - }, - - listVoices: (() => { - let lastLoadedModelId: string | null = null - return async (config: Record) => { - try { - const workerManager = await getKokoroAdapter() - const modelId = config.model as string - - // Reload the model if it hasn't been loaded yet or if the model ID changed - if (workerManager.state !== 'ready' || (modelId && modelId !== lastLoadedModelId)) { - if (modelId) { - const modelDef = KOKORO_MODELS.find(m => m.id === modelId) - if (modelDef) { - if (modelDef.platform === 'webgpu') { - const hasWebGPU = getCachedWebGPUCapabilities()?.supported ?? (typeof navigator !== 'undefined' && !!navigator.gpu) - if (!hasWebGPU) { - throw new Error('WebGPU is required for this model but is not available in your browser') - } - } - - await workerManager.loadModel(modelDef.quantization, modelDef.platform) - lastLoadedModelId = modelId - } - } - } - - const modelVoices = workerManager.getVoices() - - // Language code mapping - const languageMap: Record = { - 'en-us': { code: 'en-US', title: 'English (US)' }, - 'en-gb': { code: 'en-GB', title: 'English (UK)' }, - 'ja': { code: 'ja', title: 'Japanese' }, - 'zh-cn': { code: 'zh-CN', title: 'Chinese (Mandarin)' }, - 'es': { code: 'es', title: 'Spanish' }, - 'fr': { code: 'fr', title: 'French' }, - 'hi': { code: 'hi', title: 'Hindi' }, - 'it': { code: 'it', title: 'Italian' }, - 'pt-br': { code: 'pt-BR', title: 'Portuguese (Brazil)' }, - } - - // Transform the voices object to the expected array format - return Object.entries(modelVoices).map(([id, voice]: [string, { language: string, name: string, gender: string }]) => { - const languageCode = voice.language.toLowerCase() - const languageInfo = languageMap[languageCode] || { code: languageCode, title: voice.language } - - return { - id, - name: `${voice.name} (${voice.gender}, ${languageInfo.title.split('(')[0].trim()})`, - provider: 'kokoro-local', - languages: [languageInfo], - gender: voice.gender.toLowerCase(), - } - }) - } - catch (error) { - console.error('Failed to fetch Kokoro voices:', error) - // Return empty array if model not loaded yet - return [] - } - } - })(), - }, - - validators: { - chatPingCheckAvailable: false, - validateProviderConfig: async (config: any) => { - const model = config.model as string - - if (!model) { - return { - errors: [new Error('No model selected')], - reason: 'Please select a model from the dropdown menu', - valid: false, - } - } - - if (!KOKORO_MODELS.some(m => m.id === model)) { - return { - errors: [new Error(`Invalid model: ${model}`)], - reason: `Invalid model. Must be one of: ${KOKORO_MODELS.map(m => m.id).join(', ')}`, - valid: false, - } - } - - return { - errors: [], - reason: '', - valid: true, - } - }, - }, - }, - 'google-gemini-audio-speech': buildGoogleGeminiSpeechProvider(v => baseUrlValidator.value(v)), - } - const VISION_PROVIDER_ID_PREFIX = 'vision-' function createVisionProviderMetadata(metadata: ProviderMetadata): ProviderMetadata { @@ -2329,16 +262,11 @@ export const useProvidersStore = defineStore('providers', () => { } } - // Progressive migration bridge: - // translate unified provider definitions from libs/providers to legacy store metadata. - // Existing metadata remains as fallback for providers not yet migrated. const definedProviders = listDefinedProviders() - const definedProviderIds = new Set(definedProviders.map(d => d.id)) - - const translatedProviderMetadata = convertProviderDefinitionsToMetadata( + const providerMetadata = convertProviderDefinitionsToMetadata( definedProviders, t, - providerMetadata, + {}, ) const providerValidationIntervalMsById = new Map() @@ -2353,28 +281,12 @@ export const useProvidersStore = defineStore('providers', () => { } } - // Merge unified registry definitions into providerMetadata. - // Unified defineProvider() entries always take precedence over legacy hand-written - // metadata. Legacy entries are kept only as fallback for providers not yet migrated - // to defineProvider(). - // TODO: progressively migrate legacy speech/transcription providers to defineProvider() - // and remove the hand-written metadata above entirely. - for (const [providerId, translated] of Object.entries(translatedProviderMetadata)) { - providerMetadata[providerId] = translated - } - for (const metadata of Object.values(providerMetadata) .filter(metadata => metadata.category === 'chat') .map(createVisionProviderMetadata)) { providerMetadata[metadata.id] = metadata } - for (const metadata of Object.values(providerMetadata)) { - if (definedProviderIds.has(metadata.id)) - continue - Object.assign(metadata, resolveProviderSourceMetadata(metadata)) - } - // const validatedCredentials = ref>({}) const providerRuntimeState = ref>({}) const providerValidationInFlight = new Map>() @@ -2784,8 +696,7 @@ export const useProvidersStore = defineStore('providers', () => { return getProviderMetadata(providerId) } - // Get all providers metadata (for settings page). - // Order: defined providers first (already sorted by order in registry), then legacy-only providers. + // Get all provider metadata in registry order for the settings page. const allProvidersMetadata = computed(() => { const localize = (metadata: ProviderMetadata) => ({ ...metadata, @@ -2794,15 +705,9 @@ export const useProvidersStore = defineStore('providers', () => { configured: providerRuntimeState.value[metadata.id]?.isConfigured || false, }) - const ordered = definedProviders + return definedProviders .filter(d => providerMetadata[d.id]) .map(d => localize(providerMetadata[d.id])) - - const legacy = Object.values(providerMetadata) - .filter(m => !definedProviderIds.has(m.id)) - .map(localize) - - return [...ordered, ...legacy] }) function getTranscriptionFeatures(providerId: string) {