mirror of
https://github.com/moeru-ai/airi.git
synced 2026-08-14 00:48:06 +00:00
refactor(providers): now simplified
This commit is contained in:
Neko Ayaka
@@ -0,0 +1,107 @@
|
||||
import type { TranscriptionProviderWithExtraOptions } from '@xsai-ext/providers/utils'
|
||||
|
||||
import type { AliyunRealtimeSpeechExtraOptions } from '../../../../stores/providers/aliyun/stream-transcription'
|
||||
|
||||
import { z } from 'zod'
|
||||
|
||||
import { createAliyunNLSProvider } from '../../../../stores/providers/aliyun/stream-transcription'
|
||||
import { defineProvider } from '../registry'
|
||||
|
||||
const aliyunNlsRegions = [
|
||||
'cn-shanghai',
|
||||
'cn-shanghai-internal',
|
||||
'cn-beijing',
|
||||
'cn-beijing-internal',
|
||||
'cn-shenzhen',
|
||||
'cn-shenzhen-internal',
|
||||
] as const
|
||||
|
||||
const aliyunNlsConfigSchema = z.object({
|
||||
accessKeyId: z.string(),
|
||||
accessKeySecret: z.string(),
|
||||
appKey: z.string(),
|
||||
region: z.enum(aliyunNlsRegions).default('cn-shanghai'),
|
||||
})
|
||||
|
||||
type AliyunNlsConfig = z.input<typeof aliyunNlsConfigSchema>
|
||||
|
||||
export const providerAliyunNlsTranscription = defineProvider<AliyunNlsConfig>({
|
||||
id: 'aliyun-nls-transcription',
|
||||
name: 'Aliyun NLS',
|
||||
nameLocalize: ({ t }) => t('settings.pages.providers.provider.aliyun-nls.title'),
|
||||
description: 'nls-console.aliyun.com',
|
||||
descriptionLocalize: ({ t }) => t('settings.pages.providers.provider.aliyun-nls.description'),
|
||||
tasks: ['speech-to-text', 'automatic-speech-recognition', 'asr', 'stt', 'streaming-transcription'],
|
||||
icon: 'i-lobe-icons:alibabacloud',
|
||||
capabilities: {
|
||||
transcription: {
|
||||
protocol: 'websocket',
|
||||
generateOutput: false,
|
||||
streamOutput: true,
|
||||
streamInput: true,
|
||||
},
|
||||
},
|
||||
createProviderConfig: () => aliyunNlsConfigSchema,
|
||||
createProvider(config) {
|
||||
const accessKeyId = config.accessKeyId.trim()
|
||||
const accessKeySecret = config.accessKeySecret.trim()
|
||||
const appKey = config.appKey.trim()
|
||||
if (!accessKeyId || !accessKeySecret || !appKey)
|
||||
throw new Error('Aliyun NLS credentials are incomplete.')
|
||||
|
||||
const provider = createAliyunNLSProvider(accessKeyId, accessKeySecret, appKey, {
|
||||
region: config.region ?? 'cn-shanghai',
|
||||
})
|
||||
|
||||
return {
|
||||
transcription: (model: string, extraOptions?: AliyunRealtimeSpeechExtraOptions) => provider.speech(model, {
|
||||
...extraOptions,
|
||||
sessionOptions: {
|
||||
format: 'pcm',
|
||||
sample_rate: 16000,
|
||||
enable_punctuation_prediction: true,
|
||||
enable_intermediate_result: true,
|
||||
enable_words: true,
|
||||
...extraOptions?.sessionOptions,
|
||||
},
|
||||
}),
|
||||
} as TranscriptionProviderWithExtraOptions<string, AliyunRealtimeSpeechExtraOptions>
|
||||
},
|
||||
validationRequiredWhen: config => Boolean(config.accessKeyId?.trim() && config.accessKeySecret?.trim() && config.appKey?.trim()),
|
||||
validators: {
|
||||
validateConfig: [
|
||||
({ t }) => ({
|
||||
id: 'aliyun-nls:check-config',
|
||||
name: t('settings.pages.providers.catalog.edit.validators.openai-compatible.check-config.title'),
|
||||
validator: async (config) => {
|
||||
const errors: Array<{ error: unknown }> = []
|
||||
if (!config.accessKeyId?.trim())
|
||||
errors.push({ error: new Error('Access Key ID is required.') })
|
||||
if (!config.accessKeySecret?.trim())
|
||||
errors.push({ error: new Error('Access Key Secret is required.') })
|
||||
if (!config.appKey?.trim())
|
||||
errors.push({ error: new Error('App Key is required.') })
|
||||
if (config.region && !aliyunNlsRegions.includes(config.region))
|
||||
errors.push({ error: new Error('Region is invalid.') })
|
||||
|
||||
return {
|
||||
errors,
|
||||
reason: errors.map(item => (item.error as Error).message).join(', '),
|
||||
reasonKey: '',
|
||||
valid: errors.length === 0,
|
||||
}
|
||||
},
|
||||
}),
|
||||
],
|
||||
},
|
||||
extraMethods: {
|
||||
listModels: async () => [{
|
||||
id: 'aliyun-nls-v1',
|
||||
name: 'Aliyun NLS Realtime',
|
||||
provider: 'aliyun-nls-transcription',
|
||||
description: 'Realtime streaming transcription using Aliyun NLS.',
|
||||
contextLength: 0,
|
||||
deprecated: false,
|
||||
}],
|
||||
},
|
||||
})
|
||||
@@ -0,0 +1,77 @@
|
||||
import { isStageTamagotchi } from '@proj-airi/stage-shared'
|
||||
import { z } from 'zod'
|
||||
|
||||
import { createWebSpeechAPIProvider } from '../../../../stores/providers/web-speech-api'
|
||||
import { defineProvider } from '../registry'
|
||||
|
||||
const webSpeechApiConfigSchema = z.object({
|
||||
language: z.string().default('en-US'),
|
||||
continuous: z.boolean().default(true),
|
||||
interimResults: z.boolean().default(true),
|
||||
maxAlternatives: z.number().int().positive().default(1),
|
||||
})
|
||||
|
||||
function isWebSpeechApiAvailable() {
|
||||
if (typeof window === 'undefined')
|
||||
return false
|
||||
|
||||
return 'webkitSpeechRecognition' in window || 'SpeechRecognition' in window
|
||||
}
|
||||
|
||||
export const providerBrowserWebSpeechApi = defineProvider({
|
||||
id: 'browser-web-speech-api',
|
||||
name: 'Web Speech API (Browser)',
|
||||
nameLocalize: ({ t }) => t('settings.pages.providers.provider.browser-web-speech-api.title'),
|
||||
description: 'Browser-native speech recognition. No API keys.',
|
||||
descriptionLocalize: ({ t }) => t('settings.pages.providers.provider.browser-web-speech-api.description'),
|
||||
tasks: ['speech-to-text', 'automatic-speech-recognition', 'asr', 'stt', 'streaming-transcription'],
|
||||
icon: 'i-solar:microphone-bold-duotone',
|
||||
requiresCredentials: false,
|
||||
capabilities: {
|
||||
transcription: {
|
||||
protocol: 'http',
|
||||
generateOutput: false,
|
||||
streamOutput: true,
|
||||
streamInput: true,
|
||||
},
|
||||
},
|
||||
|
||||
// Electron uses Chromium, but it does not include the Google API keys that
|
||||
// the Web Speech API needs. The provider only works in browser contexts.
|
||||
isAvailableBy: () => !isStageTamagotchi() && isWebSpeechApiAvailable(),
|
||||
createProviderConfig: () => webSpeechApiConfigSchema,
|
||||
createProvider: createWebSpeechAPIProvider,
|
||||
validationRequiredWhen: () => false,
|
||||
|
||||
extraMethods: {
|
||||
listModels: async () => [
|
||||
{
|
||||
id: 'web-speech-api',
|
||||
name: 'Web Speech API',
|
||||
provider: 'browser-web-speech-api',
|
||||
description: 'Browser-native speech recognition (no API keys required)',
|
||||
contextLength: 0,
|
||||
deprecated: false,
|
||||
},
|
||||
],
|
||||
},
|
||||
validators: {
|
||||
validateConfig: [
|
||||
({ t }) => ({
|
||||
id: 'browser-web-speech-api:check-availability',
|
||||
name: t('settings.pages.providers.catalog.edit.validators.openai-compatible.check-config.title'),
|
||||
validator: async () => {
|
||||
const valid = isWebSpeechApiAvailable()
|
||||
return {
|
||||
errors: valid
|
||||
? []
|
||||
: [{ error: new Error('Web Speech API is not available. It requires a browser context with SpeechRecognition support (Chrome, Edge, Safari).') }],
|
||||
reason: valid ? '' : 'Web Speech API is not available in this environment.',
|
||||
reasonKey: '',
|
||||
valid,
|
||||
}
|
||||
},
|
||||
}),
|
||||
],
|
||||
},
|
||||
})
|
||||
@@ -1,4 +1,4 @@
|
||||
import { createChatProvider, createModelProvider, merge } from '@xsai-ext/providers/utils'
|
||||
import { createChatProvider, createModelProvider, createSpeechProvider, createTranscriptionProvider, merge } from '@xsai-ext/providers/utils'
|
||||
import { z } from 'zod'
|
||||
|
||||
import { ProviderValidationCheck } from '../../types'
|
||||
@@ -55,3 +55,50 @@ export const providerCometAPI = defineProvider<CometApiConfig>({
|
||||
}),
|
||||
},
|
||||
})
|
||||
|
||||
export const providerCometAPISpeech = defineProvider<CometApiConfig>({
|
||||
id: 'comet-api-speech',
|
||||
name: 'CometAPI Speech',
|
||||
nameLocalize: ({ t }) => t('settings.pages.providers.provider.comet-api.title'),
|
||||
description: 'cometapi.com',
|
||||
descriptionLocalize: ({ t }) => t('settings.pages.providers.provider.comet-api.description'),
|
||||
tasks: ['text-to-speech'],
|
||||
icon: 'i-lobe-icons:cometapi',
|
||||
createProviderConfig: providerCometAPI.createProviderConfig,
|
||||
createProvider(config) {
|
||||
return merge(
|
||||
createModelProvider({ apiKey: config.apiKey, baseURL: config.baseUrl! }),
|
||||
createSpeechProvider({ apiKey: config.apiKey, baseURL: config.baseUrl! }),
|
||||
)
|
||||
},
|
||||
validationRequiredWhen: config => Boolean(config.apiKey?.trim()),
|
||||
validators: createOpenAICompatibleValidators({ checks: [ProviderValidationCheck.ModelList] }),
|
||||
})
|
||||
|
||||
export const providerCometAPITranscription = defineProvider<CometApiConfig>({
|
||||
id: 'comet-api-transcription',
|
||||
name: 'CometAPI Transcription',
|
||||
nameLocalize: ({ t }) => t('settings.pages.providers.provider.comet-api.title'),
|
||||
description: 'cometapi.com',
|
||||
descriptionLocalize: ({ t }) => t('settings.pages.providers.provider.comet-api.description'),
|
||||
tasks: ['speech-to-text', 'automatic-speech-recognition', 'asr', 'stt'],
|
||||
icon: 'i-lobe-icons:cometapi',
|
||||
capabilities: {
|
||||
transcription: { protocol: 'http', generateOutput: true, streamOutput: false, streamInput: false },
|
||||
},
|
||||
createProviderConfig: providerCometAPI.createProviderConfig,
|
||||
createProvider(config) {
|
||||
const provider = merge(
|
||||
createModelProvider({ apiKey: config.apiKey, baseURL: config.baseUrl! }),
|
||||
createTranscriptionProvider({ apiKey: config.apiKey, baseURL: config.baseUrl! }),
|
||||
)
|
||||
const transcription = provider.transcription.bind(provider)
|
||||
provider.transcription = (model: string, extraOptions?: Record<string, unknown>) => ({
|
||||
...transcription(model),
|
||||
...extraOptions,
|
||||
})
|
||||
return provider
|
||||
},
|
||||
validationRequiredWhen: config => Boolean(config.apiKey?.trim()),
|
||||
validators: createOpenAICompatibleValidators({ checks: [ProviderValidationCheck.ModelList] }),
|
||||
})
|
||||
|
||||
@@ -0,0 +1,133 @@
|
||||
import type { SpeechProviderWithExtraOptions } from '@xsai-ext/providers/utils'
|
||||
import type { ListVoicesOptions, UnElevenLabsOptions, VoiceProviderWithExtraOptions } from 'unspeech'
|
||||
import type { ComposerTranslation } from 'vue-i18n'
|
||||
|
||||
import { createUnElevenLabs, listVoices } from 'unspeech'
|
||||
import { z } from 'zod'
|
||||
|
||||
import { models as elevenLabsModels } from '../../../../stores/providers/elevenlabs/list-models'
|
||||
import { defineProvider } from '../registry'
|
||||
|
||||
const elevenLabsConfigSchema = z.object({
|
||||
apiKey: z.string(),
|
||||
baseUrl: z.string().default('https://unspeech.hyp3r.link/v1/'),
|
||||
voiceSettings: z.object({
|
||||
similarityBoost: z.number().default(0.75),
|
||||
stability: z.number().default(0.5),
|
||||
}).default({ similarityBoost: 0.75, stability: 0.5 }),
|
||||
})
|
||||
|
||||
type ElevenLabsConfig = z.input<typeof elevenLabsConfigSchema>
|
||||
|
||||
function toListVoicesOptions(provider: VoiceProviderWithExtraOptions<UnElevenLabsOptions>): ListVoicesOptions {
|
||||
const { fetch: _fetch, ...voiceOptions } = provider.voice()
|
||||
return voiceOptions
|
||||
}
|
||||
|
||||
function createElevenLabsValidators() {
|
||||
return {
|
||||
validateConfig: [
|
||||
({ t }: { t: ComposerTranslation }) => ({
|
||||
id: 'elevenlabs:check-config',
|
||||
name: t('settings.pages.providers.catalog.edit.validators.openai-compatible.check-config.title'),
|
||||
validator: async (config: ElevenLabsConfig) => {
|
||||
const errors: Array<{ error: unknown }> = []
|
||||
const apiKey = config.apiKey?.trim() ?? ''
|
||||
const baseUrl = config.baseUrl?.trim() ?? ''
|
||||
|
||||
if (!apiKey)
|
||||
errors.push({ error: new Error('API key is required.') })
|
||||
|
||||
if (!baseUrl) {
|
||||
errors.push({ error: new Error('Base URL is required.') })
|
||||
}
|
||||
else {
|
||||
try {
|
||||
if (!new URL(baseUrl).host)
|
||||
errors.push({ error: new Error('Base URL is not absolute. Try to include a scheme (http:// or https://).') })
|
||||
else if (!baseUrl.endsWith('/'))
|
||||
errors.push({ error: new Error('Base URL must end with a trailing slash (/).') })
|
||||
}
|
||||
catch {
|
||||
errors.push({ error: new Error('Base URL is not absolute. Try to include a scheme (http:// or https://).') })
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
errors,
|
||||
reason: errors.map(item => (item.error as Error).message).join(', '),
|
||||
reasonKey: '',
|
||||
valid: errors.length === 0,
|
||||
}
|
||||
},
|
||||
}),
|
||||
],
|
||||
}
|
||||
}
|
||||
|
||||
export const providerElevenLabs = defineProvider<ElevenLabsConfig>({
|
||||
id: 'elevenlabs',
|
||||
name: 'ElevenLabs',
|
||||
nameLocalize: ({ t }) => t('settings.pages.providers.provider.elevenlabs.title'),
|
||||
description: 'elevenlabs.io',
|
||||
descriptionLocalize: ({ t }) => t('settings.pages.providers.provider.elevenlabs.description'),
|
||||
tasks: ['text-to-speech'],
|
||||
icon: 'i-simple-icons:elevenlabs',
|
||||
|
||||
createProviderConfig: ({ t }) => elevenLabsConfigSchema.extend({
|
||||
apiKey: elevenLabsConfigSchema.shape.apiKey.meta({
|
||||
labelLocalized: t('settings.pages.providers.catalog.edit.config.common.fields.field.api-key.label'),
|
||||
descriptionLocalized: t('settings.pages.providers.catalog.edit.config.common.fields.field.api-key.description'),
|
||||
placeholderLocalized: t('settings.pages.providers.catalog.edit.config.common.fields.field.api-key.placeholder'),
|
||||
type: 'password',
|
||||
}),
|
||||
baseUrl: elevenLabsConfigSchema.shape.baseUrl.meta({
|
||||
labelLocalized: t('settings.pages.providers.catalog.edit.config.common.fields.field.base-url.label'),
|
||||
descriptionLocalized: t('settings.pages.providers.catalog.edit.config.common.fields.field.base-url.description'),
|
||||
placeholderLocalized: t('settings.pages.providers.catalog.edit.config.common.fields.field.base-url.placeholder'),
|
||||
}),
|
||||
}),
|
||||
createProvider(config) {
|
||||
return createUnElevenLabs(config.apiKey.trim(), config.baseUrl?.trim() ?? 'https://unspeech.hyp3r.link/v1/') as SpeechProviderWithExtraOptions<string, UnElevenLabsOptions>
|
||||
},
|
||||
|
||||
validationRequiredWhen: config => Boolean(config.apiKey?.trim() && config.baseUrl?.trim()),
|
||||
validators: createElevenLabsValidators(),
|
||||
extraMethods: {
|
||||
listModels: async () => elevenLabsModels.map(model => ({
|
||||
id: model.model_id,
|
||||
name: model.name,
|
||||
provider: 'elevenlabs',
|
||||
description: model.description,
|
||||
contextLength: 0,
|
||||
deprecated: false,
|
||||
})),
|
||||
listVoices: async (config) => {
|
||||
const provider = createUnElevenLabs(config.apiKey.trim(), config.baseUrl?.trim() ?? 'https://unspeech.hyp3r.link/v1/') as VoiceProviderWithExtraOptions<UnElevenLabsOptions>
|
||||
const voices = await listVoices(toListVoicesOptions(provider))
|
||||
if (!Array.isArray(voices))
|
||||
return []
|
||||
|
||||
// Keep the default ElevenLabs voices together at the end of the list.
|
||||
const ariaIndex = voices.findIndex(voice => voice.name.includes('Aria'))
|
||||
const billIndex = voices.findIndex(voice => voice.name.includes('Bill'))
|
||||
const startIndex = ariaIndex !== -1 ? ariaIndex : 0
|
||||
const endIndex = billIndex !== -1 ? billIndex : voices.length - 1
|
||||
const lowerIndex = Math.min(startIndex, endIndex)
|
||||
const higherIndex = Math.max(startIndex, endIndex)
|
||||
const rearrangedVoices = [
|
||||
...voices.slice(0, lowerIndex),
|
||||
...voices.slice(higherIndex + 1),
|
||||
...voices.slice(lowerIndex, higherIndex + 1),
|
||||
]
|
||||
|
||||
return rearrangedVoices.map(voice => ({
|
||||
id: voice.id,
|
||||
name: voice.name,
|
||||
provider: 'elevenlabs',
|
||||
previewURL: voice.preview_audio_url,
|
||||
languages: voice.languages,
|
||||
}))
|
||||
},
|
||||
},
|
||||
})
|
||||
@@ -0,0 +1,172 @@
|
||||
import { toWavFromPCM16 } from '@proj-airi/audio/encoding'
|
||||
import { z } from 'zod'
|
||||
|
||||
import { defineProvider } from '../registry'
|
||||
|
||||
const DEFAULT_BASE_URL = 'https://generativelanguage.googleapis.com/v1beta/'
|
||||
const DEFAULT_MODEL = 'gemini-2.5-flash-preview-tts'
|
||||
|
||||
const googleGeminiSpeechConfigSchema = z.object({
|
||||
apiKey: z.string(),
|
||||
baseUrl: z.string().default(DEFAULT_BASE_URL),
|
||||
})
|
||||
|
||||
type GoogleGeminiSpeechConfig = z.input<typeof googleGeminiSpeechConfigSchema>
|
||||
|
||||
const googleGeminiTtsModels = [
|
||||
'gemini-2.5-flash-preview-tts',
|
||||
'gemini-2.5-pro-preview-tts',
|
||||
'gemini-3.1-flash-tts-preview',
|
||||
] as const
|
||||
|
||||
const googleGeminiTtsVoices: Array<[string, string]> = [
|
||||
['Zephyr', 'Bright'],
|
||||
['Puck', 'Upbeat'],
|
||||
['Charon', 'Informative'],
|
||||
['Kore', 'Firm'],
|
||||
['Fenrir', 'Excitable'],
|
||||
['Leda', 'Youthful'],
|
||||
['Orus', 'Firm'],
|
||||
['Aoede', 'Breezy'],
|
||||
['Callirrhoe', 'Easy-going'],
|
||||
['Autonoe', 'Bright'],
|
||||
['Enceladus', 'Breathy'],
|
||||
['Iapetus', 'Clear'],
|
||||
['Umbriel', 'Easy-going'],
|
||||
['Algieba', 'Smooth'],
|
||||
['Despina', 'Smooth'],
|
||||
['Erinome', 'Clear'],
|
||||
['Algenib', 'Gravelly'],
|
||||
['Rasalgethi', 'Informative'],
|
||||
['Laomedeia', 'Upbeat'],
|
||||
['Achernar', 'Soft'],
|
||||
['Alnilam', 'Firm'],
|
||||
['Schedar', 'Even'],
|
||||
['Gacrux', 'Mature'],
|
||||
['Pulcherrima', 'Forward'],
|
||||
['Achird', 'Friendly'],
|
||||
['Zubenelgenubi', 'Casual'],
|
||||
['Vindemiatrix', 'Gentle'],
|
||||
['Sadachbia', 'Lively'],
|
||||
['Sadaltager', 'Knowledgeable'],
|
||||
['Sulafat', 'Warm'],
|
||||
]
|
||||
|
||||
function normalizeBaseUrl(baseUrl: string | undefined) {
|
||||
const value = baseUrl?.trim() || DEFAULT_BASE_URL
|
||||
return value.endsWith('/') ? value : `${value}/`
|
||||
}
|
||||
|
||||
function decodeBase64(base64: string) {
|
||||
const binary = atob(base64)
|
||||
const bytes = new Uint8Array(binary.length)
|
||||
for (let index = 0; index < binary.length; index++)
|
||||
bytes[index] = binary.charCodeAt(index)
|
||||
return bytes
|
||||
}
|
||||
|
||||
function createAudioFetch(apiKey: string, baseUrl: string) {
|
||||
return async (_input: RequestInfo | URL, init?: RequestInit) => {
|
||||
if (!init?.body || typeof init.body !== 'string')
|
||||
throw new Error('Invalid request body')
|
||||
|
||||
const body = JSON.parse(init.body) as {
|
||||
input?: string
|
||||
model?: string
|
||||
voice?: string
|
||||
temperature?: number
|
||||
}
|
||||
if (!body.input)
|
||||
throw new Error('Missing input text for Gemini TTS')
|
||||
if (!body.model)
|
||||
throw new Error('Missing model for Gemini TTS')
|
||||
|
||||
const response = await globalThis.fetch(new URL(`models/${body.model}:generateContent`, baseUrl), {
|
||||
method: 'POST',
|
||||
headers: { 'x-goog-api-key': apiKey, 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
contents: [{ parts: [{ text: body.input }] }],
|
||||
generationConfig: {
|
||||
responseModalities: ['AUDIO'],
|
||||
speechConfig: {
|
||||
voiceConfig: { prebuiltVoiceConfig: { voiceName: body.voice || 'Kore' } },
|
||||
},
|
||||
...(body.temperature !== undefined ? { temperature: body.temperature } : {}),
|
||||
},
|
||||
}),
|
||||
})
|
||||
if (!response.ok)
|
||||
throw new Error(`Gemini TTS request failed: ${response.status} ${await response.text().catch(() => '')}`)
|
||||
|
||||
const data = await response.json() as {
|
||||
candidates?: Array<{ content?: { parts?: Array<{ inlineData?: { data?: string } }> } }>
|
||||
}
|
||||
const audio = data.candidates?.[0]?.content?.parts?.find(part => part.inlineData)?.inlineData?.data
|
||||
if (!audio)
|
||||
throw new Error('Gemini TTS response missing audio data')
|
||||
|
||||
return new Response(toWavFromPCM16(decodeBase64(audio), 24000), {
|
||||
status: 200,
|
||||
headers: { 'Content-Type': 'audio/wav' },
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
export const providerGoogleGeminiAudioSpeech = defineProvider<GoogleGeminiSpeechConfig>({
|
||||
id: 'google-gemini-audio-speech',
|
||||
name: 'Google Gemini',
|
||||
nameLocalize: ({ t }) => t('settings.pages.providers.provider.google-gemini-audio-speech.title'),
|
||||
description: 'aistudio.google.com',
|
||||
descriptionLocalize: ({ t }) => t('settings.pages.providers.provider.google-gemini-audio-speech.description'),
|
||||
tasks: ['text-to-speech', 'tts'],
|
||||
icon: 'i-lobe-icons:gemini',
|
||||
iconColor: 'i-lobe-icons:gemini-color',
|
||||
createProviderConfig: () => googleGeminiSpeechConfigSchema,
|
||||
createProvider(config) {
|
||||
const apiKey = config.apiKey.trim()
|
||||
const baseUrl = normalizeBaseUrl(config.baseUrl)
|
||||
return {
|
||||
speech: (model?: string, options?: Record<string, unknown>) => ({
|
||||
baseURL: baseUrl,
|
||||
fetch: createAudioFetch(apiKey, baseUrl),
|
||||
...options,
|
||||
model: model || (typeof options?.model === 'string' ? options.model : DEFAULT_MODEL),
|
||||
}),
|
||||
}
|
||||
},
|
||||
validationRequiredWhen: config => Boolean(config.apiKey?.trim()),
|
||||
validators: {
|
||||
validateConfig: [
|
||||
({ t }) => ({
|
||||
id: 'google-gemini-audio-speech:check-config',
|
||||
name: t('settings.pages.providers.catalog.edit.validators.openai-compatible.check-config.title'),
|
||||
validator: async (config) => {
|
||||
const valid = Boolean(config.apiKey?.trim())
|
||||
return {
|
||||
errors: valid ? [] : [{ error: new Error('API Key is required.') }],
|
||||
reason: valid ? '' : 'API Key is required.',
|
||||
reasonKey: '',
|
||||
valid,
|
||||
}
|
||||
},
|
||||
}),
|
||||
],
|
||||
},
|
||||
extraMethods: {
|
||||
listModels: async () => googleGeminiTtsModels.map(id => ({
|
||||
id,
|
||||
name: id.split('-').map(word => `${word[0].toUpperCase()}${word.slice(1)}`).join(' '),
|
||||
provider: 'google-gemini-audio-speech',
|
||||
description: 'Gemini API text-to-speech model',
|
||||
capabilities: ['text-to-speech'],
|
||||
})),
|
||||
listVoices: async () => googleGeminiTtsVoices.map(([id, style]) => ({
|
||||
id,
|
||||
name: id,
|
||||
provider: 'google-gemini-audio-speech',
|
||||
description: style,
|
||||
languages: [{ code: 'auto', title: 'Auto' }],
|
||||
compatibleModels: [...googleGeminiTtsModels],
|
||||
})),
|
||||
},
|
||||
})
|
||||
@@ -0,0 +1,92 @@
|
||||
import { errorMessageFrom } from '@moeru/std'
|
||||
import { z } from 'zod'
|
||||
|
||||
import { defineProvider } from '../registry'
|
||||
|
||||
const indexTtsConfigSchema = z.object({
|
||||
baseUrl: z.string().default('http://localhost:11996/tts/'),
|
||||
model: z.string().default('IndexTTS-1.5'),
|
||||
})
|
||||
|
||||
type IndexTtsConfig = z.input<typeof indexTtsConfigSchema>
|
||||
|
||||
function voicesUrl(config: IndexTtsConfig) {
|
||||
return `${config.baseUrl ?? 'http://localhost:11996/tts/'}audio/voices`
|
||||
}
|
||||
|
||||
export const providerIndexTtsVllm = defineProvider<IndexTtsConfig>({
|
||||
id: 'index-tts-vllm',
|
||||
name: 'Index-TTS by Bilibili',
|
||||
nameLocalize: ({ t }) => t('settings.pages.providers.provider.index-tts-vllm.title'),
|
||||
description: 'index-tts.github.io',
|
||||
descriptionLocalize: ({ t }) => t('settings.pages.providers.provider.index-tts-vllm.description'),
|
||||
tasks: ['text-to-speech'],
|
||||
iconColor: 'i-lobe-icons:bilibiliindex',
|
||||
createProviderConfig: () => indexTtsConfigSchema,
|
||||
createProvider(config) {
|
||||
return {
|
||||
speech: () => ({
|
||||
baseURL: config.baseUrl ?? 'http://localhost:11996/tts/',
|
||||
model: config.model || 'IndexTTS-1.5',
|
||||
}),
|
||||
}
|
||||
},
|
||||
validationRequiredWhen: config => Boolean(config.baseUrl?.trim()),
|
||||
validators: {
|
||||
validateConfig: [
|
||||
({ t }) => ({
|
||||
id: 'index-tts-vllm:check-config',
|
||||
name: t('settings.pages.providers.catalog.edit.validators.openai-compatible.check-config.title'),
|
||||
validator: async (config) => {
|
||||
const baseUrl = config.baseUrl?.trim() ?? ''
|
||||
if (!baseUrl) {
|
||||
const reason = 'Base URL is required. Default to http://localhost:11996/tts/ for Index-TTS.'
|
||||
return { errors: [{ error: new Error(reason) }], reason, reasonKey: '', valid: false }
|
||||
}
|
||||
|
||||
const controller = new AbortController()
|
||||
const timeout = setTimeout(() => controller.abort(), 5000)
|
||||
try {
|
||||
const response = await fetch(voicesUrl(config), { signal: controller.signal })
|
||||
if (!response.ok) {
|
||||
const reason = `IndexTTS unreachable: HTTP ${response.status} ${response.statusText}`
|
||||
return { errors: [{ error: new Error(reason) }], reason, reasonKey: '', valid: false }
|
||||
}
|
||||
}
|
||||
catch (error) {
|
||||
const reason = `IndexTTS connection failed: ${errorMessageFrom(error) ?? 'Unknown error'}`
|
||||
return { errors: [{ error }], reason, reasonKey: '', valid: false }
|
||||
}
|
||||
finally {
|
||||
clearTimeout(timeout)
|
||||
}
|
||||
|
||||
return { errors: [], reason: '', reasonKey: '', valid: true }
|
||||
},
|
||||
}),
|
||||
],
|
||||
},
|
||||
extraMethods: {
|
||||
listModels: async () => [{
|
||||
id: 'IndexTTS-1.5',
|
||||
name: 'IndexTTS-1.5',
|
||||
provider: 'index-tts-vllm',
|
||||
description: 'Default model for Index-TTS vLLM deployment',
|
||||
contextLength: 0,
|
||||
deprecated: false,
|
||||
}],
|
||||
listVoices: async (config) => {
|
||||
const response = await fetch(voicesUrl(config))
|
||||
if (!response.ok)
|
||||
throw new Error(`Failed to fetch voices: ${response.statusText}`)
|
||||
|
||||
const voices = await response.json() as Record<string, unknown>
|
||||
return Object.keys(voices).map(voice => ({
|
||||
id: voice,
|
||||
name: voice,
|
||||
provider: 'index-tts-vllm',
|
||||
languages: [{ code: 'cn', title: 'Chinese' }, { code: 'en', title: 'English' }],
|
||||
}))
|
||||
},
|
||||
},
|
||||
})
|
||||
@@ -1,21 +1,30 @@
|
||||
import './amazon-bedrock'
|
||||
import './openai'
|
||||
import './openai-audio'
|
||||
import './aihubmix'
|
||||
import './aliyun-nls'
|
||||
import './lm-studio'
|
||||
import './local-audio'
|
||||
import './index-tts-vllm'
|
||||
import './kokoro-local'
|
||||
import './azure-openai'
|
||||
import './openai-compatible'
|
||||
import './atlascloud'
|
||||
import './volcengine-coding-plan'
|
||||
import './byteplus'
|
||||
import './byteplus-coding-plan'
|
||||
import './browser-web-speech-api'
|
||||
import './n1n'
|
||||
import './openpaths'
|
||||
import './openrouter-ai'
|
||||
import './openrouter-audio-speech'
|
||||
import './nvidia'
|
||||
import './groq'
|
||||
import './anthropic'
|
||||
import './google-generative-ai'
|
||||
import './google-gemini-audio-speech'
|
||||
import './deepseek'
|
||||
import './elevenlabs'
|
||||
import './302-ai'
|
||||
import './cerebras-ai'
|
||||
import './together-ai'
|
||||
@@ -26,15 +35,20 @@ import './fireworks-ai'
|
||||
import './featherless-ai'
|
||||
import './comet-api'
|
||||
import './perplexity-ai'
|
||||
import './player2-speech'
|
||||
import './minimax'
|
||||
import './minimax-speech'
|
||||
import './mistral-ai'
|
||||
import './moonshot-ai'
|
||||
import './modelscope'
|
||||
import './ollama'
|
||||
import './mimo'
|
||||
import './mimo-audio'
|
||||
import './cloudflare-workers-ai'
|
||||
import './azure-ai-foundry'
|
||||
import './official'
|
||||
import './speech-noop'
|
||||
import './unspeech'
|
||||
|
||||
export {
|
||||
getDefaultStreamingModel,
|
||||
|
||||
@@ -0,0 +1,184 @@
|
||||
import type { ProgressInfo } from '@xsai-transformers/shared/types'
|
||||
|
||||
import type { VoiceKey } from '../../../../workers/kokoro/types'
|
||||
|
||||
import { getCachedWebGPUCapabilities } from '@proj-airi/stage-shared/webgpu'
|
||||
import { z } from 'zod'
|
||||
|
||||
import { getDefaultKokoroModel, KOKORO_MODELS, kokoroModelsToModelInfo } from '../../../../workers/kokoro/constants'
|
||||
import { getKokoroAdapter } from '../../../inference/adapters/kokoro'
|
||||
import { defineProvider } from '../registry'
|
||||
|
||||
interface KokoroVoice {
|
||||
language: string
|
||||
name: string
|
||||
gender: string
|
||||
}
|
||||
|
||||
const languageByCode: Record<string, { code: string, title: string }> = {
|
||||
'en-us': { code: 'en-US', title: 'English (US)' },
|
||||
'en-gb': { code: 'en-GB', title: 'English (UK)' },
|
||||
'ja': { code: 'ja', title: 'Japanese' },
|
||||
'zh-cn': { code: 'zh-CN', title: 'Chinese (Mandarin)' },
|
||||
'es': { code: 'es', title: 'Spanish' },
|
||||
'fr': { code: 'fr', title: 'French' },
|
||||
'hi': { code: 'hi', title: 'Hindi' },
|
||||
'it': { code: 'it', title: 'Italian' },
|
||||
'pt-br': { code: 'pt-BR', title: 'Portuguese (Brazil)' },
|
||||
}
|
||||
|
||||
function getWebGpuState() {
|
||||
const capabilities = getCachedWebGPUCapabilities()
|
||||
return {
|
||||
supported: capabilities?.supported ?? (typeof navigator !== 'undefined' && Boolean(navigator.gpu)),
|
||||
fp16Supported: capabilities?.fp16Supported ?? false,
|
||||
}
|
||||
}
|
||||
|
||||
function getModel(modelId: string) {
|
||||
return KOKORO_MODELS.find(model => model.id === modelId)
|
||||
}
|
||||
|
||||
function assertModelSupported(modelId: string) {
|
||||
const model = getModel(modelId)
|
||||
if (!model)
|
||||
throw new Error(`Invalid model: ${modelId}. Must be one of: ${KOKORO_MODELS.map(item => item.id).join(', ')}`)
|
||||
|
||||
if (model.platform === 'webgpu' && !getWebGpuState().supported)
|
||||
throw new Error('WebGPU is required for this model but is not available in your browser')
|
||||
|
||||
return model
|
||||
}
|
||||
|
||||
function progressInfo(progress: { file?: string, percent: number, loaded?: number, total?: number }): ProgressInfo {
|
||||
return {
|
||||
name: progress.file ?? '',
|
||||
file: progress.file ?? '',
|
||||
progress: progress.percent >= 0 ? progress.percent : 0,
|
||||
status: 'progress',
|
||||
loaded: progress.loaded ?? 0,
|
||||
total: progress.total ?? 0,
|
||||
}
|
||||
}
|
||||
|
||||
let lastLoadedModelId: string | null = null
|
||||
|
||||
export const providerKokoroLocal = defineProvider({
|
||||
id: 'kokoro-local',
|
||||
name: 'Kokoro TTS',
|
||||
nameLocalize: ({ t }) => t('settings.pages.providers.provider.kokoro-local.title'),
|
||||
description: 'Local text-to-speech using Kokoro-82M.',
|
||||
descriptionLocalize: ({ t }) => t('settings.pages.providers.provider.kokoro-local.description'),
|
||||
tasks: ['text-to-speech'],
|
||||
icon: 'i-lobe-icons:speaker',
|
||||
requiresCredentials: false,
|
||||
createProviderConfig: () => {
|
||||
const capabilities = getWebGpuState()
|
||||
return z.object({
|
||||
model: z.string().default(getDefaultKokoroModel(capabilities.supported, capabilities.fp16Supported)),
|
||||
voiceId: z.string().default(''),
|
||||
})
|
||||
},
|
||||
createProvider() {
|
||||
const adapterPromise = getKokoroAdapter()
|
||||
return {
|
||||
speech: () => ({
|
||||
baseURL: 'http://kokoro-local/v1/',
|
||||
model: 'kokoro-82m',
|
||||
fetch: async (_input: RequestInfo | URL, init?: RequestInit) => {
|
||||
if (!init?.body || typeof init.body !== 'string')
|
||||
throw new Error('Invalid request body')
|
||||
|
||||
const body = JSON.parse(init.body) as { input?: string, voice?: string }
|
||||
if (!body.voice)
|
||||
throw new Error('Voice parameter is required')
|
||||
|
||||
try {
|
||||
const adapter = await adapterPromise
|
||||
if (!(body.voice in adapter.getVoices()))
|
||||
throw new Error(`Unknown Kokoro voice: ${body.voice}`)
|
||||
const buffer = await adapter.generate(body.input ?? '', body.voice as VoiceKey)
|
||||
return new Response(buffer, {
|
||||
status: 200,
|
||||
headers: { 'Content-Type': 'audio/wav' },
|
||||
})
|
||||
}
|
||||
catch (error) {
|
||||
console.error('Kokoro TTS generation failed:', error)
|
||||
throw error
|
||||
}
|
||||
},
|
||||
}),
|
||||
}
|
||||
},
|
||||
validationRequiredWhen: () => false,
|
||||
validators: {
|
||||
validateConfig: [
|
||||
({ t }) => ({
|
||||
id: 'kokoro-local:check-config',
|
||||
name: t('settings.pages.providers.catalog.edit.validators.openai-compatible.check-config.title'),
|
||||
validator: async (config) => {
|
||||
if (!config.model) {
|
||||
return {
|
||||
errors: [{ error: new Error('No model selected') }],
|
||||
reason: 'Please select a model from the dropdown menu',
|
||||
reasonKey: '',
|
||||
valid: false,
|
||||
}
|
||||
}
|
||||
if (!getModel(config.model)) {
|
||||
const reason = `Invalid model. Must be one of: ${KOKORO_MODELS.map(model => model.id).join(', ')}`
|
||||
return { errors: [{ error: new Error(`Invalid model: ${config.model}`) }], reason, reasonKey: '', valid: false }
|
||||
}
|
||||
return { errors: [], reason: '', reasonKey: '', valid: true }
|
||||
},
|
||||
}),
|
||||
],
|
||||
},
|
||||
extraMethods: {
|
||||
listModels: async () => {
|
||||
const capabilities = getWebGpuState()
|
||||
return kokoroModelsToModelInfo(capabilities.supported, undefined, capabilities.fp16Supported)
|
||||
},
|
||||
loadModel: async (config, _provider, hooks) => {
|
||||
const model = assertModelSupported(config.model)
|
||||
try {
|
||||
const adapter = await getKokoroAdapter()
|
||||
await adapter.loadModel(model.quantization, model.platform, {
|
||||
onProgress: hooks?.onProgress ? progress => hooks.onProgress?.(progressInfo(progress)) : undefined,
|
||||
})
|
||||
}
|
||||
catch (error) {
|
||||
console.error('Failed to load Kokoro model:', error)
|
||||
throw error
|
||||
}
|
||||
},
|
||||
listVoices: async (config) => {
|
||||
try {
|
||||
const adapter = await getKokoroAdapter()
|
||||
if (adapter.state !== 'ready' || config.model !== lastLoadedModelId) {
|
||||
const model = assertModelSupported(config.model)
|
||||
await adapter.loadModel(model.quantization, model.platform)
|
||||
lastLoadedModelId = config.model
|
||||
}
|
||||
|
||||
return Object.entries(adapter.getVoices() as Record<string, KokoroVoice>).map(([id, voice]) => {
|
||||
const languageCode = voice.language.toLowerCase()
|
||||
const language = languageByCode[languageCode] || { code: languageCode, title: voice.language }
|
||||
return {
|
||||
id,
|
||||
name: `${voice.name} (${voice.gender}, ${language.title.split('(')[0].trim()})`,
|
||||
provider: 'kokoro-local',
|
||||
languages: [language],
|
||||
gender: voice.gender.toLowerCase(),
|
||||
}
|
||||
})
|
||||
}
|
||||
catch (error) {
|
||||
console.error('Failed to fetch Kokoro voices:', error)
|
||||
// Voice discovery can run before model loading. An empty list is safe.
|
||||
return []
|
||||
}
|
||||
},
|
||||
},
|
||||
})
|
||||
@@ -0,0 +1,160 @@
|
||||
import type { ComposerTranslation } from 'vue-i18n'
|
||||
|
||||
import { isStageTamagotchi } from '@proj-airi/stage-shared'
|
||||
import { isWebGPUSupported } from '@proj-airi/stage-shared/webgpu'
|
||||
import { createOpenAI } from '@xsai-ext/providers/create'
|
||||
import { z } from 'zod'
|
||||
|
||||
import { defineProvider } from '../registry'
|
||||
|
||||
const localAudioConfigSchema = z.object({
|
||||
apiKey: z.string().optional(),
|
||||
baseUrl: z.string().optional().default(''),
|
||||
})
|
||||
|
||||
type LocalAudioConfig = z.input<typeof localAudioConfigSchema>
|
||||
|
||||
function createLocalAudioConfigSchema(t: ComposerTranslation) {
|
||||
return localAudioConfigSchema.extend({
|
||||
apiKey: localAudioConfigSchema.shape.apiKey.meta({
|
||||
labelLocalized: t('settings.pages.providers.catalog.edit.config.common.fields.field.api-key.label'),
|
||||
descriptionLocalized: t('settings.pages.providers.catalog.edit.config.common.fields.field.api-key.description'),
|
||||
placeholderLocalized: t('settings.pages.providers.catalog.edit.config.common.fields.field.api-key.placeholder'),
|
||||
type: 'password',
|
||||
}),
|
||||
baseUrl: localAudioConfigSchema.shape.baseUrl.meta({
|
||||
labelLocalized: t('settings.pages.providers.catalog.edit.config.common.fields.field.base-url.label'),
|
||||
descriptionLocalized: t('settings.pages.providers.catalog.edit.config.common.fields.field.base-url.description'),
|
||||
placeholderLocalized: t('settings.pages.providers.catalog.edit.config.common.fields.field.base-url.placeholder'),
|
||||
}),
|
||||
})
|
||||
}
|
||||
|
||||
function normalizeBaseUrl(baseUrl: string | undefined) {
|
||||
const normalized = baseUrl?.trim() ?? ''
|
||||
return normalized && !normalized.endsWith('/') ? `${normalized}/` : normalized
|
||||
}
|
||||
|
||||
function createLocalAudioProvider(config: LocalAudioConfig) {
|
||||
return createOpenAI(config.apiKey?.trim() ?? '', normalizeBaseUrl(config.baseUrl))
|
||||
}
|
||||
|
||||
function createLocalTranscriptionProvider(config: LocalAudioConfig) {
|
||||
const provider = createLocalAudioProvider(config)
|
||||
const transcription = provider.transcription.bind(provider)
|
||||
provider.transcription = (model: string, extraOptions?: Record<string, unknown>) => ({
|
||||
...transcription(model),
|
||||
...extraOptions,
|
||||
})
|
||||
return provider
|
||||
}
|
||||
|
||||
function createLocalAudioValidators() {
|
||||
return {
|
||||
validateConfig: [
|
||||
({ t }: { t: ComposerTranslation }) => ({
|
||||
id: 'local-audio:check-config',
|
||||
name: t('settings.pages.providers.catalog.edit.validators.openai-compatible.check-config.title'),
|
||||
validator: async (config: LocalAudioConfig) => {
|
||||
const valid = Boolean(config.baseUrl)
|
||||
const reason = valid
|
||||
? ''
|
||||
: 'Base URL is required. This is likely a bug, report to developers on https://github.com/moeru-ai/airi/issues.'
|
||||
return {
|
||||
errors: valid ? [] : [{ error: new Error('Base URL is required.') }],
|
||||
reason,
|
||||
reasonKey: '',
|
||||
valid,
|
||||
}
|
||||
},
|
||||
}),
|
||||
],
|
||||
}
|
||||
}
|
||||
|
||||
async function isBrowserAndMemoryEnough() {
|
||||
if (isStageTamagotchi())
|
||||
return false
|
||||
|
||||
if (await isWebGPUSupported())
|
||||
return true
|
||||
|
||||
if ('navigator' in globalThis && globalThis.navigator != null && 'deviceMemory' in globalThis.navigator && typeof globalThis.navigator.deviceMemory === 'number') {
|
||||
// The browser model needs at least 8 GB of system memory without WebGPU.
|
||||
return globalThis.navigator.deviceMemory >= 8
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
export const providerAppLocalAudioSpeech = defineProvider<LocalAudioConfig>({
|
||||
id: 'app-local-audio-speech',
|
||||
name: 'App (Local)',
|
||||
nameLocalize: ({ t }) => t('settings.pages.providers.provider.app-local-audio-speech.title'),
|
||||
description: 'https://github.com/huggingface/candle',
|
||||
descriptionLocalize: ({ t }) => t('settings.pages.providers.provider.app-local-audio-speech.description'),
|
||||
tasks: ['text-to-speech', 'tts'],
|
||||
icon: 'i-lobe-icons:huggingface',
|
||||
isAvailableBy: isStageTamagotchi,
|
||||
createProviderConfig: ({ t }) => createLocalAudioConfigSchema(t),
|
||||
createProvider: createLocalAudioProvider,
|
||||
validators: createLocalAudioValidators(),
|
||||
})
|
||||
|
||||
export const providerAppLocalAudioTranscription = defineProvider<LocalAudioConfig>({
|
||||
id: 'app-local-audio-transcription',
|
||||
name: 'App (Local)',
|
||||
nameLocalize: ({ t }) => t('settings.pages.providers.provider.app-local-audio-transcription.title'),
|
||||
description: 'https://github.com/huggingface/candle',
|
||||
descriptionLocalize: ({ t }) => t('settings.pages.providers.provider.app-local-audio-transcription.description'),
|
||||
tasks: ['speech-to-text', 'automatic-speech-recognition', 'asr', 'stt'],
|
||||
icon: 'i-lobe-icons:huggingface',
|
||||
isAvailableBy: isStageTamagotchi,
|
||||
capabilities: {
|
||||
transcription: {
|
||||
protocol: 'http',
|
||||
generateOutput: true,
|
||||
streamOutput: false,
|
||||
streamInput: false,
|
||||
},
|
||||
},
|
||||
createProviderConfig: ({ t }) => createLocalAudioConfigSchema(t),
|
||||
createProvider: createLocalTranscriptionProvider,
|
||||
validators: createLocalAudioValidators(),
|
||||
})
|
||||
|
||||
export const providerBrowserLocalAudioSpeech = defineProvider<LocalAudioConfig>({
|
||||
id: 'browser-local-audio-speech',
|
||||
name: 'Browser (Local)',
|
||||
nameLocalize: ({ t }) => t('settings.pages.providers.provider.browser-local-audio-speech.title'),
|
||||
description: 'https://github.com/moeru-ai/xsai-transformers',
|
||||
descriptionLocalize: ({ t }) => t('settings.pages.providers.provider.browser-local-audio-speech.description'),
|
||||
tasks: ['text-to-speech', 'tts'],
|
||||
icon: 'i-lobe-icons:huggingface',
|
||||
isAvailableBy: isBrowserAndMemoryEnough,
|
||||
createProviderConfig: ({ t }) => createLocalAudioConfigSchema(t),
|
||||
createProvider: createLocalAudioProvider,
|
||||
validators: createLocalAudioValidators(),
|
||||
})
|
||||
|
||||
export const providerBrowserLocalAudioTranscription = defineProvider<LocalAudioConfig>({
|
||||
id: 'browser-local-audio-transcription',
|
||||
name: 'Browser (Local)',
|
||||
nameLocalize: ({ t }) => t('settings.pages.providers.provider.browser-local-audio-transcription.title'),
|
||||
description: 'https://github.com/moeru-ai/xsai-transformers',
|
||||
descriptionLocalize: ({ t }) => t('settings.pages.providers.provider.browser-local-audio-transcription.description'),
|
||||
tasks: ['speech-to-text', 'automatic-speech-recognition', 'asr', 'stt'],
|
||||
icon: 'i-lobe-icons:huggingface',
|
||||
isAvailableBy: isBrowserAndMemoryEnough,
|
||||
capabilities: {
|
||||
transcription: {
|
||||
protocol: 'http',
|
||||
generateOutput: true,
|
||||
streamOutput: false,
|
||||
streamInput: false,
|
||||
},
|
||||
},
|
||||
createProviderConfig: ({ t }) => createLocalAudioConfigSchema(t),
|
||||
createProvider: createLocalTranscriptionProvider,
|
||||
validators: createLocalAudioValidators(),
|
||||
})
|
||||
@@ -0,0 +1,262 @@
|
||||
import { z } from 'zod'
|
||||
|
||||
import { defineProvider } from '../registry'
|
||||
|
||||
const mimoSpeechConfigSchema = z.object({
|
||||
apiKey: z.string(),
|
||||
baseUrl: z.string().default('https://api.xiaomimimo.com/v1/'),
|
||||
model: z.string().default('mimo-v2.5-tts'),
|
||||
voice: z.string().default('mimo_default'),
|
||||
format: z.string().default('wav'),
|
||||
stylePrompt: z.string().optional(),
|
||||
voiceSample: z.string().optional(),
|
||||
})
|
||||
|
||||
const mimoTranscriptionConfigSchema = z.object({
|
||||
apiKey: z.string(),
|
||||
baseUrl: z.string().default('https://api.xiaomimimo.com/v1/'),
|
||||
model: z.string().default('mimo-v2-omni'),
|
||||
})
|
||||
|
||||
type MimoSpeechConfig = z.input<typeof mimoSpeechConfigSchema>
|
||||
type MimoTranscriptionConfig = z.input<typeof mimoTranscriptionConfigSchema>
|
||||
type MimoConfig = MimoSpeechConfig | MimoTranscriptionConfig
|
||||
|
||||
function normalizeBaseUrl(baseUrl: string | undefined) {
|
||||
return `${(baseUrl || 'https://api.xiaomimimo.com/v1/').replace(/\/+$/, '')}/`
|
||||
}
|
||||
|
||||
function createMimoValidators<TConfig extends MimoConfig>(id: string) {
|
||||
return {
|
||||
validateConfig: [
|
||||
({ t }: { t: (key: string) => string }) => ({
|
||||
id: `${id}:check-config`,
|
||||
name: t('settings.pages.providers.catalog.edit.validators.openai-compatible.check-config.title'),
|
||||
validator: async (config: TConfig) => {
|
||||
const errors: Array<{ error: unknown }> = []
|
||||
if (!config.apiKey?.trim())
|
||||
errors.push({ error: new Error('API key is required.') })
|
||||
if (!config.baseUrl?.trim())
|
||||
errors.push({ error: new Error('Base URL is required.') })
|
||||
|
||||
return {
|
||||
errors,
|
||||
reason: errors.map(item => (item.error as Error).message).join(', '),
|
||||
reasonKey: '',
|
||||
valid: errors.length === 0,
|
||||
}
|
||||
},
|
||||
}),
|
||||
],
|
||||
}
|
||||
}
|
||||
|
||||
function createMimoSpeechProvider(config: MimoSpeechConfig) {
|
||||
const apiKey = config.apiKey?.trim() ?? ''
|
||||
const baseUrl = normalizeBaseUrl(config.baseUrl)
|
||||
const defaultModel = config.model || 'mimo-v2.5-tts'
|
||||
const defaultVoice = config.voice || 'mimo_default'
|
||||
const defaultFormat = config.format || 'wav'
|
||||
|
||||
return {
|
||||
speech: () => ({
|
||||
baseURL: baseUrl,
|
||||
model: defaultModel,
|
||||
fetch: async (_input: RequestInfo | URL, init?: RequestInit) => {
|
||||
if (!init?.body || typeof init.body !== 'string')
|
||||
throw new Error('Invalid request body')
|
||||
|
||||
const body = JSON.parse(init.body) as {
|
||||
input?: string
|
||||
model?: string
|
||||
response_format?: string
|
||||
style_prompt?: string
|
||||
voice_sample?: string
|
||||
voice?: string
|
||||
}
|
||||
const model = body.model || defaultModel
|
||||
const format = body.response_format || defaultFormat
|
||||
const stylePrompt = body.style_prompt?.trim() || config.stylePrompt?.trim() || ''
|
||||
const voiceSample = body.voice_sample?.trim() || config.voiceSample?.trim() || ''
|
||||
const userPrompt = model === 'mimo-v2.5-tts-voiceclone'
|
||||
? stylePrompt
|
||||
: stylePrompt || 'Use a natural, clear speaking style.'
|
||||
|
||||
const audio: Record<string, string> = { format }
|
||||
if (model === 'mimo-v2.5-tts-voiceclone') {
|
||||
if (!voiceSample)
|
||||
throw new Error('MiMo voice clone requires a base64 audio sample in data URI format.')
|
||||
audio.voice = voiceSample
|
||||
}
|
||||
else if (model === 'mimo-v2.5-tts') {
|
||||
audio.voice = body.voice || defaultVoice
|
||||
}
|
||||
|
||||
if (model === 'mimo-v2.5-tts-voicedesign' && !stylePrompt)
|
||||
throw new Error('MiMo voice design requires a style prompt in the user message.')
|
||||
|
||||
const response = await fetch(new URL('chat/completions', baseUrl), {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json', 'api-key': apiKey },
|
||||
body: JSON.stringify({
|
||||
model,
|
||||
messages: [
|
||||
{ role: 'user', content: userPrompt },
|
||||
{ role: 'assistant', content: body.input ?? '' },
|
||||
],
|
||||
audio,
|
||||
}),
|
||||
})
|
||||
if (!response.ok || !response.body)
|
||||
throw new Error(`MiMo TTS request failed: ${response.status} ${response.statusText}`)
|
||||
|
||||
const data = await response.json() as {
|
||||
choices?: Array<{ message?: { audio?: { data?: string } } }>
|
||||
}
|
||||
const audioBase64 = data.choices?.[0]?.message?.audio?.data
|
||||
if (!audioBase64)
|
||||
throw new Error('MiMo TTS response missing audio data')
|
||||
|
||||
const binary = atob(audioBase64)
|
||||
const bytes = new Uint8Array(binary.length)
|
||||
for (let index = 0; index < binary.length; index++)
|
||||
bytes[index] = binary.charCodeAt(index)
|
||||
|
||||
let contentType = `audio/${format}`
|
||||
if (format === 'wav')
|
||||
contentType = 'audio/wav'
|
||||
else if (format === 'mp3')
|
||||
contentType = 'audio/mpeg'
|
||||
|
||||
return new Response(bytes.buffer, {
|
||||
status: 200,
|
||||
headers: { 'Content-Type': contentType },
|
||||
})
|
||||
},
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
function audioFormatFromDataUri(dataUri: string) {
|
||||
const mimeType = dataUri.split(';')[0]?.split(':')[1] || 'audio/wav'
|
||||
const format = mimeType.split('/')[1] || 'wav'
|
||||
if (format === 'webm' || format === 'mp4')
|
||||
return format
|
||||
if (format === 'mpeg' || format === 'mp3')
|
||||
return 'mp3'
|
||||
return 'wav'
|
||||
}
|
||||
|
||||
function readBlobAsDataUri(file: Blob) {
|
||||
return new Promise<string>((resolve, reject) => {
|
||||
const reader = new FileReader()
|
||||
reader.onload = () => resolve(String(reader.result))
|
||||
reader.onerror = () => reject(new Error('Failed to read audio file'))
|
||||
reader.readAsDataURL(file)
|
||||
})
|
||||
}
|
||||
|
||||
function createMimoTranscriptionProvider(config: MimoTranscriptionConfig) {
|
||||
const apiKey = config.apiKey?.trim() ?? ''
|
||||
const baseUrl = normalizeBaseUrl(config.baseUrl)
|
||||
const defaultModel = config.model || 'mimo-v2-omni'
|
||||
|
||||
return {
|
||||
transcription: (model: string) => ({
|
||||
baseURL: baseUrl,
|
||||
model: model || defaultModel,
|
||||
headers: {},
|
||||
fetch: async (_input: RequestInfo | URL, init?: RequestInit) => {
|
||||
if (!(init?.body instanceof FormData))
|
||||
throw new Error('No audio file provided for transcription.')
|
||||
|
||||
const file = init.body.get('file')
|
||||
if (!(file instanceof Blob))
|
||||
throw new Error('No audio file provided for transcription.')
|
||||
|
||||
const modelName = String(init.body.get('model') || defaultModel)
|
||||
const dataUri = await readBlobAsDataUri(file)
|
||||
const base64Data = dataUri.split(',')[1]
|
||||
const response = await fetch(new URL('chat/completions', baseUrl), {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json', 'api-key': apiKey },
|
||||
body: JSON.stringify({
|
||||
model: modelName,
|
||||
messages: [{
|
||||
role: 'user',
|
||||
content: [
|
||||
{ type: 'text', text: 'Transcribe the audio content.' },
|
||||
{ type: 'input_audio', input_audio: { data: base64Data, format: audioFormatFromDataUri(dataUri) } },
|
||||
],
|
||||
}],
|
||||
}),
|
||||
})
|
||||
if (!response.ok) {
|
||||
const errorBody = await response.text().catch(() => '')
|
||||
throw new Error(`MiMo transcription failed: ${response.status} ${response.statusText}${errorBody ? ` — ${errorBody}` : ''}`)
|
||||
}
|
||||
|
||||
const data = await response.json() as { choices?: Array<{ message?: { content?: string } }> }
|
||||
return new Response(JSON.stringify({ text: data.choices?.[0]?.message?.content || '' }), {
|
||||
status: 200,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
})
|
||||
},
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
export const providerMimoAudioSpeech = defineProvider<MimoSpeechConfig>({
|
||||
id: 'mimo-audio-speech',
|
||||
name: 'Xiaomi MiMo',
|
||||
nameLocalize: ({ t }) => t('settings.pages.providers.provider.mimo.title'),
|
||||
description: 'api.xiaomimimo.com',
|
||||
descriptionLocalize: ({ t }) => t('settings.pages.providers.provider.mimo.description'),
|
||||
tasks: ['text-to-speech'],
|
||||
icon: 'i-simple-icons:xiaomi',
|
||||
createProviderConfig: () => mimoSpeechConfigSchema,
|
||||
createProvider: createMimoSpeechProvider,
|
||||
validationRequiredWhen: config => Boolean(config.apiKey?.trim() && config.baseUrl?.trim()),
|
||||
validators: createMimoValidators<MimoSpeechConfig>('mimo-audio-speech'),
|
||||
extraMethods: {
|
||||
listModels: async () => [
|
||||
{ id: 'mimo-v2.5-tts', name: 'MiMo v2.5 TTS', provider: 'mimo-audio-speech', description: 'Preset voice synthesis with the built-in MiMo voice list', deprecated: false },
|
||||
{ id: 'mimo-v2.5-tts-voicedesign', name: 'MiMo v2.5 TTS Voice Design', provider: 'mimo-audio-speech', description: 'Design a new voice from a natural language description', deprecated: false },
|
||||
{ id: 'mimo-v2.5-tts-voiceclone', name: 'MiMo v2.5 TTS Voice Clone', provider: 'mimo-audio-speech', description: 'Clone a voice from a base64-encoded audio sample', deprecated: false },
|
||||
],
|
||||
listVoices: async () => [
|
||||
{ id: 'mimo_default', name: 'MiMo-默认', provider: 'mimo-audio-speech', gender: 'female', languages: [{ code: 'en', title: 'English' }, { code: 'zh', title: 'Chinese' }] },
|
||||
{ id: '冰糖', name: '冰糖', provider: 'mimo-audio-speech', gender: 'female', languages: [{ code: 'zh', title: 'Chinese' }] },
|
||||
{ id: '茉莉', name: '茉莉', provider: 'mimo-audio-speech', gender: 'female', languages: [{ code: 'zh', title: 'Chinese' }] },
|
||||
{ id: '苏打', name: '苏打', provider: 'mimo-audio-speech', gender: 'male', languages: [{ code: 'zh', title: 'Chinese' }] },
|
||||
{ id: '白桦', name: '白桦', provider: 'mimo-audio-speech', gender: 'male', languages: [{ code: 'zh', title: 'Chinese' }] },
|
||||
{ id: 'Mia', name: 'Mia', provider: 'mimo-audio-speech', gender: 'female', languages: [{ code: 'en', title: 'English' }] },
|
||||
{ id: 'Chloe', name: 'Chloe', provider: 'mimo-audio-speech', gender: 'female', languages: [{ code: 'en', title: 'English' }] },
|
||||
{ id: 'Milo', name: 'Milo', provider: 'mimo-audio-speech', gender: 'male', languages: [{ code: 'en', title: 'English' }] },
|
||||
{ id: 'Dean', name: 'Dean', provider: 'mimo-audio-speech', gender: 'male', languages: [{ code: 'en', title: 'English' }] },
|
||||
],
|
||||
},
|
||||
})
|
||||
|
||||
export const providerMimoAudioTranscription = defineProvider<MimoTranscriptionConfig>({
|
||||
id: 'mimo-audio-transcription',
|
||||
name: 'Xiaomi MiMo',
|
||||
nameLocalize: ({ t }) => t('settings.pages.providers.provider.mimo.title'),
|
||||
description: 'api.xiaomimimo.com',
|
||||
descriptionLocalize: ({ t }) => t('settings.pages.providers.provider.mimo.description'),
|
||||
tasks: ['speech-to-text', 'automatic-speech-recognition', 'asr', 'stt'],
|
||||
icon: 'i-simple-icons:xiaomi',
|
||||
capabilities: {
|
||||
transcription: { protocol: 'http', generateOutput: true, streamOutput: false, streamInput: false },
|
||||
},
|
||||
createProviderConfig: () => mimoTranscriptionConfigSchema,
|
||||
createProvider: createMimoTranscriptionProvider,
|
||||
validationRequiredWhen: config => Boolean(config.apiKey?.trim() && config.baseUrl?.trim()),
|
||||
validators: createMimoValidators<MimoTranscriptionConfig>('mimo-audio-transcription'),
|
||||
extraMethods: {
|
||||
listModels: async () => [
|
||||
{ id: 'mimo-v2-omni', name: 'MiMo V2 Omni', provider: 'mimo-audio-transcription', description: 'Omni-modal model with native audio understanding and speech-to-text', contextLength: 256000, deprecated: false },
|
||||
{ id: 'mimo-v2.5', name: 'MiMo V2.5', provider: 'mimo-audio-transcription', description: 'Latest omni-modal model with audio understanding, 1M context', contextLength: 1_000_000, deprecated: false },
|
||||
],
|
||||
},
|
||||
})
|
||||
@@ -0,0 +1,152 @@
|
||||
import { z } from 'zod'
|
||||
|
||||
import { defineProvider } from '../registry'
|
||||
|
||||
const minimaxSpeechConfigSchema = z.object({
|
||||
apiKey: z.string(),
|
||||
baseUrl: z.string().default('https://api.minimax.io'),
|
||||
})
|
||||
|
||||
type MinimaxSpeechConfig = z.input<typeof minimaxSpeechConfigSchema>
|
||||
|
||||
export const providerMinimaxSpeech = defineProvider<MinimaxSpeechConfig>({
|
||||
id: 'minimax-speech',
|
||||
name: 'MiniMax Speech',
|
||||
nameLocalize: ({ t }) => t('settings.pages.providers.provider.minimax-speech.title'),
|
||||
description: 'minimax.io',
|
||||
descriptionLocalize: ({ t }) => t('settings.pages.providers.provider.minimax-speech.description'),
|
||||
tasks: ['text-to-speech'],
|
||||
icon: 'i-lobe-icons:minimax',
|
||||
iconColor: 'i-lobe-icons:minimax-color',
|
||||
createProviderConfig: () => minimaxSpeechConfigSchema,
|
||||
createProvider(config) {
|
||||
const apiKey = config.apiKey.trim()
|
||||
const baseUrl = (config.baseUrl || 'https://api.minimax.io').replace(/\/$/, '')
|
||||
|
||||
return {
|
||||
speech: () => ({
|
||||
baseURL: `${baseUrl}/v1/`,
|
||||
model: 'speech-2.8-hd',
|
||||
fetch: async (_input: RequestInfo | URL, init?: RequestInit) => {
|
||||
if (!init?.body || typeof init.body !== 'string')
|
||||
throw new Error('Invalid request body')
|
||||
|
||||
const body = JSON.parse(init.body) as { input?: string, voice?: string, model?: string }
|
||||
const response = await fetch(`${baseUrl}/v1/t2a_v2`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'Authorization': `Bearer ${apiKey}`,
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: body.model || 'speech-2.8-hd',
|
||||
text: body.input ?? '',
|
||||
stream: true,
|
||||
voice_setting: {
|
||||
voice_id: body.voice || 'English_Graceful_Lady',
|
||||
speed: 1,
|
||||
vol: 1,
|
||||
pitch: 0,
|
||||
},
|
||||
audio_setting: {
|
||||
sample_rate: 32000,
|
||||
bitrate: 128000,
|
||||
format: 'mp3',
|
||||
channel: 1,
|
||||
},
|
||||
}),
|
||||
})
|
||||
|
||||
if (!response.ok || !response.body)
|
||||
throw new Error(`MiniMax TTS request failed: ${response.status} ${response.statusText}`)
|
||||
|
||||
// MiniMax streams SSE events that contain hex-encoded audio chunks.
|
||||
const reader = response.body.getReader()
|
||||
const decoder = new TextDecoder()
|
||||
const audioChunks: Uint8Array[] = []
|
||||
let buffer = ''
|
||||
|
||||
while (true) {
|
||||
const { done, value } = await reader.read()
|
||||
if (done)
|
||||
break
|
||||
|
||||
buffer += decoder.decode(value, { stream: true })
|
||||
const lines = buffer.split('\n')
|
||||
buffer = lines.pop() || ''
|
||||
for (const line of lines) {
|
||||
if (!line.startsWith('data:'))
|
||||
continue
|
||||
|
||||
const json = line.slice(5).trim()
|
||||
if (!json || json === '[DONE]')
|
||||
continue
|
||||
|
||||
try {
|
||||
const event = JSON.parse(json) as { data?: { audio?: string, status?: number } }
|
||||
// Status 2 is the final summary. Its audio duplicates prior chunks.
|
||||
if (event.data?.audio && event.data.status !== 2) {
|
||||
const bytes = new Uint8Array(event.data.audio.length / 2)
|
||||
for (let index = 0; index < event.data.audio.length; index += 2)
|
||||
bytes[index / 2] = Number.parseInt(event.data.audio.slice(index, index + 2), 16)
|
||||
audioChunks.push(bytes)
|
||||
}
|
||||
}
|
||||
catch {
|
||||
// A malformed SSE event does not invalidate earlier audio chunks.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const combined = new Uint8Array(audioChunks.reduce((sum, chunk) => sum + chunk.length, 0))
|
||||
let offset = 0
|
||||
for (const chunk of audioChunks) {
|
||||
combined.set(chunk, offset)
|
||||
offset += chunk.length
|
||||
}
|
||||
|
||||
return new Response(combined.buffer, {
|
||||
status: 200,
|
||||
headers: { 'Content-Type': 'audio/mpeg' },
|
||||
})
|
||||
},
|
||||
}),
|
||||
}
|
||||
},
|
||||
validationRequiredWhen: config => Boolean(config.apiKey?.trim()),
|
||||
validators: {
|
||||
validateConfig: [
|
||||
({ t }) => ({
|
||||
id: 'minimax-speech:check-config',
|
||||
name: t('settings.pages.providers.catalog.edit.validators.openai-compatible.check-config.title'),
|
||||
validator: async (config) => {
|
||||
const valid = Boolean(config.apiKey?.trim())
|
||||
return {
|
||||
errors: valid ? [] : [{ error: new Error('API key is required.') }],
|
||||
reason: valid ? '' : 'API key is required.',
|
||||
reasonKey: '',
|
||||
valid,
|
||||
}
|
||||
},
|
||||
}),
|
||||
],
|
||||
},
|
||||
extraMethods: {
|
||||
listModels: async () => [
|
||||
{ id: 'speech-2.8-hd', name: 'Speech 2.8 HD', provider: 'minimax-speech', description: 'High-definition TTS model with natural prosody', deprecated: false },
|
||||
{ id: 'speech-2.8-turbo', name: 'Speech 2.8 Turbo', provider: 'minimax-speech', description: 'Fast TTS model for low-latency scenarios', deprecated: false },
|
||||
],
|
||||
listVoices: async () => [
|
||||
{ id: 'English_Graceful_Lady', name: 'Graceful Lady', provider: 'minimax-speech', gender: 'female', languages: [{ code: 'en', title: 'English' }] },
|
||||
{ id: 'English_Insightful_Speaker', name: 'Insightful Speaker', provider: 'minimax-speech', gender: 'male', languages: [{ code: 'en', title: 'English' }] },
|
||||
{ id: 'English_radiant_girl', name: 'Radiant Girl', provider: 'minimax-speech', gender: 'female', languages: [{ code: 'en', title: 'English' }] },
|
||||
{ id: 'English_Persuasive_Man', name: 'Persuasive Man', provider: 'minimax-speech', gender: 'male', languages: [{ code: 'en', title: 'English' }] },
|
||||
{ id: 'English_Lucky_Robot', name: 'Lucky Robot', provider: 'minimax-speech', gender: 'neutral', languages: [{ code: 'en', title: 'English' }] },
|
||||
{ id: 'English_expressive_narrator', name: 'Expressive Narrator', provider: 'minimax-speech', gender: 'neutral', languages: [{ code: 'en', title: 'English' }] },
|
||||
{ id: 'Mandarin_Gentle_Woman', name: 'Gentle Woman', provider: 'minimax-speech', gender: 'female', languages: [{ code: 'zh', title: 'Chinese' }] },
|
||||
{ id: 'Mandarin_Steadfast_Man', name: 'Steadfast Man', provider: 'minimax-speech', gender: 'male', languages: [{ code: 'zh', title: 'Chinese' }] },
|
||||
{ id: 'Mandarin_Sweet_Girl', name: 'Sweet Girl', provider: 'minimax-speech', gender: 'female', languages: [{ code: 'zh', title: 'Chinese' }] },
|
||||
{ id: 'Mandarin_Magnetic_Gentleman', name: 'Magnetic Gentleman', provider: 'minimax-speech', gender: 'male', languages: [{ code: 'zh', title: 'Chinese' }] },
|
||||
],
|
||||
},
|
||||
})
|
||||
@@ -0,0 +1,266 @@
|
||||
import type { ComposerTranslation } from 'vue-i18n'
|
||||
|
||||
import { createOpenAI } from '@xsai-ext/providers/create'
|
||||
import { listModels } from '@xsai/model'
|
||||
import { z } from 'zod'
|
||||
|
||||
import { defineProvider } from '../registry'
|
||||
|
||||
const OPENAI_BASE_URL = 'https://api.openai.com/v1/'
|
||||
|
||||
const openAIAudioConfigSchema = z.object({
|
||||
apiKey: z.string(),
|
||||
baseUrl: z.string().default(OPENAI_BASE_URL),
|
||||
})
|
||||
|
||||
const openAICompatibleAudioConfigSchema = z.object({
|
||||
apiKey: z.string(),
|
||||
baseUrl: z.string().default(''),
|
||||
})
|
||||
|
||||
type OpenAIAudioConfig = z.input<typeof openAIAudioConfigSchema>
|
||||
type OpenAICompatibleAudioConfig = z.input<typeof openAICompatibleAudioConfigSchema>
|
||||
type AudioConfig = OpenAIAudioConfig | OpenAICompatibleAudioConfig
|
||||
|
||||
function createAudioConfigSchema<T extends typeof openAIAudioConfigSchema | typeof openAICompatibleAudioConfigSchema>(schema: T, t: ComposerTranslation) {
|
||||
return schema.extend({
|
||||
apiKey: schema.shape.apiKey.meta({
|
||||
labelLocalized: t('settings.pages.providers.catalog.edit.config.common.fields.field.api-key.label'),
|
||||
descriptionLocalized: t('settings.pages.providers.catalog.edit.config.common.fields.field.api-key.description'),
|
||||
placeholderLocalized: t('settings.pages.providers.catalog.edit.config.common.fields.field.api-key.placeholder'),
|
||||
type: 'password',
|
||||
}),
|
||||
baseUrl: schema.shape.baseUrl.meta({
|
||||
labelLocalized: t('settings.pages.providers.catalog.edit.config.common.fields.field.base-url.label'),
|
||||
descriptionLocalized: t('settings.pages.providers.catalog.edit.config.common.fields.field.base-url.description'),
|
||||
placeholderLocalized: t('settings.pages.providers.catalog.edit.config.common.fields.field.base-url.placeholder'),
|
||||
}),
|
||||
})
|
||||
}
|
||||
|
||||
function normalizeBaseUrl(baseUrl: string | undefined) {
|
||||
const value = baseUrl?.trim() ?? ''
|
||||
return value && !value.endsWith('/') ? `${value}/` : value
|
||||
}
|
||||
|
||||
function createAudioProvider(config: AudioConfig) {
|
||||
return createOpenAI(config.apiKey.trim(), normalizeBaseUrl(config.baseUrl))
|
||||
}
|
||||
|
||||
function createTranscriptionProvider(config: AudioConfig) {
|
||||
const provider = createAudioProvider(config)
|
||||
const transcription = provider.transcription.bind(provider)
|
||||
provider.transcription = (model: string, extraOptions?: Record<string, unknown>) => ({
|
||||
...transcription(model),
|
||||
...extraOptions,
|
||||
})
|
||||
return provider
|
||||
}
|
||||
|
||||
function createAudioValidators<TConfig extends AudioConfig>() {
|
||||
return {
|
||||
validateConfig: [
|
||||
({ t }: { t: ComposerTranslation }) => ({
|
||||
id: 'openai-audio:check-config',
|
||||
name: t('settings.pages.providers.catalog.edit.validators.openai-compatible.check-config.title'),
|
||||
validator: async (config: TConfig) => {
|
||||
const errors: Array<{ error: unknown }> = []
|
||||
const apiKey = config.apiKey?.trim() ?? ''
|
||||
const baseUrl = config.baseUrl?.trim() ?? ''
|
||||
|
||||
if (!apiKey)
|
||||
errors.push({ error: new Error('API Key is required') })
|
||||
if (!baseUrl)
|
||||
errors.push({ error: new Error('Base URL is required.') })
|
||||
|
||||
if (baseUrl) {
|
||||
try {
|
||||
const url = new URL(baseUrl)
|
||||
if (!url.host)
|
||||
errors.push({ error: new Error('Base URL is not absolute. Try to include a scheme (http:// or https://).') })
|
||||
else if (!baseUrl.endsWith('/'))
|
||||
errors.push({ error: new Error('Base URL must end with a trailing slash (/).') })
|
||||
}
|
||||
catch {
|
||||
errors.push({ error: new Error('Base URL is not absolute. Try to include a scheme (http:// or https://).') })
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
errors,
|
||||
reason: errors.map(item => (item.error as Error).message).join(', '),
|
||||
reasonKey: '',
|
||||
valid: errors.length === 0,
|
||||
}
|
||||
},
|
||||
}),
|
||||
],
|
||||
}
|
||||
}
|
||||
|
||||
const openAISpeechModels = [
|
||||
{
|
||||
id: 'tts-1',
|
||||
name: 'TTS-1',
|
||||
provider: 'openai-audio-speech',
|
||||
description: 'Optimized for real-time text-to-speech tasks',
|
||||
contextLength: 0,
|
||||
deprecated: false,
|
||||
},
|
||||
{
|
||||
id: 'tts-1-hd',
|
||||
name: 'TTS-1-HD',
|
||||
provider: 'openai-audio-speech',
|
||||
description: 'Higher fidelity audio output',
|
||||
contextLength: 0,
|
||||
deprecated: false,
|
||||
},
|
||||
{
|
||||
id: 'gpt-4o-mini-tts',
|
||||
name: 'GPT-4o Mini TTS',
|
||||
provider: 'openai-audio-speech',
|
||||
description: 'GPT-4o Mini optimized for text-to-speech',
|
||||
contextLength: 0,
|
||||
deprecated: false,
|
||||
},
|
||||
{
|
||||
id: 'gpt-4o-mini-tts-2025-12-15',
|
||||
name: 'GPT-4o Mini TTS (2025-12-15)',
|
||||
provider: 'openai-audio-speech',
|
||||
description: 'GPT-4o Mini TTS snapshot from 2025-12-15',
|
||||
contextLength: 0,
|
||||
deprecated: false,
|
||||
},
|
||||
]
|
||||
|
||||
// OpenAI does not provide a voice-list endpoint. This list follows the
|
||||
// create-speech API. The TTS-1 models support nine voices. GPT-4o Mini TTS
|
||||
// also supports ballad, verse, marin, and cedar.
|
||||
const openAISpeechVoices = [
|
||||
{ id: 'alloy', models: ['tts-1', 'tts-1-hd', 'gpt-4o-mini-tts', 'gpt-4o-mini-tts-2025-12-15'] },
|
||||
{ id: 'ash', models: ['tts-1', 'tts-1-hd', 'gpt-4o-mini-tts', 'gpt-4o-mini-tts-2025-12-15'] },
|
||||
{ id: 'ballad', models: ['gpt-4o-mini-tts', 'gpt-4o-mini-tts-2025-12-15'] },
|
||||
{ id: 'coral', models: ['tts-1', 'tts-1-hd', 'gpt-4o-mini-tts', 'gpt-4o-mini-tts-2025-12-15'] },
|
||||
{ id: 'echo', models: ['tts-1', 'tts-1-hd', 'gpt-4o-mini-tts', 'gpt-4o-mini-tts-2025-12-15'] },
|
||||
{ id: 'fable', models: ['tts-1', 'tts-1-hd', 'gpt-4o-mini-tts', 'gpt-4o-mini-tts-2025-12-15'] },
|
||||
{ id: 'onyx', models: ['tts-1', 'tts-1-hd', 'gpt-4o-mini-tts', 'gpt-4o-mini-tts-2025-12-15'] },
|
||||
{ id: 'nova', models: ['tts-1', 'tts-1-hd', 'gpt-4o-mini-tts', 'gpt-4o-mini-tts-2025-12-15'] },
|
||||
{ id: 'sage', models: ['tts-1', 'tts-1-hd', 'gpt-4o-mini-tts', 'gpt-4o-mini-tts-2025-12-15'] },
|
||||
{ id: 'shimmer', models: ['tts-1', 'tts-1-hd', 'gpt-4o-mini-tts', 'gpt-4o-mini-tts-2025-12-15'] },
|
||||
{ id: 'verse', models: ['gpt-4o-mini-tts', 'gpt-4o-mini-tts-2025-12-15'] },
|
||||
{ id: 'marin', models: ['gpt-4o-mini-tts', 'gpt-4o-mini-tts-2025-12-15'] },
|
||||
{ id: 'cedar', models: ['gpt-4o-mini-tts', 'gpt-4o-mini-tts-2025-12-15'] },
|
||||
].map(voice => ({
|
||||
id: voice.id,
|
||||
name: `${voice.id[0].toUpperCase()}${voice.id.slice(1)}`,
|
||||
provider: 'openai-audio-speech',
|
||||
languages: [],
|
||||
compatibleModels: voice.models,
|
||||
}))
|
||||
|
||||
const openAITranscriptionModels = [
|
||||
['gpt-4o-transcribe', 'GPT-4o Transcribe', 'High-quality transcription model'],
|
||||
['gpt-4o-mini-transcribe', 'GPT-4o Mini Transcribe', 'Faster, cost-effective transcription model'],
|
||||
['gpt-4o-mini-transcribe-2025-12-15', 'GPT-4o Mini Transcribe (2025-12-15)', 'GPT-4o Mini Transcribe snapshot from 2025-12-15'],
|
||||
['whisper-1', 'Whisper-1', 'Powered by our open source Whisper V2 model'],
|
||||
['gpt-4o-transcribe-diarize', 'GPT-4o Transcribe Diarize', 'Transcription with speaker diarization'],
|
||||
].map(([id, name, description]) => ({
|
||||
id,
|
||||
name,
|
||||
provider: 'openai-audio-transcription',
|
||||
description,
|
||||
contextLength: 0,
|
||||
deprecated: false,
|
||||
}))
|
||||
|
||||
export const providerOpenAIAudioSpeech = defineProvider<OpenAIAudioConfig>({
|
||||
id: 'openai-audio-speech',
|
||||
name: 'OpenAI',
|
||||
nameLocalize: ({ t }) => t('settings.pages.providers.provider.openai.title'),
|
||||
description: 'openai.com',
|
||||
descriptionLocalize: ({ t }) => t('settings.pages.providers.provider.openai.description'),
|
||||
tasks: ['text-to-speech'],
|
||||
icon: 'i-lobe-icons:openai',
|
||||
createProviderConfig: ({ t }) => createAudioConfigSchema(openAIAudioConfigSchema, t),
|
||||
createProvider: createAudioProvider,
|
||||
validationRequiredWhen: config => Boolean(config.apiKey?.trim() && config.baseUrl?.trim()),
|
||||
validators: createAudioValidators<OpenAIAudioConfig>(),
|
||||
extraMethods: {
|
||||
listModels: async () => openAISpeechModels,
|
||||
listVoices: async () => openAISpeechVoices,
|
||||
},
|
||||
})
|
||||
|
||||
export const providerOpenAICompatibleAudioSpeech = defineProvider<OpenAICompatibleAudioConfig>({
|
||||
id: 'openai-compatible-audio-speech',
|
||||
name: 'OpenAI Compatible',
|
||||
nameLocalize: ({ t }) => t('settings.pages.providers.provider.openai-compatible.title'),
|
||||
description: 'Connect to any API that follows the OpenAI specification.',
|
||||
descriptionLocalize: ({ t }) => t('settings.pages.providers.provider.openai-compatible.description'),
|
||||
tasks: ['text-to-speech'],
|
||||
icon: 'i-lobe-icons:openai',
|
||||
createProviderConfig: ({ t }) => createAudioConfigSchema(openAICompatibleAudioConfigSchema, t),
|
||||
createProvider: createAudioProvider,
|
||||
validationRequiredWhen: config => Boolean(config.apiKey?.trim() && config.baseUrl?.trim()),
|
||||
validators: createAudioValidators<OpenAICompatibleAudioConfig>(),
|
||||
extraMethods: {
|
||||
listVoices: async () => [],
|
||||
listModels: async (config) => {
|
||||
const apiKey = config.apiKey?.trim() ?? ''
|
||||
const baseUrl = normalizeBaseUrl(config.baseUrl)
|
||||
if (!apiKey || !baseUrl)
|
||||
return []
|
||||
|
||||
const models = await listModels({ apiKey, baseURL: baseUrl })
|
||||
return models
|
||||
.filter(model => model.id.toLowerCase().includes('tts'))
|
||||
.map(model => ({
|
||||
id: model.id,
|
||||
name: model.id,
|
||||
provider: 'openai-compatible-audio-speech',
|
||||
description: '',
|
||||
contextLength: 0,
|
||||
deprecated: false,
|
||||
}))
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
export const providerOpenAIAudioTranscription = defineProvider<OpenAIAudioConfig>({
|
||||
id: 'openai-audio-transcription',
|
||||
name: 'OpenAI',
|
||||
nameLocalize: ({ t }) => t('settings.pages.providers.provider.openai.title'),
|
||||
description: 'openai.com',
|
||||
descriptionLocalize: ({ t }) => t('settings.pages.providers.provider.openai.description'),
|
||||
tasks: ['speech-to-text', 'automatic-speech-recognition', 'asr', 'stt'],
|
||||
icon: 'i-lobe-icons:openai',
|
||||
capabilities: {
|
||||
transcription: { protocol: 'http', generateOutput: true, streamOutput: false, streamInput: false },
|
||||
},
|
||||
createProviderConfig: ({ t }) => createAudioConfigSchema(openAIAudioConfigSchema, t),
|
||||
createProvider: createTranscriptionProvider,
|
||||
validationRequiredWhen: config => Boolean(config.apiKey?.trim() && config.baseUrl?.trim()),
|
||||
validators: createAudioValidators<OpenAIAudioConfig>(),
|
||||
extraMethods: {
|
||||
listModels: async () => openAITranscriptionModels,
|
||||
},
|
||||
})
|
||||
|
||||
export const providerOpenAICompatibleAudioTranscription = defineProvider<OpenAICompatibleAudioConfig>({
|
||||
id: 'openai-compatible-audio-transcription',
|
||||
name: 'OpenAI Compatible',
|
||||
nameLocalize: ({ t }) => t('settings.pages.providers.provider.openai-compatible.title'),
|
||||
description: 'Connect to any API that follows the OpenAI specification.',
|
||||
descriptionLocalize: ({ t }) => t('settings.pages.providers.provider.openai-compatible.description'),
|
||||
tasks: ['speech-to-text', 'automatic-speech-recognition', 'asr', 'stt'],
|
||||
icon: 'i-lobe-icons:openai',
|
||||
capabilities: {
|
||||
transcription: { protocol: 'http', generateOutput: true, streamOutput: false, streamInput: false },
|
||||
},
|
||||
createProviderConfig: ({ t }) => createAudioConfigSchema(openAICompatibleAudioConfigSchema, t),
|
||||
createProvider: createTranscriptionProvider,
|
||||
validationRequiredWhen: config => Boolean(config.apiKey?.trim() && config.baseUrl?.trim()),
|
||||
validators: createAudioValidators<OpenAICompatibleAudioConfig>(),
|
||||
// Transcription model names are not reliably available from /v1/models.
|
||||
extraMethods: { listModels: async () => [] },
|
||||
})
|
||||
@@ -0,0 +1,200 @@
|
||||
import { toWavFromPCM16 } from '@proj-airi/audio/encoding'
|
||||
import { z } from 'zod'
|
||||
|
||||
import { OPENROUTER_ATTRIBUTION_HEADERS } from '../openrouter-ai'
|
||||
import { defineProvider } from '../registry'
|
||||
|
||||
const DEFAULT_BASE_URL = 'https://openrouter.ai/api/v1/'
|
||||
const DEFAULT_MODEL = 'openai/gpt-audio-mini'
|
||||
|
||||
const openRouterAudioConfigSchema = z.object({
|
||||
apiKey: z.string(),
|
||||
baseUrl: z.string().default(DEFAULT_BASE_URL),
|
||||
})
|
||||
|
||||
type OpenRouterAudioConfig = z.input<typeof openRouterAudioConfigSchema>
|
||||
|
||||
const openAIVoices = [
|
||||
'alloy',
|
||||
'ash',
|
||||
'ballad',
|
||||
'coral',
|
||||
'echo',
|
||||
'fable',
|
||||
'onyx',
|
||||
'nova',
|
||||
'sage',
|
||||
'shimmer',
|
||||
'verse',
|
||||
'marin',
|
||||
'cedar',
|
||||
] as const
|
||||
|
||||
function normalizeBaseUrl(baseUrl: string | undefined) {
|
||||
const value = baseUrl?.trim() || DEFAULT_BASE_URL
|
||||
return value.endsWith('/') ? value : `${value}/`
|
||||
}
|
||||
|
||||
function ttsPrompt(input: string) {
|
||||
return `Read this text aloud exactly as written, without any commentary or extra words:\n\n${input}`
|
||||
}
|
||||
|
||||
async function collectAudioChunks(body: ReadableStream<Uint8Array>) {
|
||||
const reader = body.getReader()
|
||||
const decoder = new TextDecoder()
|
||||
const chunks: string[] = []
|
||||
let buffer = ''
|
||||
let done = false
|
||||
|
||||
while (!done) {
|
||||
const result = await reader.read()
|
||||
if (result.done)
|
||||
break
|
||||
|
||||
buffer += decoder.decode(result.value, { stream: true })
|
||||
const lines = buffer.split('\n')
|
||||
buffer = lines.pop() || ''
|
||||
for (const line of lines) {
|
||||
if (!line.startsWith('data: '))
|
||||
continue
|
||||
|
||||
const data = line.slice('data: '.length).trim()
|
||||
if (data === '[DONE]') {
|
||||
done = true
|
||||
break
|
||||
}
|
||||
|
||||
try {
|
||||
const event = JSON.parse(data) as {
|
||||
choices?: Array<{ delta?: { audio?: { data?: string } } }>
|
||||
}
|
||||
const audio = event.choices?.[0]?.delta?.audio?.data
|
||||
if (audio)
|
||||
chunks.push(audio)
|
||||
}
|
||||
catch (error) {
|
||||
console.warn('Skipping malformed SSE chunk from OpenRouter audio stream:', data, error)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return chunks
|
||||
}
|
||||
|
||||
function decodeBase64Pcm(chunks: string[]) {
|
||||
const binary = atob(chunks.join(''))
|
||||
const bytes = new Uint8Array(binary.length)
|
||||
for (let index = 0; index < binary.length; index++)
|
||||
bytes[index] = binary.charCodeAt(index)
|
||||
return bytes
|
||||
}
|
||||
|
||||
function createAudioFetch(apiKey: string, baseUrl: string, model: string) {
|
||||
return async (_input: RequestInfo | URL, init?: RequestInit) => {
|
||||
if (!init?.body || typeof init.body !== 'string')
|
||||
throw new Error('Invalid request body')
|
||||
|
||||
const body = JSON.parse(init.body) as { input?: string, voice?: string }
|
||||
const response = await globalThis.fetch(new URL('chat/completions', baseUrl), {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Authorization': `Bearer ${apiKey}`,
|
||||
'Content-Type': 'application/json',
|
||||
...OPENROUTER_ATTRIBUTION_HEADERS,
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model,
|
||||
messages: [{ role: 'user', content: ttsPrompt(body.input ?? '') }],
|
||||
modalities: ['text', 'audio'],
|
||||
audio: { voice: body.voice, format: 'pcm16' },
|
||||
stream: true,
|
||||
}),
|
||||
})
|
||||
if (!response.ok)
|
||||
throw new Error(`OpenRouter audio request failed: ${response.status} ${await response.text()}`)
|
||||
if (!response.body)
|
||||
throw new Error('OpenRouter audio response has no body')
|
||||
|
||||
const wav = toWavFromPCM16(decodeBase64Pcm(await collectAudioChunks(response.body)), 24000)
|
||||
return new Response(new Blob([wav], { type: 'audio/wav' }), {
|
||||
status: 200,
|
||||
headers: { 'Content-Type': 'audio/wav' },
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
export const providerOpenRouterAudioSpeech = defineProvider<OpenRouterAudioConfig>({
|
||||
id: 'openrouter-audio-speech',
|
||||
name: 'OpenRouter',
|
||||
nameLocalize: ({ t }) => t('settings.pages.providers.provider.openrouter-audio-speech.title'),
|
||||
description: 'openrouter.ai',
|
||||
descriptionLocalize: ({ t }) => t('settings.pages.providers.provider.openrouter-audio-speech.description'),
|
||||
tasks: ['text-to-speech'],
|
||||
icon: 'i-lobe-icons:openrouter',
|
||||
createProviderConfig: () => openRouterAudioConfigSchema,
|
||||
createProvider(config) {
|
||||
const apiKey = config.apiKey.trim()
|
||||
const baseUrl = normalizeBaseUrl(config.baseUrl)
|
||||
return {
|
||||
speech: (model?: string) => {
|
||||
const resolvedModel = model || DEFAULT_MODEL
|
||||
return {
|
||||
baseURL: baseUrl,
|
||||
model: resolvedModel,
|
||||
fetch: createAudioFetch(apiKey, baseUrl, resolvedModel),
|
||||
}
|
||||
},
|
||||
}
|
||||
},
|
||||
validationRequiredWhen: config => Boolean(config.apiKey?.trim()),
|
||||
validators: {
|
||||
validateConfig: [
|
||||
({ t }) => ({
|
||||
id: 'openrouter-audio-speech:check-config',
|
||||
name: t('settings.pages.providers.catalog.edit.validators.openai-compatible.check-config.title'),
|
||||
validator: async (config) => {
|
||||
const valid = Boolean(config.apiKey?.trim())
|
||||
return {
|
||||
errors: valid ? [] : [{ error: new Error('API Key is required.') }],
|
||||
reason: valid ? '' : 'API Key is required.',
|
||||
reasonKey: '',
|
||||
valid,
|
||||
}
|
||||
},
|
||||
}),
|
||||
],
|
||||
},
|
||||
extraMethods: {
|
||||
listModels: async (config) => {
|
||||
try {
|
||||
const response = await fetch(new URL('models?output_modality=audio', normalizeBaseUrl(config.baseUrl)), {
|
||||
headers: OPENROUTER_ATTRIBUTION_HEADERS,
|
||||
})
|
||||
if (!response.ok)
|
||||
return []
|
||||
|
||||
const data = await response.json() as {
|
||||
data?: Array<{ id: string, name?: string, description?: string, context_length?: number }>
|
||||
}
|
||||
return (data.data ?? []).map(model => ({
|
||||
id: model.id,
|
||||
name: model.name || model.id,
|
||||
provider: 'openrouter-audio-speech',
|
||||
description: model.description || '',
|
||||
contextLength: model.context_length || 0,
|
||||
deprecated: false,
|
||||
}))
|
||||
}
|
||||
catch (error) {
|
||||
console.error('Failed to fetch OpenRouter audio models:', error)
|
||||
return []
|
||||
}
|
||||
},
|
||||
listVoices: async () => openAIVoices.map(id => ({
|
||||
id,
|
||||
name: `${id[0].toUpperCase()}${id.slice(1)}`,
|
||||
provider: 'openrouter-audio-speech',
|
||||
languages: [],
|
||||
})),
|
||||
},
|
||||
})
|
||||
@@ -0,0 +1,107 @@
|
||||
import { errorMessageFrom } from '@moeru/std'
|
||||
import { createPlayer2 } from '@xsai-ext/providers/special/create'
|
||||
import { z } from 'zod'
|
||||
|
||||
import { defineProvider } from '../registry'
|
||||
|
||||
const player2ConfigSchema = z.object({
|
||||
baseUrl: z.string().default('http://localhost:4315/v1/'),
|
||||
})
|
||||
|
||||
type Player2Config = z.input<typeof player2ConfigSchema>
|
||||
type Player2VoiceLanguage = keyof typeof player2VoiceLanguages
|
||||
|
||||
const player2VoiceLanguages = {
|
||||
american_english: { code: 'en', title: 'English' },
|
||||
british_english: { code: 'en', title: 'English' },
|
||||
japanese: { code: 'ja', title: 'Japanese' },
|
||||
mandarin_chinese: { code: 'zh', title: 'Chinese' },
|
||||
spanish: { code: 'es', title: 'Spanish' },
|
||||
french: { code: 'fr', title: 'French' },
|
||||
hindi: { code: 'hi', title: 'Hindi' },
|
||||
italian: { code: 'it', title: 'Italian' },
|
||||
brazilian_portuguese: { code: 'pt', title: 'Portuguese' },
|
||||
} as const
|
||||
|
||||
function normalizeBaseUrl(baseUrl: string | undefined) {
|
||||
const value = baseUrl?.trim() ?? 'http://localhost:4315/v1/'
|
||||
return value.endsWith('/') ? value : `${value}/`
|
||||
}
|
||||
|
||||
export const providerPlayer2Speech = defineProvider<Player2Config>({
|
||||
id: 'player2-speech',
|
||||
name: 'Player2 Speech',
|
||||
nameLocalize: ({ t }) => t('settings.pages.providers.provider.player2.title'),
|
||||
description: 'player2.game',
|
||||
descriptionLocalize: ({ t }) => t('settings.pages.providers.provider.player2.description'),
|
||||
tasks: ['text-to-speech'],
|
||||
icon: 'i-lobe-icons:player2',
|
||||
createProviderConfig: () => player2ConfigSchema,
|
||||
createProvider: config => createPlayer2(normalizeBaseUrl(config.baseUrl), 'airi'),
|
||||
validationRequiredWhen: config => Boolean(config.baseUrl?.trim()),
|
||||
validators: {
|
||||
validateConfig: [
|
||||
({ t }) => ({
|
||||
id: 'player2-speech:check-config',
|
||||
name: t('settings.pages.providers.catalog.edit.validators.openai-compatible.check-config.title'),
|
||||
validator: async (config) => {
|
||||
const valid = Boolean(config.baseUrl?.trim())
|
||||
const reason = valid ? '' : 'Base URL is required. Default to http://localhost:4315/v1/'
|
||||
return { errors: valid ? [] : [{ error: new Error(reason) }], reason, reasonKey: '', valid }
|
||||
},
|
||||
}),
|
||||
],
|
||||
validateProvider: [
|
||||
({ t }) => ({
|
||||
id: 'player2-speech:check-connectivity',
|
||||
name: t('settings.pages.providers.catalog.edit.validators.openai-compatible.check-connectivity.title'),
|
||||
validator: async (config) => {
|
||||
const controller = new AbortController()
|
||||
const timeout = setTimeout(() => controller.abort(), 5000)
|
||||
try {
|
||||
const response = await fetch(new URL('health', normalizeBaseUrl(config.baseUrl)), {
|
||||
headers: { 'player2-game-key': 'airi' },
|
||||
signal: controller.signal,
|
||||
})
|
||||
if (!response.ok) {
|
||||
const reason = `Player2 speech unreachable: HTTP ${response.status} ${response.statusText}`
|
||||
return { errors: [{ error: new Error(reason) }], reason, reasonKey: '', valid: false }
|
||||
}
|
||||
}
|
||||
catch (error) {
|
||||
const reason = `Player2 speech connection failed: ${errorMessageFrom(error) ?? 'Unknown error'}`
|
||||
return { errors: [{ error }], reason, reasonKey: '', valid: false }
|
||||
}
|
||||
finally {
|
||||
clearTimeout(timeout)
|
||||
}
|
||||
|
||||
return { errors: [], reason: '', reasonKey: '', valid: true }
|
||||
},
|
||||
}),
|
||||
],
|
||||
},
|
||||
extraMethods: {
|
||||
listModels: async () => [{
|
||||
id: 'player2-tts',
|
||||
name: 'Player2 Speech',
|
||||
provider: 'player2-speech',
|
||||
description: 'Default model for Player2 speech endpoint',
|
||||
contextLength: 0,
|
||||
deprecated: false,
|
||||
}],
|
||||
listVoices: async (config) => {
|
||||
const response = await fetch(new URL('tts/voices', normalizeBaseUrl(config.baseUrl)))
|
||||
const data = await response.json() as {
|
||||
voices?: Array<{ id: string, language: Player2VoiceLanguage, name: string, gender: string }>
|
||||
}
|
||||
return (data.voices ?? []).map(voice => ({
|
||||
id: voice.id,
|
||||
name: voice.name,
|
||||
provider: 'player2-speech',
|
||||
gender: voice.gender,
|
||||
languages: [player2VoiceLanguages[voice.language]],
|
||||
}))
|
||||
},
|
||||
},
|
||||
})
|
||||
@@ -0,0 +1,122 @@
|
||||
import type { ComposerTranslation } from 'vue-i18n'
|
||||
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { z } from 'zod'
|
||||
|
||||
import { providerBrowserWebSpeechApi } from './browser-web-speech-api'
|
||||
import { providerElevenLabs } from './elevenlabs'
|
||||
import {
|
||||
providerAppLocalAudioSpeech,
|
||||
providerAppLocalAudioTranscription,
|
||||
providerBrowserLocalAudioSpeech,
|
||||
providerBrowserLocalAudioTranscription,
|
||||
} from './local-audio'
|
||||
import { getDefinedProvider } from './registry'
|
||||
import { providerSpeechNoop } from './speech-noop'
|
||||
|
||||
import './index'
|
||||
|
||||
const translate = ((key: string) => key) as unknown as ComposerTranslation
|
||||
|
||||
describe('migrated provider definitions', () => {
|
||||
it('registers every provider that moved out of the legacy store', () => {
|
||||
const providerIds = [
|
||||
'speech-noop',
|
||||
'app-local-audio-speech',
|
||||
'app-local-audio-transcription',
|
||||
'browser-local-audio-speech',
|
||||
'browser-local-audio-transcription',
|
||||
'openai-audio-speech',
|
||||
'openai-compatible-audio-speech',
|
||||
'openai-audio-transcription',
|
||||
'openai-compatible-audio-transcription',
|
||||
'aliyun-nls-transcription',
|
||||
'browser-web-speech-api',
|
||||
'elevenlabs',
|
||||
'deepgram-tts',
|
||||
'microsoft-speech',
|
||||
'index-tts-vllm',
|
||||
'alibaba-cloud-model-studio',
|
||||
'volcengine',
|
||||
'minimax-speech',
|
||||
'openrouter-audio-speech',
|
||||
'mimo-audio-speech',
|
||||
'comet-api-speech',
|
||||
'comet-api-transcription',
|
||||
'mimo-audio-transcription',
|
||||
'player2-speech',
|
||||
'kokoro-local',
|
||||
'google-gemini-audio-speech',
|
||||
]
|
||||
|
||||
for (const providerId of providerIds)
|
||||
expect(getDefinedProvider(providerId), providerId).toBeDefined()
|
||||
})
|
||||
|
||||
it('creates the no-op speech provider through ProviderDefinition', () => {
|
||||
const provider = providerSpeechNoop.createProvider({})
|
||||
|
||||
expect(provider).toHaveProperty('speech')
|
||||
expect('speech' in provider && provider.speech('unused')).toMatchObject({
|
||||
baseURL: 'http://speech-noop.invalid/v1/',
|
||||
model: 'noop',
|
||||
})
|
||||
})
|
||||
|
||||
it('keeps local audio providers split by inference task', () => {
|
||||
expect(providerAppLocalAudioSpeech.tasks).toContain('text-to-speech')
|
||||
expect(providerBrowserLocalAudioSpeech.tasks).toContain('text-to-speech')
|
||||
expect(providerAppLocalAudioTranscription.tasks).toContain('speech-to-text')
|
||||
expect(providerBrowserLocalAudioTranscription.tasks).toContain('speech-to-text')
|
||||
expect(providerAppLocalAudioTranscription.capabilities?.transcription).toEqual({
|
||||
protocol: 'http',
|
||||
generateOutput: true,
|
||||
streamOutput: false,
|
||||
streamInput: false,
|
||||
})
|
||||
})
|
||||
|
||||
it('keeps the local audio base URL validation in the definition', async () => {
|
||||
const validator = providerAppLocalAudioSpeech.validators?.validateConfig?.[0]({ t: translate })
|
||||
|
||||
const missing = await validator?.validator({}, { t: translate })
|
||||
const configured = await validator?.validator({ baseUrl: 'http://localhost:1234/v1/' }, { t: translate })
|
||||
|
||||
expect(missing?.valid).toBe(false)
|
||||
expect(missing?.reason).toContain('Base URL is required.')
|
||||
expect(configured?.valid).toBe(true)
|
||||
})
|
||||
|
||||
it('describes Web Speech API streaming support without runtime state', async () => {
|
||||
const defaults = z.parse(providerBrowserWebSpeechApi.createProviderConfig({ t: translate }), {})
|
||||
|
||||
expect(defaults).toEqual({
|
||||
language: 'en-US',
|
||||
continuous: true,
|
||||
interimResults: true,
|
||||
maxAlternatives: 1,
|
||||
})
|
||||
expect(providerBrowserWebSpeechApi.capabilities?.transcription).toEqual({
|
||||
protocol: 'http',
|
||||
generateOutput: false,
|
||||
streamOutput: true,
|
||||
streamInput: true,
|
||||
})
|
||||
expect(await providerBrowserWebSpeechApi.isAvailableBy?.()).toBe(false)
|
||||
})
|
||||
|
||||
it('keeps ElevenLabs configuration and model discovery in the definition', async () => {
|
||||
const defaults = z.parse(providerElevenLabs.createProviderConfig({ t: translate }), { apiKey: 'test' })
|
||||
const models = await providerElevenLabs.extraMethods?.listModels?.(defaults, providerElevenLabs.createProvider(defaults))
|
||||
|
||||
expect(defaults).toMatchObject({
|
||||
baseUrl: 'https://unspeech.hyp3r.link/v1/',
|
||||
voiceSettings: {
|
||||
similarityBoost: 0.75,
|
||||
stability: 0.5,
|
||||
},
|
||||
})
|
||||
expect(models?.length).toBeGreaterThan(0)
|
||||
expect(models?.every(model => model.provider === 'elevenlabs')).toBe(true)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,32 @@
|
||||
import { z } from 'zod'
|
||||
|
||||
import { defineProvider } from '../registry'
|
||||
|
||||
const speechNoopConfigSchema = z.object({})
|
||||
|
||||
export const providerSpeechNoop = defineProvider({
|
||||
id: 'speech-noop',
|
||||
name: 'None',
|
||||
nameLocalize: ({ t }) => t('settings.pages.providers.provider.speech-noop.title'),
|
||||
description: 'No speech output.',
|
||||
descriptionLocalize: ({ t }) => t('settings.pages.providers.provider.speech-noop.description'),
|
||||
tasks: ['text-to-speech', 'tts'],
|
||||
icon: 'i-solar:volume-cross-bold-duotone',
|
||||
requiresCredentials: false,
|
||||
|
||||
createProviderConfig: () => speechNoopConfigSchema,
|
||||
createProvider() {
|
||||
return {
|
||||
speech: () => ({
|
||||
baseURL: 'http://speech-noop.invalid/v1/',
|
||||
model: 'noop',
|
||||
}),
|
||||
}
|
||||
},
|
||||
|
||||
validationRequiredWhen: () => false,
|
||||
extraMethods: {
|
||||
listModels: async () => [],
|
||||
listVoices: async () => [],
|
||||
},
|
||||
})
|
||||
@@ -0,0 +1,239 @@
|
||||
import type {
|
||||
ListVoicesOptions,
|
||||
UnAlibabaCloudOptions,
|
||||
UnDeepgramOptions,
|
||||
UnMicrosoftOptions,
|
||||
UnVolcengineOptions,
|
||||
VoiceProviderWithExtraOptions,
|
||||
} from 'unspeech'
|
||||
import type { ComposerTranslation } from 'vue-i18n'
|
||||
|
||||
import {
|
||||
createUnAlibabaCloud,
|
||||
createUnDeepgram,
|
||||
createUnMicrosoft,
|
||||
createUnVolcengine,
|
||||
listVoices,
|
||||
} from 'unspeech'
|
||||
import { z } from 'zod'
|
||||
|
||||
import { defineProvider } from '../registry'
|
||||
|
||||
const unspeechConfigSchema = z.object({
|
||||
apiKey: z.string(),
|
||||
baseUrl: z.string().default('https://unspeech.hyp3r.link/v1/'),
|
||||
})
|
||||
|
||||
const microsoftSpeechConfigSchema = unspeechConfigSchema.extend({
|
||||
region: z.string().optional(),
|
||||
})
|
||||
|
||||
const volcengineSpeechConfigSchema = unspeechConfigSchema.extend({
|
||||
app: z.object({ appId: z.string() }),
|
||||
})
|
||||
|
||||
type UnspeechConfig = z.input<typeof unspeechConfigSchema>
|
||||
type MicrosoftSpeechConfig = z.input<typeof microsoftSpeechConfigSchema>
|
||||
type VolcengineSpeechConfig = z.input<typeof volcengineSpeechConfigSchema>
|
||||
|
||||
function createUnspeechConfigSchema(schema: typeof volcengineSpeechConfigSchema, t: ComposerTranslation): typeof volcengineSpeechConfigSchema
|
||||
function createUnspeechConfigSchema(schema: typeof microsoftSpeechConfigSchema, t: ComposerTranslation): typeof microsoftSpeechConfigSchema
|
||||
function createUnspeechConfigSchema(schema: typeof unspeechConfigSchema, t: ComposerTranslation): typeof unspeechConfigSchema
|
||||
function createUnspeechConfigSchema(
|
||||
schema: typeof unspeechConfigSchema | typeof microsoftSpeechConfigSchema | typeof volcengineSpeechConfigSchema,
|
||||
t: ComposerTranslation,
|
||||
) {
|
||||
return schema.extend({
|
||||
apiKey: schema.shape.apiKey.meta({
|
||||
labelLocalized: t('settings.pages.providers.catalog.edit.config.common.fields.field.api-key.label'),
|
||||
descriptionLocalized: t('settings.pages.providers.catalog.edit.config.common.fields.field.api-key.description'),
|
||||
placeholderLocalized: t('settings.pages.providers.catalog.edit.config.common.fields.field.api-key.placeholder'),
|
||||
type: 'password',
|
||||
}),
|
||||
baseUrl: schema.shape.baseUrl.meta({
|
||||
labelLocalized: t('settings.pages.providers.catalog.edit.config.common.fields.field.base-url.label'),
|
||||
descriptionLocalized: t('settings.pages.providers.catalog.edit.config.common.fields.field.base-url.description'),
|
||||
placeholderLocalized: t('settings.pages.providers.catalog.edit.config.common.fields.field.base-url.placeholder'),
|
||||
}),
|
||||
})
|
||||
}
|
||||
|
||||
function toListVoicesOptions<T>(provider: VoiceProviderWithExtraOptions<T>, options?: T): ListVoicesOptions {
|
||||
const { fetch: _fetch, ...voiceOptions } = provider.voice(options)
|
||||
return voiceOptions
|
||||
}
|
||||
|
||||
function validateUnspeechConfig(config: UnspeechConfig, requireAppId = false) {
|
||||
const errors: Array<{ error: unknown }> = []
|
||||
const apiKey = config.apiKey?.trim() ?? ''
|
||||
const baseUrl = config.baseUrl?.trim() ?? ''
|
||||
|
||||
if (!apiKey)
|
||||
errors.push({ error: new Error('API key is required.') })
|
||||
if (!baseUrl)
|
||||
errors.push({ error: new Error('Base URL is required.') })
|
||||
|
||||
if (baseUrl) {
|
||||
try {
|
||||
const url = new URL(baseUrl)
|
||||
if (!url.host)
|
||||
errors.push({ error: new Error('Base URL is not absolute. Try to include a scheme (http:// or https://).') })
|
||||
else if (!baseUrl.endsWith('/'))
|
||||
errors.push({ error: new Error('Base URL must end with a trailing slash (/).') })
|
||||
}
|
||||
catch {
|
||||
errors.push({ error: new Error('Base URL is not absolute. Try to include a scheme (http:// or https://).') })
|
||||
}
|
||||
}
|
||||
|
||||
if (requireAppId) {
|
||||
const appId = 'app' in config && config.app && typeof config.app === 'object' && 'appId' in config.app
|
||||
? String(config.app.appId).trim()
|
||||
: ''
|
||||
if (!appId)
|
||||
errors.push({ error: new Error('App ID is required.') })
|
||||
}
|
||||
|
||||
return {
|
||||
errors,
|
||||
reason: errors.map(item => (item.error as Error).message).join(', '),
|
||||
reasonKey: '',
|
||||
valid: errors.length === 0,
|
||||
}
|
||||
}
|
||||
|
||||
function createUnspeechValidators<TConfig extends UnspeechConfig>(id: string, requireAppId = false) {
|
||||
return {
|
||||
validateConfig: [
|
||||
({ t }: { t: ComposerTranslation }) => ({
|
||||
id: `${id}:check-config`,
|
||||
name: t('settings.pages.providers.catalog.edit.validators.openai-compatible.check-config.title'),
|
||||
validator: async (config: TConfig) => validateUnspeechConfig(config, requireAppId),
|
||||
}),
|
||||
],
|
||||
}
|
||||
}
|
||||
|
||||
export const providerDeepgramTts = defineProvider<UnspeechConfig>({
|
||||
id: 'deepgram-tts',
|
||||
name: 'Deepgram',
|
||||
nameLocalize: ({ t }) => t('settings.pages.providers.provider.deepgram-tts.title'),
|
||||
description: 'deepgram.com',
|
||||
descriptionLocalize: ({ t }) => t('settings.pages.providers.provider.deepgram-tts.description'),
|
||||
tasks: ['text-to-speech'],
|
||||
icon: 'i-simple-icons:deepgram',
|
||||
createProviderConfig: ({ t }) => createUnspeechConfigSchema(unspeechConfigSchema, t),
|
||||
createProvider: config => createUnDeepgram(config.apiKey.trim(), config.baseUrl?.trim() ?? ''),
|
||||
validationRequiredWhen: config => Boolean(config.apiKey?.trim() && config.baseUrl?.trim()),
|
||||
validators: createUnspeechValidators('deepgram-tts'),
|
||||
extraMethods: {
|
||||
listModels: async () => [
|
||||
{ id: 'aura-2', name: 'Aura 2', provider: 'deepgram-tts', description: 'Latest generation Aura model', deprecated: false },
|
||||
{ id: 'aura-1', name: 'Aura 1', provider: 'deepgram-tts', description: 'First generation Aura model', deprecated: false },
|
||||
{ id: 'aura', name: 'Aura (Legacy)', provider: 'deepgram-tts', description: 'Original Aura model', deprecated: true },
|
||||
],
|
||||
listVoices: async (config) => {
|
||||
const provider = createUnDeepgram(config.apiKey.trim(), config.baseUrl?.trim() ?? '') as VoiceProviderWithExtraOptions<UnDeepgramOptions>
|
||||
const voices = await listVoices(toListVoicesOptions(provider))
|
||||
return voices.map(voice => ({
|
||||
id: voice.id,
|
||||
name: voice.name,
|
||||
provider: 'deepgram-tts',
|
||||
description: voice.description,
|
||||
languages: voice.languages,
|
||||
gender: voice.labels?.gender,
|
||||
}))
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
export const providerMicrosoftSpeech = defineProvider<MicrosoftSpeechConfig>({
|
||||
id: 'microsoft-speech',
|
||||
name: 'Microsoft / Azure Speech',
|
||||
nameLocalize: ({ t }) => t('settings.pages.providers.provider.microsoft-speech.title'),
|
||||
description: 'speech.microsoft.com',
|
||||
descriptionLocalize: ({ t }) => t('settings.pages.providers.provider.microsoft-speech.description'),
|
||||
tasks: ['text-to-speech'],
|
||||
iconColor: 'i-lobe-icons:microsoft',
|
||||
createProviderConfig: ({ t }) => createUnspeechConfigSchema(microsoftSpeechConfigSchema, t),
|
||||
createProvider: config => createUnMicrosoft(config.apiKey.trim(), config.baseUrl?.trim() ?? ''),
|
||||
validationRequiredWhen: config => Boolean(config.apiKey?.trim() && config.baseUrl?.trim()),
|
||||
validators: createUnspeechValidators('microsoft-speech'),
|
||||
extraMethods: {
|
||||
listModels: async () => [{ id: 'v1', name: 'v1', provider: 'microsoft-speech', description: '', deprecated: false }],
|
||||
listVoices: async (config) => {
|
||||
const provider = createUnMicrosoft(config.apiKey.trim(), config.baseUrl?.trim() ?? '') as VoiceProviderWithExtraOptions<UnMicrosoftOptions>
|
||||
const voices = await listVoices(toListVoicesOptions(provider, { region: config.region ?? '' }))
|
||||
return voices.map(voice => ({
|
||||
id: voice.id,
|
||||
name: voice.name,
|
||||
provider: 'microsoft-speech',
|
||||
previewURL: voice.preview_audio_url,
|
||||
languages: voice.languages,
|
||||
gender: voice.labels?.gender,
|
||||
}))
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
export const providerAlibabaCloudModelStudio = defineProvider<UnspeechConfig>({
|
||||
id: 'alibaba-cloud-model-studio',
|
||||
name: 'Alibaba Cloud Model Studio',
|
||||
nameLocalize: ({ t }) => t('settings.pages.providers.provider.alibaba-cloud-model-studio.title'),
|
||||
description: 'bailian.console.aliyun.com',
|
||||
descriptionLocalize: ({ t }) => t('settings.pages.providers.provider.alibaba-cloud-model-studio.description'),
|
||||
tasks: ['text-to-speech'],
|
||||
iconColor: 'i-lobe-icons:alibabacloud',
|
||||
createProviderConfig: ({ t }) => createUnspeechConfigSchema(unspeechConfigSchema, t),
|
||||
createProvider: config => createUnAlibabaCloud(config.apiKey.trim(), config.baseUrl?.trim() ?? ''),
|
||||
validationRequiredWhen: config => Boolean(config.apiKey?.trim() && config.baseUrl?.trim()),
|
||||
validators: createUnspeechValidators('alibaba-cloud-model-studio'),
|
||||
extraMethods: {
|
||||
listModels: async () => [
|
||||
{ id: 'cosyvoice-v1', name: 'CosyVoice', provider: 'alibaba-cloud-model-studio', description: '', deprecated: false },
|
||||
{ id: 'cosyvoice-v2', name: 'CosyVoice (New)', provider: 'alibaba-cloud-model-studio', description: '', deprecated: false },
|
||||
],
|
||||
listVoices: async (config) => {
|
||||
const provider = createUnAlibabaCloud(config.apiKey.trim(), config.baseUrl?.trim() ?? '') as VoiceProviderWithExtraOptions<UnAlibabaCloudOptions>
|
||||
const voices = await listVoices(toListVoicesOptions(provider))
|
||||
return voices.map(voice => ({
|
||||
id: voice.id,
|
||||
name: voice.name,
|
||||
provider: 'alibaba-cloud-model-studio',
|
||||
compatibleModels: voice.compatible_models,
|
||||
previewURL: voice.preview_audio_url,
|
||||
languages: voice.languages,
|
||||
gender: voice.labels?.gender,
|
||||
}))
|
||||
},
|
||||
},
|
||||
})
|
||||
|
||||
export const providerVolcengineSpeech = defineProvider<VolcengineSpeechConfig>({
|
||||
id: 'volcengine',
|
||||
name: 'Volcengine',
|
||||
nameLocalize: ({ t }) => t('settings.pages.providers.provider.volcengine.title'),
|
||||
description: 'volcengine.com',
|
||||
descriptionLocalize: ({ t }) => t('settings.pages.providers.provider.volcengine.description'),
|
||||
tasks: ['text-to-speech'],
|
||||
iconColor: 'i-lobe-icons:volcengine',
|
||||
createProviderConfig: ({ t }) => createUnspeechConfigSchema(volcengineSpeechConfigSchema, t),
|
||||
createProvider: config => createUnVolcengine(config.apiKey.trim(), config.baseUrl?.trim() ?? ''),
|
||||
validationRequiredWhen: config => Boolean(config.apiKey?.trim() && config.baseUrl?.trim() && config.app?.appId.trim()),
|
||||
validators: createUnspeechValidators<VolcengineSpeechConfig>('volcengine', true),
|
||||
extraMethods: {
|
||||
listModels: async () => [{ id: 'v1', name: 'v1', provider: 'volcano-engine', description: '', deprecated: false }],
|
||||
listVoices: async (config) => {
|
||||
const provider = createUnVolcengine(config.apiKey.trim(), config.baseUrl?.trim() ?? '') as VoiceProviderWithExtraOptions<UnVolcengineOptions>
|
||||
const voices = await listVoices(toListVoicesOptions(provider))
|
||||
return voices.map(voice => ({
|
||||
id: voice.id,
|
||||
name: voice.name,
|
||||
provider: 'volcano-engine',
|
||||
previewURL: voice.preview_audio_url,
|
||||
languages: voice.languages,
|
||||
gender: voice.labels?.gender,
|
||||
}))
|
||||
},
|
||||
},
|
||||
})
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user