diff --git a/apps/server/src/app.ts b/apps/server/src/app.ts index 1f4d103d0..d7fac2ac4 100644 --- a/apps/server/src/app.ts +++ b/apps/server/src/app.ts @@ -59,6 +59,7 @@ import { createAdminFluxGrantsRoutes } from './routes/admin/flux-grants' import { createAdminUsersRoutes } from './routes/admin/users' import { createAdminVoicePackRoutes } from './routes/admin/voice-packs' import { createAudioSpeechWsHandlers } from './routes/audio-speech-ws' +import { createAudioTranscriptionStreamHandler } from './routes/audio-transcription-stream/route' import { createAuthRoutes } from './routes/auth' import { createCharacterRoutes } from './routes/characters' import { createChatWsHandlers } from './routes/chat-ws' @@ -215,6 +216,16 @@ export async function buildApp(deps: AppDeps) { }) })) + // Realtime ASR proxy. Mounted before the global bodyLimit middleware because + // the request body is a live microphone PCM stream rather than a bounded JSON + // payload. Auth is resolved manually here for the same reason. + app.post('/api/v1/audio/transcriptions/stream', createAudioTranscriptionStreamHandler({ + auth: deps.auth, + env: deps.env, + configKV: deps.configKV, + envelopeCrypto: deps.envelopeCrypto, + })) + // Cross-instance config invalidation. The subscriber owns its own // connection + lifecycle metrics; see services/llm-router/config-sync-subscriber.ts. createConfigSyncSubscriber({ diff --git a/apps/server/src/routes/admin/config/router/index.ts b/apps/server/src/routes/admin/config/router/index.ts index fb8bf1db2..514e93c2b 100644 --- a/apps/server/src/routes/admin/config/router/index.ts +++ b/apps/server/src/routes/admin/config/router/index.ts @@ -82,6 +82,24 @@ const StepfunSliceSchema = object({ existingKeyEntryId: optional(pipe(string(), nonEmpty(), maxLength(200), NO_PIPE)), }) +const AliyunNlsAsrSliceSchema = object({ + kind: literal('aliyun-nls-asr'), + modelName: pipe(string(), nonEmpty('modelName is required'), maxLength(200), NO_PIPE), + accessKeyId: pipe(string(), nonEmpty('accessKeyId is required'), maxLength(200)), + appKey: pipe(string(), nonEmpty('appKey is required'), maxLength(200)), + region: optional(picklist([ + 'cn-shanghai', + 'cn-shanghai-internal', + 'cn-beijing', + 'cn-beijing-internal', + 'cn-shenzhen', + 'cn-shenzhen-internal', + ], 'region must be a supported Aliyun NLS region')), + plaintextKey: optional(pipe(string(), nonEmpty('plaintextKey must not be empty when provided'), maxLength(MAX_KEY_LENGTH))), + keyEntryId: optional(pipe(string(), nonEmpty(), maxLength(200), NO_PIPE)), + existingKeyEntryId: optional(pipe(string(), nonEmpty(), maxLength(200), NO_PIPE)), +}) + /** * `restBaseURL` is the unspeech REST root (http(s)://host:port, no path). * `streaming.upstreamURL` must be ws:// or wss:// — http(s):// here is almost @@ -120,6 +138,7 @@ const SliceSchema = variant('kind', [ AzureSliceSchema, DashscopeSliceSchema, StepfunSliceSchema, + AliyunNlsAsrSliceSchema, UnspeechSliceSchema, ]) @@ -169,6 +188,8 @@ const BodySchema = object({ * { "kind": "stepfun", "modelName": "stepfun/stepaudio-2.5-tts", * "upstreamModel": "stepaudio-2.5-tts", * "defaultVoice": "cixingnansheng", "plaintextKey": "..." }, + * { "kind": "aliyun-nls-asr", "modelName": "auto", + * "accessKeyId": "...", "appKey": "...", "plaintextKey": "..." }, * { "kind": "unspeech", * "restBaseURL": "http://airi-unspeech.railway.internal:5933", * "streaming": { diff --git a/apps/server/src/routes/audio-transcription-stream/route.test.ts b/apps/server/src/routes/audio-transcription-stream/route.test.ts new file mode 100644 index 000000000..64275a05b --- /dev/null +++ b/apps/server/src/routes/audio-transcription-stream/route.test.ts @@ -0,0 +1,72 @@ +import type { RouterConfig } from '../../services/domain/llm-router/types' + +import { Buffer } from 'node:buffer' + +import { describe, expect, it } from 'vitest' + +import { createEnvelopeCrypto } from '../../utils/envelope-crypto' +import { resolveOfficialAliyunNlsCredentials } from './route' + +function createRouterConfig(overrides?: Partial): RouterConfig { + return { + llm: { models: {} }, + tts: { models: {} }, + defaults: { + perAttemptTimeoutMs: 30000, + fullChainTimeoutMs: 60000, + fallbackHttpCodes: [401, 402, 403, 429, 500, 502, 503, 504], + }, + ...overrides, + } +} + +describe('resolveOfficialAliyunNlsCredentials', () => { + /** + * @example + * resolveOfficialAliyunNlsCredentials(routerConfig, envelope, 'auto') + */ + it('returns null when official ASR model config is absent', () => { + const envelope = createEnvelopeCrypto({ masterKey: Buffer.alloc(32, 7) }) + + const credentials = resolveOfficialAliyunNlsCredentials(createRouterConfig(), envelope, 'auto') + + expect(credentials).toBeNull() + }) + + /** + * @example + * resolveOfficialAliyunNlsCredentials(routerConfig, envelope, 'auto') + */ + it('decrypts Aliyun NLS credentials from LLM_ROUTER_CONFIG.asr', () => { + const envelope = createEnvelopeCrypto({ masterKey: Buffer.alloc(32, 7) }) + const ciphertext = envelope.encryptKey(' secret ', { + modelName: 'auto', + keyEntryId: 'aliyun-nls-asr-prod-1', + }) + + const credentials = resolveOfficialAliyunNlsCredentials(createRouterConfig({ + asr: { + models: { + auto: { + provider: 'aliyun-nls', + upstreams: [{ + keys: [{ id: 'aliyun-nls-asr-prod-1', ciphertext }], + adapterParams: { + accessKeyId: ' ak ', + appKey: ' app ', + region: '', + }, + }], + }, + }, + }, + }), envelope, 'auto') + + expect(credentials).toEqual({ + accessKeyId: 'ak', + accessKeySecret: 'secret', + appKey: 'app', + region: 'cn-shanghai', + }) + }) +}) diff --git a/apps/server/src/routes/audio-transcription-stream/route.ts b/apps/server/src/routes/audio-transcription-stream/route.ts new file mode 100644 index 000000000..a693ed2f7 --- /dev/null +++ b/apps/server/src/routes/audio-transcription-stream/route.ts @@ -0,0 +1,139 @@ +import type { Context } from 'hono' + +import type { AuthInstance } from '../../libs/auth' +import type { Env } from '../../libs/env' +import type { ConfigKVService } from '../../services/adapters/config-kv' +import type { RouterConfig } from '../../services/domain/llm-router/types' +import type { EnvelopeCrypto } from '../../utils/envelope-crypto' + +import { resolveRequestAuth } from '../../libs/request-auth' +import { createKeyRotator } from '../../services/domain/llm-router/key-rotator' +import { createServiceUnavailableError, createUnauthorizedError } from '../../utils/error' +import { createAliyunNlsStreamResponse } from './session' + +type AliyunNlsRegion = 'cn-shanghai' | 'cn-shanghai-internal' | 'cn-beijing' | 'cn-beijing-internal' | 'cn-shenzhen' | 'cn-shenzhen-internal' + +const ALIYUN_NLS_REGION_FALLBACK: AliyunNlsRegion = 'cn-shanghai' +const ALIYUN_NLS_REGIONS = new Set([ + 'cn-shanghai', + 'cn-shanghai-internal', + 'cn-beijing', + 'cn-beijing-internal', + 'cn-shenzhen', + 'cn-shenzhen-internal', +]) + +const OFFICIAL_ASR_MODEL_NAME = 'auto' + +function stringAdapterParam(params: Record | undefined, key: string): string { + const value = params?.[key] + return typeof value === 'string' ? value.trim() : '' +} + +/** + * Resolves optional official Aliyun NLS credentials from router config. + * + * Use when: + * - The realtime transcription route needs to decide whether official ASR is configured. + * + * Expects: + * - `LLM_ROUTER_CONFIG.asr.models[modelName]` is an `aliyun-nls` model. + * - The first upstream key ciphertext stores the access key secret. + * - `adapterParams.accessKeyId` and `adapterParams.appKey` are present. + * + * Returns: + * - Decrypted credentials, or `null` when any required config is missing. + */ +export function resolveOfficialAliyunNlsCredentials( + routerConfig: RouterConfig | null | undefined, + envelopeCrypto: EnvelopeCrypto, + modelName: string = OFFICIAL_ASR_MODEL_NAME, +) { + const model = routerConfig?.asr?.models[modelName] + const upstream = model?.upstreams[0] + if (model?.provider !== 'aliyun-nls' || !upstream) + return null + + const iterator = createKeyRotator(upstream, envelopeCrypto, modelName, null, model.provider)[Symbol.iterator]() + const next = iterator.next() + if (next.done) + return null + + const accessKeySecretBytes = next.value.plaintext + try { + const accessKeyId = stringAdapterParam(upstream.adapterParams, 'accessKeyId') + const accessKeySecret = accessKeySecretBytes.toString('utf8').trim() + const appKey = stringAdapterParam(upstream.adapterParams, 'appKey') + const rawRegion = stringAdapterParam(upstream.adapterParams, 'region') + if (!accessKeyId || !accessKeySecret || !appKey) + return null + + const region = ALIYUN_NLS_REGIONS.has(rawRegion as AliyunNlsRegion) + ? rawRegion as AliyunNlsRegion + : ALIYUN_NLS_REGION_FALLBACK + + return { + accessKeyId, + accessKeySecret, + appKey, + region, + } + } + finally { + accessKeySecretBytes.fill(0) + } +} + +async function resolveOfficialAliyunNlsCredentialsFromConfig(input: { + configKV: ConfigKVService + envelopeCrypto: EnvelopeCrypto +}) { + const routerConfig = await input.configKV.getOptional('LLM_ROUTER_CONFIG') + const credentials = resolveOfficialAliyunNlsCredentials(routerConfig, input.envelopeCrypto) + if (!credentials) + return null + + return credentials +} + +/** + * Handles official realtime transcription audio upload streams. + * + * Use when: + * - A browser client POSTs the Hearing PCM stream and expects SSE transcript deltas. + * + * Expects: + * - Authentication has not yet run through normal session middleware because this route is mounted before body limits. + * + * Returns: + * - An SSE response that mirrors `@xsai/stream-transcription` delta events. + */ +export function createAudioTranscriptionStreamHandler(input: { + auth: AuthInstance + env: Env + configKV: ConfigKVService + envelopeCrypto: EnvelopeCrypto +}) { + return async function handleAudioTranscriptionStream(c: Context) { + const session = await resolveRequestAuth( + input.auth, + input.env, + c.req.raw.headers, + ) + if (!session?.user) + throw createUnauthorizedError() + + const credentials = await resolveOfficialAliyunNlsCredentialsFromConfig(input) + if (!credentials) + throw createServiceUnavailableError('Official ASR transcription is not configured in LLM_ROUTER_CONFIG.asr.models.auto', 'CONFIG_NOT_SET') + + const audioStream = c.req.raw.body + if (!audioStream) + throw createServiceUnavailableError('Streaming transcription request is missing audio body', 'REQUEST_BODY_NOT_STREAMABLE') + + return createAliyunNlsStreamResponse({ + audioStream: audioStream as ReadableStream, + credentials, + }) + } +} diff --git a/apps/server/src/routes/audio-transcription-stream/session.test.ts b/apps/server/src/routes/audio-transcription-stream/session.test.ts new file mode 100644 index 000000000..eed388e87 --- /dev/null +++ b/apps/server/src/routes/audio-transcription-stream/session.test.ts @@ -0,0 +1,142 @@ +import type { AddressInfo } from 'node:net' + +import { Buffer } from 'node:buffer' +import { createServer } from 'node:http' + +import { afterEach, describe, expect, it } from 'vitest' +import { WebSocketServer } from 'ws' + +import { createAliyunNlsStreamResponse } from './session' + +interface MockAliyunUpstream { + url: string + receivedTextFrames: string[] + receivedBinaryFrames: Buffer[] + close: () => Promise +} + +async function startMockAliyunUpstream(): Promise { + const receivedTextFrames: string[] = [] + const receivedBinaryFrames: Buffer[] = [] + const httpServer = createServer() + const wss = new WebSocketServer({ server: httpServer }) + + wss.on('connection', (ws) => { + ws.on('message', (data, isBinary) => { + if (isBinary) { + receivedBinaryFrames.push(Buffer.from(data as Buffer)) + return + } + + const text = data.toString() + receivedTextFrames.push(text) + const parsed = JSON.parse(text) as { header?: { name?: string } } + if (parsed.header?.name === 'StartTranscription') { + ws.send(JSON.stringify({ + header: { name: 'TranscriptionStarted' }, + payload: { session_id: 'mock-session' }, + })) + } + if (parsed.header?.name === 'StopTranscription') { + ws.send(JSON.stringify({ + header: { name: 'SentenceEnd' }, + payload: { result: 'hello airi' }, + })) + ws.send(JSON.stringify({ + header: { name: 'TranscriptionCompleted' }, + })) + } + }) + }) + + await new Promise((resolve) => { + httpServer.listen(0, '127.0.0.1', resolve) + }) + + const { port } = httpServer.address() as AddressInfo + + return { + url: `ws://127.0.0.1:${port}`, + receivedTextFrames, + receivedBinaryFrames, + async close() { + wss.close() + await new Promise(resolve => httpServer.close(() => resolve())) + }, + } +} + +function streamOf(chunks: Uint8Array[]) { + return new ReadableStream({ + start(controller) { + for (const chunk of chunks) + controller.enqueue(chunk) + controller.close() + }, + }) +} + +async function readText(stream: ReadableStream) { + const reader = stream.getReader() + const decoder = new TextDecoder() + let text = '' + while (true) { + const { done, value } = await reader.read() + if (done) + break + text += decoder.decode(value, { stream: true }) + } + text += decoder.decode() + return text +} + +describe('createAliyunNlsStreamResponse', () => { + let upstream: MockAliyunUpstream | undefined + + afterEach(async () => { + await upstream?.close() + upstream = undefined + }) + + /** + * @example + * createAliyunNlsStreamResponse({ audioStream, credentials }) + */ + it('bridges client audio chunks to Aliyun NLS and emits SSE transcript deltas', async () => { + upstream = await startMockAliyunUpstream() + + const response = createAliyunNlsStreamResponse({ + audioStream: streamOf([Buffer.from([1, 2]), Buffer.from([3, 4])]), + credentials: { + accessKeyId: 'ak', + accessKeySecret: 'secret', + appKey: 'app', + region: 'cn-shanghai', + }, + createToken: async () => ({ token: 'mock-token', expiresAt: Date.now() + 3600_000 }), + websocketBaseURL: upstream.url, + }) + + const body = await readText(response.body!) + + expect(body).toContain('data: {"delta":"hello airi\\n","type":"transcript.text.delta"}') + expect(body).toContain('data: {"delta":"","type":"transcript.text.done"}') + expect(upstream.receivedBinaryFrames).toEqual([ + Buffer.from([1, 2]), + Buffer.from([3, 4]), + ]) + + const startFrame = JSON.parse(upstream.receivedTextFrames[0]) as { + header: { appkey: string, name: string } + payload: { format: string, sample_rate: number, enable_intermediate_result: boolean } + } + expect(startFrame.header.appkey).toBe('app') + expect(startFrame.header.name).toBe('StartTranscription') + expect(startFrame.payload.format).toBe('pcm') + expect(startFrame.payload.sample_rate).toBe(16000) + expect(startFrame.payload.enable_intermediate_result).toBe(true) + + const stopFrame = JSON.parse(upstream.receivedTextFrames.at(-1)!) as { header: { name: string } } + expect(stopFrame.header.name).toBe('StopTranscription') + }) +}) diff --git a/apps/server/src/routes/audio-transcription-stream/session.ts b/apps/server/src/routes/audio-transcription-stream/session.ts new file mode 100644 index 000000000..ec50070d8 --- /dev/null +++ b/apps/server/src/routes/audio-transcription-stream/session.ts @@ -0,0 +1,231 @@ +import { createHmac, randomUUID } from 'node:crypto' + +import WebSocket from 'ws' + +import { merge } from '@moeru/std' +import { ofetch } from 'ofetch' + +type AliyunNlsRegion = 'cn-shanghai' | 'cn-shanghai-internal' | 'cn-beijing' | 'cn-beijing-internal' | 'cn-shenzhen' | 'cn-shenzhen-internal' + +interface AliyunNlsCredentials { + accessKeyId: string + accessKeySecret: string + appKey: string + region: AliyunNlsRegion +} + +interface AliyunNlsToken { + token: string + expiresAt: number +} + +interface AliyunNlsStartPayload { + format?: 'pcm' | 'wav' | 'opus' | 'speex' | 'amr' | 'mp3' | 'aac' + sample_rate?: 8000 | 16000 + enable_intermediate_result?: boolean + enable_punctuation_prediction?: boolean + enable_inverse_text_normalization?: boolean + enable_words?: boolean + max_sentence_silence?: number +} + +interface AliyunNlsServerEvent { + header?: { + name?: string + } + payload?: { + result?: string + } +} + +interface CreateAliyunNlsStreamResponseOptions { + audioStream: ReadableStream + credentials: AliyunNlsCredentials + createToken?: (credentials: AliyunNlsCredentials) => Promise + sessionOptions?: AliyunNlsStartPayload + websocketBaseURL?: string +} + +const encoder = new TextEncoder() +const DEFAULT_SESSION_OPTIONS: AliyunNlsStartPayload = { + format: 'pcm', + sample_rate: 16000, + enable_intermediate_result: true, + enable_punctuation_prediction: true, + enable_words: true, +} + +function nlsMetaEndpointFromRegion(region: AliyunNlsRegion): URL { + return new URL(`http://nls-meta.${region}.aliyuncs.com`) +} + +function nlsWebSocketEndpointFromRegion(region: AliyunNlsRegion): URL { + const websocketURL = new URL('/ws/v1', 'https://example.com') + + switch (region) { + case 'cn-shanghai': + case 'cn-beijing': + case 'cn-shenzhen': + websocketURL.protocol = 'wss:' + websocketURL.hostname = `nls-gateway-${region}.aliyuncs.com` + break + case 'cn-shanghai-internal': + case 'cn-beijing-internal': + case 'cn-shenzhen-internal': + websocketURL.protocol = 'wss:' + websocketURL.hostname = `nls-gateway-${region}-internal.aliyuncs.com:80` + break + } + + return websocketURL +} + +function canonicalizeQuery(params: Record): string { + return Object.keys(params) + .sort() + .map(key => `${encodeURIComponent(key)}=${encodeURIComponent(params[key])}`) + .join('&') +} + +function createStringToSign(method: string, path: string, canonicalQuery: string): string { + return `${method}&${encodeURIComponent(path)}&${encodeURIComponent(canonicalQuery)}` +} + +function signStringToBase64(stringToSign: string, accessKeySecret: string): string { + return createHmac('sha1', `${accessKeySecret}&`).update(stringToSign).digest('base64') +} + +function aliyunTimestamp(date: Date): string { + return date.toISOString().replace(/\.\d{3}Z$/, 'Z') +} + +async function createAliyunNlsToken(credentials: AliyunNlsCredentials): Promise { + const params: Record = { + AccessKeyId: credentials.accessKeyId, + Action: 'CreateToken', + Format: 'JSON', + RegionId: credentials.region, + SignatureMethod: 'HMAC-SHA1', + SignatureNonce: randomUUID(), + SignatureVersion: '1.0', + Timestamp: aliyunTimestamp(new Date()), + Version: '2019-02-28', + } + const canonicalQuery = canonicalizeQuery(params) + const signature = encodeURIComponent(signStringToBase64(createStringToSign('POST', '/', canonicalQuery), credentials.accessKeySecret)) + const endpoint = nlsMetaEndpointFromRegion(credentials.region).toString().replace(/\/$/, '') + const response = await ofetch<{ + Token?: { ExpireTime?: number, Id?: string } + Message?: string + }>(`${endpoint}/?Signature=${signature}&${canonicalQuery}`, { method: 'POST' }) + + if (typeof response.Token?.Id === 'string' && typeof response.Token?.ExpireTime === 'number') + return { token: response.Token.Id, expiresAt: response.Token.ExpireTime * 1000 } + + throw new Error(`Failed to create Aliyun NLS token: ${response.Message || 'unknown error'}`) +} + +function sse(payload: { delta: string, type: 'transcript.text.delta' | 'transcript.text.done' }): Uint8Array { + return encoder.encode(`data: ${JSON.stringify(payload)}\n\n`) +} + +function createClientEvent(credentials: AliyunNlsCredentials, name: 'StartTranscription' | 'StopTranscription', sessionId: string, payload?: AliyunNlsStartPayload) { + return JSON.stringify({ + header: { + appkey: credentials.appKey, + message_id: randomUUID().replaceAll('-', ''), + task_id: sessionId, + namespace: 'SpeechTranscriber', + name, + }, + payload, + }) +} + +async function writeAudioToUpstream(audioStream: ReadableStream, ws: WebSocket, credentials: AliyunNlsCredentials, sessionId: string) { + const reader = audioStream.getReader() + try { + while (true) { + const { done, value } = await reader.read() + if (done) + break + if (value) + ws.send(value, { binary: true }) + } + } + finally { + ws.send(createClientEvent(credentials, 'StopTranscription', sessionId)) + } +} + +/** + * Streams client microphone PCM through Aliyun NLS and returns xsai-compatible SSE transcript deltas. + * + * Use when: + * - AIRI owns the Aliyun NLS credentials server-side. + * - The browser uploads a realtime audio `ReadableStream` and expects transcript deltas. + * + * Expects: + * - `audioStream` contains 16 kHz PCM chunks by default, matching the Hearing worklet output. + * + * Returns: + * - A `text/event-stream` response consumable by the existing `streamAliyunTranscription` executor. + */ +export function createAliyunNlsStreamResponse(options: CreateAliyunNlsStreamResponseOptions): Response { + const body = new ReadableStream({ + async start(controller) { + const createToken = options.createToken ?? createAliyunNlsToken + const token = await createToken(options.credentials) + const sessionId = randomUUID().replaceAll('-', '') + const upstreamURL = new URL(options.websocketBaseURL ?? nlsWebSocketEndpointFromRegion(options.credentials.region)) + upstreamURL.searchParams.set('token', token.token) + + const ws = new WebSocket(upstreamURL) + + ws.on('open', () => { + ws.send(createClientEvent(options.credentials, 'StartTranscription', sessionId, merge(DEFAULT_SESSION_OPTIONS, options.sessionOptions))) + }) + + ws.on('message', (data) => { + const event = JSON.parse(data.toString()) as AliyunNlsServerEvent + switch (event.header?.name) { + case 'TranscriptionStarted': + void writeAudioToUpstream(options.audioStream, ws, options.credentials, sessionId) + break + case 'SentenceEnd': { + const text = event.payload?.result ? `${event.payload.result}\n` : '' + if (text) + controller.enqueue(sse({ delta: text, type: 'transcript.text.delta' })) + controller.enqueue(sse({ delta: '', type: 'transcript.text.done' })) + break + } + case 'TranscriptionCompleted': + controller.close() + ws.close(1000, 'completed') + break + } + }) + + ws.on('error', (error) => { + controller.error(error) + }) + + ws.on('close', () => { + try { + controller.close() + } + catch {} + }) + }, + cancel() { + // The upstream websocket is closed by its own completion/error handlers. + }, + }) + + return new Response(body, { + headers: { + 'Cache-Control': 'no-cache', + 'Content-Type': 'text/event-stream', + }, + }) +} diff --git a/apps/server/src/services/adapters/config-kv.test.ts b/apps/server/src/services/adapters/config-kv.test.ts index 1d2a5e345..f10f9394b 100644 --- a/apps/server/src/services/adapters/config-kv.test.ts +++ b/apps/server/src/services/adapters/config-kv.test.ts @@ -108,6 +108,49 @@ describe('configKVService', () => { expect(value).toBe(500) }) + /** + * @example + * service.set('LLM_ROUTER_CONFIG', { asr: { models: { auto: model } } }) + */ + it('llm router config should preserve official ASR model config', async () => { + await service.set('LLM_ROUTER_CONFIG', { + llm: { models: {} }, + tts: { models: {} }, + asr: { + models: { + auto: { + provider: 'aliyun-nls', + upstreams: [{ + keys: [{ id: 'aliyun-nls-asr-prod-1', ciphertext: 'ciphertext' }], + adapterParams: { + accessKeyId: 'ak', + appKey: 'app', + region: 'cn-shanghai', + }, + }], + }, + }, + }, + defaults: { + perAttemptTimeoutMs: 30000, + fullChainTimeoutMs: 60000, + fallbackHttpCodes: [401, 402, 403, 429, 500, 502, 503, 504], + }, + }) + + const value = await service.getOrThrow('LLM_ROUTER_CONFIG') + const asr = value.asr + if (!asr) + throw new Error('Expected ASR config to be preserved') + + expect(asr.models.auto.provider).toBe('aliyun-nls') + expect(asr.models.auto.upstreams[0].adapterParams).toEqual({ + accessKeyId: 'ak', + appKey: 'app', + region: 'cn-shanghai', + }) + }) + it('set should store string values as JSON strings', async () => { await service.set('STRIPE_FLUX_PRODUCT_ID', 'prod_abc123') diff --git a/apps/server/src/services/adapters/config-kv.ts b/apps/server/src/services/adapters/config-kv.ts index 25d5e6b75..ca11562ff 100644 --- a/apps/server/src/services/adapters/config-kv.ts +++ b/apps/server/src/services/adapters/config-kv.ts @@ -53,6 +53,7 @@ export const llmModelSchema = object({ }) const ttsProviderSchema = picklist(['azure', 'dashscope-cosyvoice', 'stepfun', 'volcengine']) +const asrProviderSchema = picklist(['aliyun-nls']) export const ttsUpstreamSchema = object({ baseURL: pipe(string(), nonEmpty('tts.upstreams[].baseURL must not be empty')), @@ -93,6 +94,16 @@ export const ttsModelSchema = object({ fallbackTriggers: fallbackTriggersSchema, }) +export const asrUpstreamSchema = object({ + keys: pipe(array(keyEntrySchema), check(v => v.length >= 1, 'asr.upstreams[].keys must contain at least 1 entry')), + adapterParams: optional(record(string(), any()), {}), +}) + +export const asrModelSchema = object({ + provider: asrProviderSchema, + upstreams: pipe(array(asrUpstreamSchema), check(v => v.length >= 1, 'asr.models[].upstreams must contain at least 1 entry')), +}) + export const llmRouterDefaultsSchema = optional( object({ perAttemptTimeoutMs: optional(number(), 30000), @@ -109,6 +120,9 @@ export const llmRouterConfigSchema = object({ tts: object({ models: record(string(), ttsModelSchema), }), + asr: optional(object({ + models: record(string(), asrModelSchema), + })), defaults: llmRouterDefaultsSchema, }) diff --git a/apps/server/src/services/domain/admin/router-config/index.ts b/apps/server/src/services/domain/admin/router-config/index.ts index 917c89ff4..cf31fb342 100644 --- a/apps/server/src/services/domain/admin/router-config/index.ts +++ b/apps/server/src/services/domain/admin/router-config/index.ts @@ -2,7 +2,7 @@ import type Redis from 'ioredis' import type { InferOutput } from 'valibot' import type { EnvelopeCrypto } from '../../../../utils/envelope-crypto' -import type { ConfigKVService, llmModelSchema, llmRouterConfigSchema, ttsModelSchema, unspeechUpstreamSchema } from '../../../adapters/config-kv' +import type { asrModelSchema, ConfigKVService, llmModelSchema, llmRouterConfigSchema, ttsModelSchema, unspeechUpstreamSchema } from '../../../adapters/config-kv' import { useLogger } from '@guiiai/logg' @@ -24,6 +24,7 @@ const DEFAULT_KEY_ENTRY_IDS = { 'dashscope-cosyvoice': 'dashscope-tts-prod-1', 'stepfun': 'stepfun-tts-prod-1', 'unspeech': 'volcengine-prod-1', + 'aliyun-nls-asr': 'aliyun-nls-asr-prod-1', } as const const DEFAULT_FALLBACK_TRIGGERS = { @@ -34,6 +35,7 @@ const DEFAULT_FALLBACK_TRIGGERS = { type LlmRouterConfig = InferOutput type LlmModel = InferOutput type TtsModel = InferOutput +type AsrModel = InferOutput type UnspeechUpstream = InferOutput type KeyEntry = LlmModel['upstreams'][number]['keys'][number] @@ -50,6 +52,7 @@ export type SliceInput | AzureSliceInput | DashscopeSliceInput | StepfunSliceInput + | AliyunNlsAsrSliceInput | UnspeechSliceInput export interface OpenRouterSliceInput { @@ -138,6 +141,24 @@ export interface UnspeechSliceInput { } } +export interface AliyunNlsAsrSliceInput { + kind: 'aliyun-nls-asr' + /** Key under `LLM_ROUTER_CONFIG.asr.models`; the official client currently uses `auto`. */ + modelName: string + /** Aliyun AccessKey ID used for token signing. Stored in adapterParams, not encrypted. */ + accessKeyId: string + /** Aliyun NLS app key. Stored in adapterParams, not encrypted. */ + appKey: string + /** Aliyun NLS region; defaults to cn-shanghai. */ + region?: 'cn-shanghai' | 'cn-shanghai-internal' | 'cn-beijing' | 'cn-beijing-internal' | 'cn-shenzhen' | 'cn-shenzhen-internal' + /** Aliyun AccessKey secret. Encrypted in-place; never echoed back. */ + plaintextKey?: string + /** @default 'aliyun-nls-asr-prod-1' */ + keyEntryId?: string + /** Existing key entry to preserve when `plaintextKey` is omitted. */ + existingKeyEntryId?: string +} + interface LlmModelSlice { target: 'llm-router' surface: 'llm' @@ -156,6 +177,15 @@ interface TtsModelSlice { keyEntryId: string } +interface AsrModelSlice { + target: 'llm-router' + surface: 'asr' + kind: 'aliyun-nls-asr' + modelName: string + model: AsrModel + keyEntryId: string +} + interface UnspeechSlice { target: 'unspeech' kind: 'unspeech' @@ -164,7 +194,7 @@ interface UnspeechSlice { keyEntryId: string | null } -type BuiltSlice = LlmModelSlice | TtsModelSlice | UnspeechSlice +type BuiltSlice = LlmModelSlice | TtsModelSlice | AsrModelSlice | UnspeechSlice /** * Encrypts an OpenRouter slice into the LLM_ROUTER_CONFIG.llm shape. @@ -310,6 +340,43 @@ export function buildStepfunSlice(input: StepfunSliceInput, envelope: EnvelopeCr } } +/** + * Encrypts an Aliyun NLS ASR slice into the LLM_ROUTER_CONFIG.asr shape. + * + * Use when: + * - Admin posts an `aliyun-nls-asr` slice for the official realtime + * transcription proxy. + * + * Expects: + * - `plaintextKey` is the Aliyun AccessKey secret. `accessKeyId` and `appKey` + * are non-secret routing params stored in `adapterParams`. + */ +export function buildAliyunNlsAsrSlice(input: AliyunNlsAsrSliceInput, envelope: EnvelopeCrypto): AsrModelSlice { + const keyEntryId = input.keyEntryId ?? DEFAULT_KEY_ENTRY_IDS['aliyun-nls-asr'] + const ciphertext = envelope.encryptKey(requiredPlaintextKey(input.plaintextKey, input.kind), { + modelName: input.modelName, + keyEntryId, + }) + return { + target: 'llm-router', + surface: 'asr', + kind: 'aliyun-nls-asr', + modelName: input.modelName, + keyEntryId, + model: { + provider: 'aliyun-nls', + upstreams: [{ + keys: [{ id: keyEntryId, ciphertext }], + adapterParams: { + accessKeyId: input.accessKeyId, + appKey: input.appKey, + region: input.region ?? 'cn-shanghai', + }, + }], + }, + } +} + /** * Encrypts an unspeech slice into the UNSPEECH_UPSTREAM shape. * @@ -486,6 +553,32 @@ function buildStepfunSlicePreservingKey(input: StepfunSliceInput, envelope: Enve } } +function buildAliyunNlsAsrSlicePreservingKey(input: AliyunNlsAsrSliceInput, envelope: EnvelopeCrypto, existing: AsrModel | undefined): AsrModelSlice { + if (input.plaintextKey?.trim()) + return buildAliyunNlsAsrSlice(input, envelope) + + const existingUpstream = existing?.upstreams[0] + const key = preservedKeyOrThrow(existingUpstream, input.existingKeyEntryId ?? input.keyEntryId, input.kind) + return { + target: 'llm-router', + surface: 'asr', + kind: 'aliyun-nls-asr', + modelName: input.modelName, + keyEntryId: key.id, + model: { + provider: 'aliyun-nls', + upstreams: [{ + keys: [key], + adapterParams: { + accessKeyId: input.accessKeyId, + appKey: input.appKey, + region: input.region ?? stringFromRecord(existingUpstream?.adapterParams, 'region') ?? 'cn-shanghai', + }, + }], + }, + } +} + function buildUnspeechSlicePreservingKey(input: UnspeechSliceInput, envelope: EnvelopeCrypto, existing: UnspeechUpstream | undefined | null): UnspeechSlice { if (!input.streaming || input.streaming.plaintextKey?.trim()) return buildUnspeechSlice(input, envelope) @@ -532,6 +625,8 @@ export function buildSlice( return buildDashscopeSlicePreservingKey(input, envelope, existing?.routerConfig?.tts.models[input.modelName]) case 'stepfun': return buildStepfunSlicePreservingKey(input, envelope, existing?.routerConfig?.tts.models[input.modelName]) + case 'aliyun-nls-asr': + return buildAliyunNlsAsrSlicePreservingKey(input, envelope, existing?.routerConfig?.asr?.models[input.modelName]) case 'unspeech': return buildUnspeechSlicePreservingKey(input, envelope, existing?.unspeech) } @@ -558,18 +653,22 @@ export function buildSlice( export function buildNextRouterConfig( mode: 'merge' | 'reset', existing: LlmRouterConfig | null | undefined, - slices: (LlmModelSlice | TtsModelSlice)[], + slices: (LlmModelSlice | TtsModelSlice | AsrModelSlice)[], ): LlmRouterConfig { const llmModels: Record = mode === 'merge' && existing?.llm?.models ? { ...existing.llm.models } : {} const ttsModels: Record = mode === 'merge' && existing?.tts?.models ? { ...existing.tts.models } : {} + const asrModels: Record + = mode === 'merge' && existing?.asr?.models ? { ...existing.asr.models } : {} for (const slice of slices) { if (slice.surface === 'llm') llmModels[slice.modelName] = slice.model - else + else if (slice.surface === 'tts') ttsModels[slice.modelName] = slice.model + else + asrModels[slice.modelName] = slice.model } // Defaults live alongside the models but aren't editable through this @@ -582,6 +681,7 @@ export function buildNextRouterConfig( return { llm: { models: llmModels }, tts: { models: ttsModels }, + asr: { models: asrModels }, defaults, } } @@ -628,7 +728,7 @@ export interface ApplyInput { export interface AppliedSummary { kind: SliceInput['kind'] target: 'llm-router' | 'unspeech' - surface?: 'llm' | 'tts' + surface?: 'llm' | 'tts' | 'asr' modelName?: string keyEntryId: string | null } @@ -678,6 +778,11 @@ function slicesFromRouterConfig(config: LlmRouterConfig | null): SliceInput[] { if (slice) slices.push(slice) } + for (const [modelName, model] of Object.entries(config.asr?.models ?? {})) { + const slice = asrSliceFromModel(modelName, model) + if (slice) + slices.push(slice) + } return slices } @@ -743,6 +848,29 @@ function ttsSliceFromModel(modelName: string, model: TtsModel): AzureSliceInput return null } +function asrSliceFromModel(modelName: string, model: AsrModel): AliyunNlsAsrSliceInput | null { + const upstream = model.upstreams[0] + const key = upstream?.keys[0] + if (model.provider !== 'aliyun-nls' || !upstream || !key) + return null + + const accessKeyId = stringFromRecord(upstream.adapterParams, 'accessKeyId') + const appKey = stringFromRecord(upstream.adapterParams, 'appKey') + if (!accessKeyId || !appKey) + return null + + const region = stringFromRecord(upstream.adapterParams, 'region') + return { + kind: 'aliyun-nls-asr', + modelName, + accessKeyId, + appKey, + region: isAliyunNlsRegion(region) ? region : undefined, + keyEntryId: key.id, + existingKeyEntryId: key.id, + } +} + function slicesFromUnspeech(unspeech: UnspeechUpstream | null): UnspeechSliceInput[] { if (!unspeech) return [] @@ -779,6 +907,15 @@ function isStepfunInputModel(value: string | undefined): value is NonNullable { + return value === 'cn-shanghai' + || value === 'cn-shanghai-internal' + || value === 'cn-beijing' + || value === 'cn-beijing-internal' + || value === 'cn-shenzhen' + || value === 'cn-shenzhen-internal' +} + interface AdminRouterConfigDeps { configKV: ConfigKVService envelope: EnvelopeCrypto @@ -875,9 +1012,9 @@ export function createAdminRouterConfigService(deps: AdminRouterConfigDeps) { if (unspeechCount > 1) throw createBadRequestError('At most one unspeech slice per request', 'INVALID_BODY') - const hasLlmTtsInput = input.slices.some(s => s.kind !== 'unspeech') + const hasRouterInput = input.slices.some(s => s.kind !== 'unspeech') const hasUnspeechInput = input.slices.some(s => s.kind === 'unspeech') - const shouldReadRouterConfig = hasLlmTtsInput + const shouldReadRouterConfig = hasRouterInput && (input.mode === 'merge' || input.slices.some(sliceNeedsExistingKey)) const shouldReadUnspeech = hasUnspeechInput const [existingRouterConfig, existingUnspeech] = await Promise.all([ @@ -892,14 +1029,14 @@ export function createAdminRouterConfigService(deps: AdminRouterConfigDeps) { unspeech: existingUnspeech, })) - const llmTtsSlices = built.filter((s): s is LlmModelSlice | TtsModelSlice => s.target === 'llm-router') + const routerSlices = built.filter((s): s is LlmModelSlice | TtsModelSlice | AsrModelSlice => s.target === 'llm-router') const unspeechSlice = built.find((s): s is UnspeechSlice => s.target === 'unspeech') - // Step 2: build the next LLM_ROUTER_CONFIG tree if any LLM/TTS slice + // Step 2: build the next LLM_ROUTER_CONFIG tree if any LLM/TTS/ASR slice // was supplied. `merge` reads existing first; `reset` skips the read. let nextRouterConfig: LlmRouterConfig | undefined - if (llmTtsSlices.length > 0) { - nextRouterConfig = buildNextRouterConfig(input.mode, existingRouterConfig, llmTtsSlices) + if (routerSlices.length > 0) { + nextRouterConfig = buildNextRouterConfig(input.mode, existingRouterConfig, routerSlices) } // Step 3: build the next UNSPEECH_UPSTREAM. Streaming `models` + diff --git a/apps/server/src/services/domain/admin/router-config/tests/admin-router-config.test.ts b/apps/server/src/services/domain/admin/router-config/tests/admin-router-config.test.ts index eaad23f29..9d61f12d5 100644 --- a/apps/server/src/services/domain/admin/router-config/tests/admin-router-config.test.ts +++ b/apps/server/src/services/domain/admin/router-config/tests/admin-router-config.test.ts @@ -7,6 +7,7 @@ import { randomBytes } from 'node:crypto' import { beforeEach, describe, expect, it, vi } from 'vitest' import { + buildAliyunNlsAsrSlice, buildAzureSlice, buildDashscopeSlice, buildNextRouterConfig, @@ -178,6 +179,40 @@ describe('buildAzureSlice', () => { }) }) +describe('buildAliyunNlsAsrSlice', () => { + /** + * @example + * buildAliyunNlsAsrSlice({ kind: 'aliyun-nls-asr', modelName: 'auto', accessKeyId: 'ak', appKey: 'app', plaintextKey: 'secret' }, envelope) + */ + it('encrypts the access key secret under the ASR model AAD', () => { + const envelope = freshEnvelope() + const built = buildAliyunNlsAsrSlice({ + kind: 'aliyun-nls-asr', + modelName: 'auto', + accessKeyId: 'ak', + appKey: 'app', + plaintextKey: 'secret', + }, envelope) + + expect(built.target).toBe('llm-router') + expect(built.surface).toBe('asr') + expect(built.modelName).toBe('auto') + expect(built.keyEntryId).toBe('aliyun-nls-asr-prod-1') + expect(built.model.provider).toBe('aliyun-nls') + expect(built.model.upstreams[0].adapterParams).toEqual({ + accessKeyId: 'ak', + appKey: 'app', + region: 'cn-shanghai', + }) + + const decrypted = envelope.decryptKey(built.model.upstreams[0].keys[0].ciphertext, { + modelName: 'auto', + keyEntryId: 'aliyun-nls-asr-prod-1', + }) + expect(decrypted.toString('utf8')).toBe('secret') + }) +}) + describe('buildDashscopeSlice', () => { it.each([ ['intl', 'dashscope-intl.aliyuncs.com'], diff --git a/apps/server/src/services/domain/llm-router/types.ts b/apps/server/src/services/domain/llm-router/types.ts index 5e8144cd9..6c8e72fab 100644 --- a/apps/server/src/services/domain/llm-router/types.ts +++ b/apps/server/src/services/domain/llm-router/types.ts @@ -7,6 +7,8 @@ import type { InferOutput } from 'valibot' // here. // Source: apps/server/src/services/config-kv.ts (llmRouterConfigSchema). import type { + asrModelSchema, + asrUpstreamSchema, fallbackTriggersSchema, keyEntrySchema, llmModelSchema, @@ -48,6 +50,16 @@ export type TtsUpstream = InferOutput */ export type TtsModel = InferOutput +/** + * ASR model entry — provider tag + ordered upstreams for realtime transcription. + */ +export type AsrModel = InferOutput + +/** + * ASR upstream — one provider credential set plus adapter params. + */ +export type AsrUpstream = InferOutput + /** * Per-(upstream) fallback trigger config: which upstream HTTP codes should * cause the router to move on to the next key/upstream. diff --git a/apps/ui-admin/src/components/llm-router/RouterSliceEditor.vue b/apps/ui-admin/src/components/llm-router/RouterSliceEditor.vue index ab5534c89..50a6b7c1f 100644 --- a/apps/ui-admin/src/components/llm-router/RouterSliceEditor.vue +++ b/apps/ui-admin/src/components/llm-router/RouterSliceEditor.vue @@ -5,6 +5,7 @@ import { Button, FieldInput, FieldSelect, FieldTextArea } from '@proj-airi/ui' import { computed } from 'vue' import { + ALIYUN_NLS_REGION_OPTIONS, DASHSCOPE_REGION_OPTIONS, STEPFUN_MODEL_OPTIONS, } from '../../modules/router-config-form' @@ -30,6 +31,8 @@ const title = computed(() => { return 'DashScope CosyVoice' case 'stepfun': return 'StepFun TTS' + case 'aliyun-nls-asr': + return 'Aliyun NLS ASR' case 'unspeech': return 'UnSpeech' default: @@ -119,7 +122,16 @@ const streamingKeyPlaceholder = computed(() => { -
+
+ + + + + + +
+ +