mirror of
https://github.com/moeru-ai/airi.git
synced 2026-08-14 08:52:42 +00:00
fix(stage-ui): add chat provider fallback
This commit is contained in:
@@ -15,6 +15,8 @@ export type {
|
||||
ChatOrchestratorLifecycleRecord,
|
||||
ChatOrchestratorLLMPort,
|
||||
ChatOrchestratorPromptProjection,
|
||||
ChatOrchestratorProviderCandidate,
|
||||
ChatOrchestratorProviderCandidateSource,
|
||||
ChatOrchestratorRuntime,
|
||||
ChatOrchestratorRuntimeDeps,
|
||||
ChatOrchestratorRuntimeState,
|
||||
|
||||
@@ -3,6 +3,7 @@ import type { Message } from '@xsai/shared-chat'
|
||||
|
||||
import type { ChatHistoryItem, ContextMessage, StreamingAssistantMessage } from '../types/chat'
|
||||
import type { StreamEvent, StreamOptions } from '../types/llm'
|
||||
import type { ChatOrchestratorSendOptions } from './chat-orchestrator-runtime'
|
||||
|
||||
import { ContextUpdateStrategy } from '@proj-airi/server-shared/types'
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
@@ -148,6 +149,101 @@ function createHarness() {
|
||||
* await runtime.ingest('hello', { model, chatProvider })
|
||||
*/
|
||||
describe('createChatOrchestratorRuntime', () => {
|
||||
// ROOT CAUSE:
|
||||
//
|
||||
// The runtime accepted only one provider and model for a chat turn. A provider
|
||||
// failure therefore ended the turn even when the caller had another usable provider.
|
||||
//
|
||||
// The runtime now tries the next candidate before the failed attempt emits output.
|
||||
it('falls back before output and stores the user turn once', async () => {
|
||||
const harness = createHarness()
|
||||
const fallbackProvider = {
|
||||
chat: () => ({ baseURL: 'https://fallback.example.com/' }),
|
||||
} as unknown as ChatProvider
|
||||
const unavailableCandidate = vi.fn(async () => undefined)
|
||||
const fallbackCandidate = vi.fn(async () => ({
|
||||
model: 'fallback-model',
|
||||
providerId: 'fallback-provider',
|
||||
chatProvider: fallbackProvider,
|
||||
}))
|
||||
|
||||
harness.stream.mockRejectedValueOnce(new Error('primary provider unavailable'))
|
||||
harness.stream.mockImplementationOnce(async (_model, _chatProvider, _messages, options) => {
|
||||
await options?.onStreamEvent?.({ type: 'text-delta', text: 'fallback reply' })
|
||||
await options?.onStreamEvent?.({ type: 'finish', finishReason: 'stop' })
|
||||
})
|
||||
|
||||
await harness.runtime.ingest('use a fallback', {
|
||||
model: 'primary-model',
|
||||
providerId: 'primary-provider',
|
||||
chatProvider: provider,
|
||||
fallbackCandidates: [unavailableCandidate, fallbackCandidate],
|
||||
} as ChatOrchestratorSendOptions)
|
||||
|
||||
expect(harness.stream).toHaveBeenCalledTimes(2)
|
||||
expect(unavailableCandidate).toHaveBeenCalledTimes(1)
|
||||
expect(fallbackCandidate).toHaveBeenCalledTimes(1)
|
||||
expect(harness.stream.mock.calls[0]?.slice(0, 2)).toEqual(['primary-model', provider])
|
||||
expect(harness.stream.mock.calls[1]?.slice(0, 2)).toEqual(['fallback-model', fallbackProvider])
|
||||
expect(harness.sessionMessages['session-1']?.filter(message => message.role === 'user')).toHaveLength(1)
|
||||
expect(harness.sessionMessages['session-1']?.filter(message => message.role === 'assistant')).toHaveLength(1)
|
||||
expect(harness.telemetry.chatActivationSucceeded).toEqual([
|
||||
expect.objectContaining({
|
||||
model: 'fallback-model',
|
||||
provider: 'fallback-provider',
|
||||
}),
|
||||
])
|
||||
})
|
||||
|
||||
it('does not fall back after the provider emits visible output', async () => {
|
||||
const harness = createHarness()
|
||||
const fallbackProvider = {
|
||||
chat: () => ({ baseURL: 'https://fallback.example.com/' }),
|
||||
} as unknown as ChatProvider
|
||||
|
||||
harness.stream.mockImplementationOnce(async (_model, _chatProvider, _messages, options) => {
|
||||
await options?.onStreamEvent?.({ type: 'text-delta', text: 'partial reply' })
|
||||
throw new Error('stream failed after output')
|
||||
})
|
||||
|
||||
await expect(harness.runtime.ingest('do not duplicate this turn', {
|
||||
model: 'primary-model',
|
||||
providerId: 'primary-provider',
|
||||
chatProvider: provider,
|
||||
fallbackCandidates: [{
|
||||
model: 'fallback-model',
|
||||
providerId: 'fallback-provider',
|
||||
chatProvider: fallbackProvider,
|
||||
}],
|
||||
} as ChatOrchestratorSendOptions)).rejects.toThrow('stream failed after output')
|
||||
|
||||
expect(harness.stream).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it('does not fall back for a non-retryable request error', async () => {
|
||||
const harness = createHarness()
|
||||
const fallbackCandidate = vi.fn(async () => ({
|
||||
model: 'fallback-model',
|
||||
providerId: 'fallback-provider',
|
||||
chatProvider: provider,
|
||||
}))
|
||||
const requestError = Object.assign(new Error('invalid request'), {
|
||||
cause: { status: 400 },
|
||||
})
|
||||
|
||||
harness.stream.mockRejectedValueOnce(requestError)
|
||||
|
||||
await expect(harness.runtime.ingest('keep this request on one provider', {
|
||||
model: 'primary-model',
|
||||
providerId: 'primary-provider',
|
||||
chatProvider: provider,
|
||||
fallbackCandidates: [fallbackCandidate],
|
||||
})).rejects.toThrow('invalid request')
|
||||
|
||||
expect(harness.stream).toHaveBeenCalledTimes(1)
|
||||
expect(fallbackCandidate).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
// ROOT CAUSE:
|
||||
//
|
||||
// The marker parser buffered 24 literal characters plus its marker-safety tail.
|
||||
|
||||
@@ -47,11 +47,29 @@ function cloneStreamingMessage(message: StreamingAssistantMessage): StreamingAss
|
||||
/**
|
||||
* Options accepted by the chat orchestrator runtime for one user send.
|
||||
*/
|
||||
export interface ChatOrchestratorProviderCandidate {
|
||||
/** Stable provider identifier used for routing and telemetry. */
|
||||
providerId: string
|
||||
/** Provider model identifier used for this attempt. */
|
||||
model: string
|
||||
/** Concrete chat provider implementation used for this attempt. */
|
||||
chatProvider: ChatProvider
|
||||
}
|
||||
|
||||
/** Resolves one fallback candidate when the previous provider cannot start a response. */
|
||||
export type ChatOrchestratorProviderCandidateSource
|
||||
= ChatOrchestratorProviderCandidate
|
||||
| (() => Promise<ChatOrchestratorProviderCandidate | undefined>)
|
||||
|
||||
export interface ChatOrchestratorSendOptions {
|
||||
/** Provider model identifier used for the outbound LLM request. */
|
||||
model: string
|
||||
/** Concrete chat provider implementation selected by the caller. */
|
||||
chatProvider: ChatProvider
|
||||
/** Provider that owns the primary model. Defaults to the active provider. */
|
||||
providerId?: string
|
||||
/** Ordered candidates tried after the primary provider fails before output. */
|
||||
fallbackCandidates?: ChatOrchestratorProviderCandidateSource[]
|
||||
/** Provider-specific request options, currently used for headers. */
|
||||
providerConfig?: Record<string, unknown>
|
||||
/** Image attachments appended to the user message content parts. */
|
||||
@@ -76,6 +94,48 @@ interface QueuedSend {
|
||||
}
|
||||
}
|
||||
|
||||
function isAbortError(error: unknown): boolean {
|
||||
return error instanceof Error && error.name === 'AbortError'
|
||||
}
|
||||
|
||||
function statusCodeFrom(error: unknown): number | undefined {
|
||||
const statusError = error as {
|
||||
cause?: {
|
||||
response?: { status?: unknown }
|
||||
status?: unknown
|
||||
statusCode?: unknown
|
||||
}
|
||||
response?: { status?: unknown }
|
||||
status?: unknown
|
||||
statusCode?: unknown
|
||||
} | undefined
|
||||
const candidates = [
|
||||
statusError?.status,
|
||||
statusError?.statusCode,
|
||||
statusError?.response?.status,
|
||||
statusError?.cause?.status,
|
||||
statusError?.cause?.statusCode,
|
||||
statusError?.cause?.response?.status,
|
||||
]
|
||||
for (const candidate of candidates) {
|
||||
if (typeof candidate === 'number')
|
||||
return candidate
|
||||
}
|
||||
|
||||
return undefined
|
||||
}
|
||||
|
||||
function canFallbackFrom(error: unknown): boolean {
|
||||
if (isAbortError(error))
|
||||
return false
|
||||
|
||||
const status = statusCodeFrom(error)
|
||||
if (status === undefined)
|
||||
return true
|
||||
|
||||
return [401, 403, 404, 408, 425, 429].includes(status) || status >= 500
|
||||
}
|
||||
|
||||
/**
|
||||
* Serializable view of a queued send waiting to be processed.
|
||||
*/
|
||||
@@ -493,7 +553,16 @@ export function createChatOrchestratorRuntime(deps: ChatOrchestratorRuntimeDeps)
|
||||
}
|
||||
patchForegroundStream(sessionId, buildingMessage)
|
||||
const sendSource = options.input ? 'voice' : 'text'
|
||||
const activeProvider = deps.getActiveProvider?.() ?? ''
|
||||
const primaryProviderId = options.providerId ?? deps.getActiveProvider?.() ?? ''
|
||||
const providerCandidateSources: ChatOrchestratorProviderCandidateSource[] = [
|
||||
{
|
||||
providerId: primaryProviderId,
|
||||
model: options.model,
|
||||
chatProvider: options.chatProvider,
|
||||
},
|
||||
...(options.fallbackCandidates ?? []),
|
||||
]
|
||||
let currentCandidate = providerCandidateSources[0] as ChatOrchestratorProviderCandidate
|
||||
// The user message is the durable start of a round, so its ID also serves
|
||||
// as the correlation key for every telemetry milestone emitted by it.
|
||||
const correlation: ChatRoundCorrelation = {
|
||||
@@ -507,7 +576,7 @@ export function createChatOrchestratorRuntime(deps: ChatOrchestratorRuntimeDeps)
|
||||
...correlation,
|
||||
source: sendSource,
|
||||
model: options.model,
|
||||
provider: activeProvider,
|
||||
provider: primaryProviderId,
|
||||
})
|
||||
}
|
||||
deps.onMessageSendStarted?.({
|
||||
@@ -565,7 +634,7 @@ export function createChatOrchestratorRuntime(deps: ChatOrchestratorRuntimeDeps)
|
||||
messageText: sendingMessage,
|
||||
source: sendSource,
|
||||
model: options.model,
|
||||
provider: activeProvider,
|
||||
provider: primaryProviderId,
|
||||
roundId,
|
||||
turnIndex,
|
||||
})
|
||||
@@ -576,7 +645,11 @@ export function createChatOrchestratorRuntime(deps: ChatOrchestratorRuntimeDeps)
|
||||
sessionMessages: sessionMessagesForSend,
|
||||
})
|
||||
|
||||
const categorizer = createStreamingCategorizer(deps.getActiveProvider())
|
||||
let categorizer: ReturnType<typeof createStreamingCategorizer> | undefined
|
||||
const getCategorizer = () => {
|
||||
categorizer ??= createStreamingCategorizer(currentCandidate.providerId)
|
||||
return categorizer
|
||||
}
|
||||
let streamPosition = 0
|
||||
|
||||
const parser = useLlmmarkerParser({
|
||||
@@ -584,9 +657,10 @@ export function createChatOrchestratorRuntime(deps: ChatOrchestratorRuntimeDeps)
|
||||
if (shouldAbort())
|
||||
return
|
||||
|
||||
categorizer.consume(literal)
|
||||
const currentCategorizer = getCategorizer()
|
||||
currentCategorizer.consume(literal)
|
||||
|
||||
const speechOnly = categorizer.filterToSpeech(literal, streamPosition)
|
||||
const speechOnly = currentCategorizer.filterToSpeech(literal, streamPosition)
|
||||
streamPosition += literal.length
|
||||
|
||||
if (speechOnly.trim()) {
|
||||
@@ -617,7 +691,7 @@ export function createChatOrchestratorRuntime(deps: ChatOrchestratorRuntimeDeps)
|
||||
if (isStaleGeneration())
|
||||
return
|
||||
|
||||
const finalCategorization = categorizeResponse(fullText, deps.getActiveProvider())
|
||||
const finalCategorization = categorizeResponse(fullText, currentCandidate.providerId)
|
||||
|
||||
const reasoningContentField = buildingMessage.categorization?.reasoning?.trim()
|
||||
buildingMessage.categorization = {
|
||||
@@ -718,116 +792,170 @@ export function createChatOrchestratorRuntime(deps: ChatOrchestratorRuntimeDeps)
|
||||
if (shouldAbort())
|
||||
return
|
||||
|
||||
const llmRequestStartedAt = monotonicNow()
|
||||
let llmFirstTokenEmitted = false
|
||||
let llmRequestStartedAt = 0
|
||||
let generationUsage: LlmUsage = { source: 'unavailable' }
|
||||
let generationUsageReported = false
|
||||
let providerTranscript: Message[] | undefined
|
||||
const providerInputMessageCount = newMessages.length
|
||||
deps.onLlmRequestStarted?.({
|
||||
...correlation,
|
||||
model: options.model,
|
||||
provider: deps.getActiveProvider() || 'unknown',
|
||||
hasVoice: !!options.input,
|
||||
})
|
||||
let generationCompleted = false
|
||||
let lastProviderError: unknown
|
||||
for (const candidateSource of providerCandidateSources) {
|
||||
let candidate: ChatOrchestratorProviderCandidate | undefined
|
||||
try {
|
||||
candidate = typeof candidateSource === 'function'
|
||||
? await candidateSource()
|
||||
: candidateSource
|
||||
}
|
||||
catch (error) {
|
||||
if (isAbortError(error))
|
||||
throw error
|
||||
lastProviderError = error
|
||||
continue
|
||||
}
|
||||
|
||||
await deps.llm.stream(options.model, options.chatProvider, newMessages as Message[], {
|
||||
headers,
|
||||
requestCorrelation: {
|
||||
conversationId: correlation.conversationId,
|
||||
roundId: correlation.roundId,
|
||||
},
|
||||
tools: options.tools,
|
||||
waitForTools: true,
|
||||
onMessages: (messages) => {
|
||||
const currentTurnMessages = messages.slice(providerInputMessageCount)
|
||||
const hasToolRound = currentTurnMessages.some(message =>
|
||||
message.role === 'tool'
|
||||
|| (message.role === 'assistant' && Boolean(message.tool_calls?.length)),
|
||||
)
|
||||
if (!candidate)
|
||||
continue
|
||||
|
||||
if (hasToolRound)
|
||||
providerTranscript = structuredClone(currentTurnMessages)
|
||||
},
|
||||
onUsage: (usage) => {
|
||||
generationUsage = usage
|
||||
deps.onLlmGeneration?.({
|
||||
...correlation,
|
||||
model: options.model,
|
||||
provider: activeProvider,
|
||||
inputTokens: usage.inputTokens,
|
||||
outputTokens: usage.outputTokens,
|
||||
totalTokens: usage.totalTokens,
|
||||
usageSource: usage.source,
|
||||
currentCandidate = candidate
|
||||
llmRequestStartedAt = monotonicNow()
|
||||
let attemptProducedOutput = false
|
||||
let llmFirstTokenEmitted = false
|
||||
generationUsage = { source: 'unavailable' }
|
||||
generationUsageReported = false
|
||||
providerTranscript = undefined
|
||||
|
||||
deps.onLlmRequestStarted?.({
|
||||
...correlation,
|
||||
model: candidate.model,
|
||||
provider: candidate.providerId || 'unknown',
|
||||
hasVoice: !!options.input,
|
||||
})
|
||||
|
||||
try {
|
||||
await deps.llm.stream(candidate.model, candidate.chatProvider, newMessages as Message[], {
|
||||
headers,
|
||||
providerId: candidate.providerId,
|
||||
requestCorrelation: {
|
||||
conversationId: correlation.conversationId,
|
||||
roundId: correlation.roundId,
|
||||
},
|
||||
tools: options.tools,
|
||||
waitForTools: true,
|
||||
onMessages: (messages) => {
|
||||
const currentTurnMessages = messages.slice(providerInputMessageCount)
|
||||
const hasToolRound = currentTurnMessages.some(message =>
|
||||
message.role === 'tool'
|
||||
|| (message.role === 'assistant' && Boolean(message.tool_calls?.length)),
|
||||
)
|
||||
|
||||
if (hasToolRound)
|
||||
providerTranscript = structuredClone(currentTurnMessages)
|
||||
},
|
||||
onUsage: (usage) => {
|
||||
generationUsage = usage
|
||||
generationUsageReported = true
|
||||
},
|
||||
onStreamEvent: async (event: StreamEvent) => {
|
||||
if (event.type !== 'error' && event.type !== 'finish')
|
||||
attemptProducedOutput = true
|
||||
|
||||
switch (event.type) {
|
||||
case 'tool-call':
|
||||
toolCallQueue.enqueue({
|
||||
type: 'tool-call',
|
||||
toolCall: event,
|
||||
})
|
||||
|
||||
break
|
||||
case 'tool-result':
|
||||
toolCallQueue.enqueue({
|
||||
type: 'tool-call-result',
|
||||
id: event.toolCallId,
|
||||
result: event.result,
|
||||
})
|
||||
|
||||
break
|
||||
case 'tool-error':
|
||||
toolCallQueue.enqueue({
|
||||
type: 'tool-call-result',
|
||||
id: event.toolCallId,
|
||||
isError: true,
|
||||
result: event.result,
|
||||
})
|
||||
|
||||
break
|
||||
case 'text-delta':
|
||||
if (!llmFirstTokenEmitted) {
|
||||
llmFirstTokenEmitted = true
|
||||
deps.onLlmFirstToken?.({
|
||||
...correlation,
|
||||
model: candidate.model,
|
||||
ttfbMs: Math.round(monotonicNow() - llmRequestStartedAt),
|
||||
})
|
||||
}
|
||||
fullText += event.text
|
||||
await parser.consume(event.text)
|
||||
break
|
||||
case 'reasoning-delta': {
|
||||
if (shouldAbort())
|
||||
return
|
||||
|
||||
const { reasoning = '' } = buildingMessage.categorization ?? {}
|
||||
const nextReasoning = reasoning + event.text
|
||||
buildingMessage.categorization = {
|
||||
speech: typeof buildingMessage.content === 'string' ? buildingMessage.content : '',
|
||||
reasoning: nextReasoning,
|
||||
}
|
||||
const crossesBoundary
|
||||
= Math.floor(nextReasoning.length / REASONING_UI_FLUSH_CHUNK_SIZE)
|
||||
> Math.floor(reasoning.length / REASONING_UI_FLUSH_CHUNK_SIZE)
|
||||
if (!reasoning || crossesBoundary)
|
||||
patchForegroundStream(sessionId, buildingMessage)
|
||||
break
|
||||
}
|
||||
case 'finish':
|
||||
break
|
||||
case 'error':
|
||||
throw event.error ?? new Error('Stream error')
|
||||
}
|
||||
},
|
||||
})
|
||||
},
|
||||
onStreamEvent: async (event: StreamEvent) => {
|
||||
switch (event.type) {
|
||||
case 'tool-call':
|
||||
toolCallQueue.enqueue({
|
||||
type: 'tool-call',
|
||||
toolCall: event,
|
||||
})
|
||||
|
||||
break
|
||||
case 'tool-result':
|
||||
toolCallQueue.enqueue({
|
||||
type: 'tool-call-result',
|
||||
id: event.toolCallId,
|
||||
result: event.result,
|
||||
})
|
||||
generationCompleted = true
|
||||
break
|
||||
}
|
||||
catch (error) {
|
||||
if (attemptProducedOutput || !canFallbackFrom(error))
|
||||
throw error
|
||||
|
||||
break
|
||||
case 'tool-error':
|
||||
toolCallQueue.enqueue({
|
||||
type: 'tool-call-result',
|
||||
id: event.toolCallId,
|
||||
isError: true,
|
||||
result: event.result,
|
||||
})
|
||||
lastProviderError = error
|
||||
console.warn(
|
||||
`[chat] Provider "${candidate.providerId}" failed before output. Trying the next configured provider.`,
|
||||
error,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
break
|
||||
case 'text-delta':
|
||||
if (!llmFirstTokenEmitted) {
|
||||
llmFirstTokenEmitted = true
|
||||
deps.onLlmFirstToken?.({
|
||||
...correlation,
|
||||
model: options.model,
|
||||
ttfbMs: Math.round(monotonicNow() - llmRequestStartedAt),
|
||||
})
|
||||
}
|
||||
fullText += event.text
|
||||
await parser.consume(event.text)
|
||||
break
|
||||
case 'reasoning-delta': {
|
||||
if (shouldAbort())
|
||||
return
|
||||
if (!generationCompleted)
|
||||
throw lastProviderError ?? new Error('No available chat provider or model found')
|
||||
|
||||
const { reasoning = '' } = buildingMessage.categorization ?? {}
|
||||
const nextReasoning = reasoning + event.text
|
||||
buildingMessage.categorization = {
|
||||
speech: typeof buildingMessage.content === 'string' ? buildingMessage.content : '',
|
||||
reasoning: nextReasoning,
|
||||
}
|
||||
const crossesBoundary
|
||||
= Math.floor(nextReasoning.length / REASONING_UI_FLUSH_CHUNK_SIZE)
|
||||
> Math.floor(reasoning.length / REASONING_UI_FLUSH_CHUNK_SIZE)
|
||||
if (!reasoning || crossesBoundary)
|
||||
patchForegroundStream(sessionId, buildingMessage)
|
||||
break
|
||||
}
|
||||
case 'finish':
|
||||
break
|
||||
case 'error':
|
||||
throw event.error ?? new Error('Stream error')
|
||||
}
|
||||
},
|
||||
})
|
||||
if (generationUsageReported) {
|
||||
deps.onLlmGeneration?.({
|
||||
...correlation,
|
||||
model: currentCandidate.model,
|
||||
provider: currentCandidate.providerId,
|
||||
inputTokens: generationUsage.inputTokens,
|
||||
outputTokens: generationUsage.outputTokens,
|
||||
totalTokens: generationUsage.totalTokens,
|
||||
usageSource: generationUsage.source,
|
||||
})
|
||||
}
|
||||
|
||||
await parser.end()
|
||||
buildingMessage.providerTranscript = providerTranscript
|
||||
deps.onAssistantResponseRendered?.({
|
||||
...correlation,
|
||||
model: options.model,
|
||||
model: currentCandidate.model,
|
||||
latencyMs: Math.round(monotonicNow() - llmRequestStartedAt),
|
||||
})
|
||||
|
||||
@@ -863,7 +991,7 @@ export function createChatOrchestratorRuntime(deps: ChatOrchestratorRuntimeDeps)
|
||||
...correlation,
|
||||
durationMs,
|
||||
hasVoice: !!options.input,
|
||||
model: options.model,
|
||||
model: currentCandidate.model,
|
||||
inputTokens: generationUsage.inputTokens,
|
||||
outputTokens: generationUsage.outputTokens,
|
||||
totalTokens: generationUsage.totalTokens,
|
||||
@@ -874,8 +1002,8 @@ export function createChatOrchestratorRuntime(deps: ChatOrchestratorRuntimeDeps)
|
||||
...correlation,
|
||||
durationMs,
|
||||
source: sendSource,
|
||||
model: options.model,
|
||||
provider: activeProvider,
|
||||
model: currentCandidate.model,
|
||||
provider: currentCandidate.providerId,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -884,8 +1012,8 @@ export function createChatOrchestratorRuntime(deps: ChatOrchestratorRuntimeDeps)
|
||||
deps.onMessageRoundFailed?.({
|
||||
...correlation,
|
||||
source: sendSource,
|
||||
model: options.model,
|
||||
provider: activeProvider,
|
||||
model: currentCandidate.model,
|
||||
provider: currentCandidate.providerId,
|
||||
failureStage: 'llm_response',
|
||||
errorCode: 'llm_response_failed',
|
||||
})
|
||||
@@ -893,8 +1021,8 @@ export function createChatOrchestratorRuntime(deps: ChatOrchestratorRuntimeDeps)
|
||||
deps.onChatActivationFailed?.({
|
||||
...correlation,
|
||||
source: sendSource,
|
||||
model: options.model,
|
||||
provider: activeProvider,
|
||||
model: currentCandidate.model,
|
||||
provider: currentCandidate.providerId,
|
||||
failureStage: 'llm_response',
|
||||
errorCode: 'llm_response_failed',
|
||||
})
|
||||
|
||||
@@ -24,6 +24,8 @@ export type StreamEvent
|
||||
export interface StreamOptions {
|
||||
abortSignal?: AbortSignal
|
||||
headers?: Record<string, string>
|
||||
/** Provider that owns this request. Internal adapters use this for provider-specific transport metadata. */
|
||||
providerId?: string
|
||||
onStreamEvent?: (event: StreamEvent) => void | Promise<void>
|
||||
/** Called once with the final xsAI message list after all tool rounds finish. */
|
||||
onMessages?: (messages: Message[]) => void | Promise<void>
|
||||
|
||||
@@ -72,11 +72,16 @@ const persistSessionMessagesMock = vi.fn()
|
||||
const forkSessionMock = vi.fn()
|
||||
const ensureSessionMock = vi.fn()
|
||||
const getProviderInstanceMock = vi.fn()
|
||||
const fetchModelsForProviderMock = vi.fn()
|
||||
const getModelsForProviderMock = vi.fn()
|
||||
const supportsModelListingMock = vi.fn()
|
||||
const getToolsByNamesMock = vi.fn<(names: string[]) => Tool[]>()
|
||||
|
||||
const activeSessionIdRef = ref('session-1')
|
||||
const activeProviderRef = ref('mock-provider')
|
||||
const activeModelRef = ref('gpt-test')
|
||||
const configuredChatProvidersMetadataRef = ref<Array<{ id: string }>>([{ id: 'mock-provider' }])
|
||||
const isAuthenticatedRef = ref(true)
|
||||
const streamingMessageRef = ref<any>({ role: 'assistant', content: '', slices: [], tool_results: [] })
|
||||
const sessionMessages: Record<string, any[]> = {}
|
||||
let currentGeneration = 1
|
||||
@@ -226,7 +231,17 @@ vi.mock('./ai/chat-llm/tools', () => ({
|
||||
|
||||
vi.mock('./providers/provider', () => ({
|
||||
useProviderStore: () => ({
|
||||
configuredChatProvidersMetadata: configuredChatProvidersMetadataRef,
|
||||
fetchModelsForProvider: fetchModelsForProviderMock,
|
||||
getModelsForProvider: getModelsForProviderMock,
|
||||
getProviderInstance: getProviderInstanceMock,
|
||||
supportsModelListing: supportsModelListingMock,
|
||||
}),
|
||||
}))
|
||||
|
||||
vi.mock('./auth', () => ({
|
||||
useAuthStore: () => ({
|
||||
isAuthenticated: isAuthenticatedRef,
|
||||
}),
|
||||
}))
|
||||
|
||||
@@ -286,6 +301,9 @@ describe('chat store contract', () => {
|
||||
forkSessionMock.mockReset()
|
||||
ensureSessionMock.mockReset()
|
||||
getProviderInstanceMock.mockReset().mockResolvedValue(provider)
|
||||
fetchModelsForProviderMock.mockReset().mockResolvedValue([])
|
||||
getModelsForProviderMock.mockReset().mockReturnValue([])
|
||||
supportsModelListingMock.mockReset().mockReturnValue(false)
|
||||
getToolsByNamesMock.mockReset().mockImplementation(names => names.map(name => ({
|
||||
type: 'function',
|
||||
function: {
|
||||
@@ -299,6 +317,9 @@ describe('chat store contract', () => {
|
||||
ioTracerMocks.startSpanMock.mockClear()
|
||||
activeSessionIdRef.value = 'session-1'
|
||||
activeProviderRef.value = 'mock-provider'
|
||||
activeModelRef.value = 'gpt-test'
|
||||
configuredChatProvidersMetadataRef.value = [{ id: 'mock-provider' }]
|
||||
isAuthenticatedRef.value = true
|
||||
streamingMessageRef.value = { role: 'assistant', content: '', slices: [], tool_results: [] }
|
||||
currentGeneration = 1
|
||||
|
||||
@@ -338,6 +359,63 @@ describe('chat store contract', () => {
|
||||
])
|
||||
})
|
||||
|
||||
// ROOT CAUSE:
|
||||
//
|
||||
// A persisted official provider with an empty model bypassed login sync.
|
||||
// The send action then rejected the turn before it reached the official auto route.
|
||||
//
|
||||
// The send action now resolves the official provider and auto model together.
|
||||
it('sends with official auto when the persisted official model is empty', async () => {
|
||||
activeProviderRef.value = 'official-provider'
|
||||
activeModelRef.value = ''
|
||||
configuredChatProvidersMetadataRef.value = [{ id: 'official-provider' }]
|
||||
llmStreamMock.mockImplementation(async (_model: string, _chatProvider: ChatProvider, _messages: Message[], options: any) => {
|
||||
await options.onStreamEvent({ type: 'text-delta', text: 'ok' })
|
||||
await options.onStreamEvent({ type: 'finish', finishReason: 'stop' })
|
||||
})
|
||||
|
||||
const store = useChatStore()
|
||||
await store.send({
|
||||
sessionId: 'session-1',
|
||||
text: 'hello',
|
||||
})
|
||||
|
||||
expect(getProviderInstanceMock).toHaveBeenCalledWith('official-provider')
|
||||
expect(llmStreamMock).toHaveBeenCalledWith(
|
||||
'auto',
|
||||
provider,
|
||||
expect.any(Array),
|
||||
expect.objectContaining({ providerId: 'official-provider' }),
|
||||
)
|
||||
})
|
||||
|
||||
it('falls back from the active provider to official auto before output', async () => {
|
||||
llmStreamMock.mockRejectedValueOnce(new Error('active provider unavailable'))
|
||||
llmStreamMock.mockImplementationOnce(async (_model: string, _chatProvider: ChatProvider, _messages: Message[], options: any) => {
|
||||
await options.onStreamEvent({ type: 'text-delta', text: 'official reply' })
|
||||
await options.onStreamEvent({ type: 'finish', finishReason: 'stop' })
|
||||
})
|
||||
|
||||
const store = useChatStore()
|
||||
await store.send({
|
||||
sessionId: 'session-1',
|
||||
text: 'use the next provider',
|
||||
})
|
||||
|
||||
expect(getProviderInstanceMock.mock.calls.map(([providerId]) => providerId)).toEqual([
|
||||
'mock-provider',
|
||||
'official-provider',
|
||||
])
|
||||
expect(llmStreamMock.mock.calls.map(([model]) => model)).toEqual(['gpt-test', 'auto'])
|
||||
expect(llmStreamMock.mock.calls[1]?.[3]?.headers).toEqual({
|
||||
[AIRI_CHAT_APP_SURFACE_HEADER]: 'web',
|
||||
[AIRI_CHAT_SESSION_ID_HEADER]: 'session-1',
|
||||
[AIRI_CHAT_ROUND_ID_HEADER]: expect.any(String),
|
||||
})
|
||||
expect(sessionMessages['session-1']?.filter(message => message.role === 'user')).toHaveLength(1)
|
||||
expect(sessionMessages['session-1']?.filter(message => message.role === 'assistant')).toHaveLength(1)
|
||||
})
|
||||
|
||||
it('forwards one correlation identity across every PostHog chat milestone', async () => {
|
||||
llmStreamMock.mockImplementation(async (_model: string, _chatProvider: ChatProvider, _messages: Message[], options: any) => {
|
||||
await options.onStreamEvent({ type: 'text-delta', text: 'ok' })
|
||||
|
||||
@@ -27,8 +27,10 @@ import { useLLM } from './ai/chat-llm/llm'
|
||||
import { resolveLlmTools } from './ai/chat-llm/tool-resolver'
|
||||
import { useLlmToolsStore } from './ai/chat-llm/tools'
|
||||
import { useLlmToolsetPromptsStore } from './ai/chat-llm/toolset-prompts'
|
||||
import { useAuthStore } from './auth'
|
||||
import { createMinecraftContext } from './chat/context-providers'
|
||||
import { useChatContextStore } from './chat/context-store'
|
||||
import { resolveChatProviderRoute } from './chat/provider-fallback'
|
||||
import { useChatSessionStore } from './chat/session-store'
|
||||
import { useChatStreamStore } from './chat/stream-store'
|
||||
import { useContextObservabilityStore } from './devtools/context-observability'
|
||||
@@ -145,10 +147,13 @@ export const useChatStore = defineStore('chat', () => {
|
||||
// the system prompt is composed, which would expose web_search on the first turn
|
||||
// without its paired prompt-injection defense.
|
||||
useWebSearchStore()
|
||||
const authStore = useAuthStore()
|
||||
const consciousnessStore = useConsciousnessStore()
|
||||
const providerStore = useProviderStore()
|
||||
const artistryAutonomousStore = useAutonomousArtistryStore()
|
||||
const { activeModel, activeProvider } = storeToRefs(consciousnessStore)
|
||||
const { isAuthenticated } = storeToRefs(authStore)
|
||||
const { configuredChatProvidersMetadata } = storeToRefs(providerStore)
|
||||
const chatSession = useChatSessionStore()
|
||||
const chatStream = useChatStreamStore()
|
||||
const chatContext = useChatContextStore()
|
||||
@@ -172,7 +177,7 @@ export const useChatStore = defineStore('chat', () => {
|
||||
) {
|
||||
let llmTextLength = 0
|
||||
const headers = { ...options?.headers }
|
||||
if (getProviderMode(activeProvider.value) === 'official' && options?.requestCorrelation) {
|
||||
if (getProviderMode(options?.providerId ?? activeProvider.value) === 'official' && options?.requestCorrelation) {
|
||||
headers[AIRI_CHAT_SESSION_ID_HEADER] = options.requestCorrelation.conversationId
|
||||
headers[AIRI_CHAT_ROUND_ID_HEADER] = options.requestCorrelation.roundId
|
||||
headers[AIRI_CHAT_APP_SURFACE_HEADER] = getConversationAnalyticsSurface()
|
||||
@@ -343,19 +348,25 @@ export const useChatStore = defineStore('chat', () => {
|
||||
}
|
||||
|
||||
async function executeSend(payload: ChatSendPayload): Promise<ChatSendResult> {
|
||||
const providerId = activeProvider.value
|
||||
const modelId = activeModel.value
|
||||
if (!providerId || !modelId)
|
||||
throw new Error('No active chat provider or model configured')
|
||||
const route = await resolveChatProviderRoute({
|
||||
activeProvider: activeProvider.value,
|
||||
activeModel: activeModel.value,
|
||||
authenticated: isAuthenticated.value,
|
||||
configuredProviderIds: configuredChatProvidersMetadata.value.map(provider => provider.id),
|
||||
}, {
|
||||
fetchModels: providerStore.fetchModelsForProvider,
|
||||
getCachedModels: providerStore.getModelsForProvider,
|
||||
getProviderInstance: providerId => providerStore.getProviderInstance<ChatProvider>(providerId),
|
||||
supportsModelListing: providerStore.supportsModelListing,
|
||||
})
|
||||
|
||||
const messageCount = chatSession.getSessionMessages(payload.sessionId).length
|
||||
const chatProvider = await providerStore.getProviderInstance<ChatProvider>(providerId)
|
||||
if (!chatProvider)
|
||||
throw new Error(`Failed to resolve chat provider "${providerId}"`)
|
||||
|
||||
await runtime.ingest(payload.text, {
|
||||
model: modelId,
|
||||
chatProvider,
|
||||
model: route.primary.model,
|
||||
providerId: route.primary.providerId,
|
||||
chatProvider: route.primary.chatProvider,
|
||||
fallbackCandidates: route.fallbackCandidates,
|
||||
attachments: payload.attachments,
|
||||
input: payload.input,
|
||||
toolReferences: payload.tools,
|
||||
|
||||
@@ -0,0 +1,98 @@
|
||||
import type { ChatProvider } from '@xsai-ext/providers/utils'
|
||||
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
|
||||
import { resolveChatProviderRoute } from './provider-fallback'
|
||||
|
||||
function provider(id: string): ChatProvider {
|
||||
return {
|
||||
chat: () => ({ baseURL: `https://${id}.example.com/` }),
|
||||
} as unknown as ChatProvider
|
||||
}
|
||||
|
||||
function createDependencies(models: Record<string, string[]>) {
|
||||
const fetchModels = vi.fn(async (providerId: string) => (models[providerId] ?? []).map(id => ({
|
||||
id,
|
||||
name: id,
|
||||
provider: providerId,
|
||||
})))
|
||||
const getProviderInstance = vi.fn(async (providerId: string) => provider(providerId))
|
||||
|
||||
return {
|
||||
dependencies: {
|
||||
fetchModels,
|
||||
getCachedModels: vi.fn(() => []),
|
||||
getProviderInstance,
|
||||
supportsModelListing: vi.fn((providerId: string) => providerId !== 'official-provider'),
|
||||
},
|
||||
fetchModels,
|
||||
getProviderInstance,
|
||||
}
|
||||
}
|
||||
|
||||
describe('chat provider fallback', () => {
|
||||
// ROOT CAUSE:
|
||||
//
|
||||
// Login sync did not repair an existing official provider with an empty model.
|
||||
// Chat rejected that persisted state before it could use the official auto route.
|
||||
//
|
||||
// The resolver now treats official-provider and auto as one runtime invariant.
|
||||
it('uses official auto for an authenticated empty selection', async () => {
|
||||
const { dependencies, fetchModels, getProviderInstance } = createDependencies({})
|
||||
|
||||
const route = await resolveChatProviderRoute({
|
||||
activeModel: '',
|
||||
activeProvider: 'official-provider',
|
||||
authenticated: true,
|
||||
configuredProviderIds: ['official-provider'],
|
||||
}, dependencies)
|
||||
|
||||
expect(route.primary.providerId).toBe('official-provider')
|
||||
expect(route.primary.model).toBe('auto')
|
||||
expect(fetchModels).not.toHaveBeenCalled()
|
||||
expect(getProviderInstance).toHaveBeenCalledWith('official-provider')
|
||||
})
|
||||
|
||||
it('skips a missing active model and uses official auto after login', async () => {
|
||||
const { dependencies, fetchModels } = createDependencies({
|
||||
openai: ['gpt-available'],
|
||||
})
|
||||
|
||||
const route = await resolveChatProviderRoute({
|
||||
activeModel: 'gpt-removed',
|
||||
activeProvider: 'openai',
|
||||
authenticated: true,
|
||||
configuredProviderIds: ['openai'],
|
||||
}, dependencies)
|
||||
|
||||
expect(route.primary.providerId).toBe('official-provider')
|
||||
expect(route.primary.model).toBe('auto')
|
||||
expect(fetchModels).toHaveBeenCalledWith('openai')
|
||||
})
|
||||
|
||||
it('skips official while logged out and keeps configured provider order', async () => {
|
||||
const { dependencies, fetchModels, getProviderInstance } = createDependencies({
|
||||
first: [],
|
||||
second: ['second-model'],
|
||||
third: ['third-model'],
|
||||
})
|
||||
|
||||
const route = await resolveChatProviderRoute({
|
||||
activeModel: '',
|
||||
activeProvider: '',
|
||||
authenticated: false,
|
||||
configuredProviderIds: ['official-provider', 'first', 'second', 'third'],
|
||||
}, dependencies)
|
||||
|
||||
expect(route.primary.providerId).toBe('second')
|
||||
expect(route.primary.model).toBe('second-model')
|
||||
expect(fetchModels.mock.calls.map(([providerId]) => providerId)).toEqual(['first', 'second'])
|
||||
expect(getProviderInstance).toHaveBeenCalledTimes(1)
|
||||
|
||||
const thirdSource = route.fallbackCandidates[0]
|
||||
expect(typeof thirdSource).toBe('function')
|
||||
const third = typeof thirdSource === 'function' ? await thirdSource() : thirdSource
|
||||
expect(third?.providerId).toBe('third')
|
||||
expect(third?.model).toBe('third-model')
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,129 @@
|
||||
import type {
|
||||
ChatOrchestratorProviderCandidate,
|
||||
ChatOrchestratorProviderCandidateSource,
|
||||
} from '@proj-airi/core-agent'
|
||||
import type { ChatProvider } from '@xsai-ext/providers/utils'
|
||||
|
||||
import type { ModelInfo } from '../../libs/providers'
|
||||
|
||||
import { errorMessageFrom } from '@moeru/std'
|
||||
|
||||
const OFFICIAL_PROVIDER_ID = 'official-provider'
|
||||
const OFFICIAL_MODEL_ID = 'auto'
|
||||
|
||||
interface ChatProviderFallbackDependencies {
|
||||
fetchModels: (providerId: string) => Promise<ModelInfo[]>
|
||||
getCachedModels: (providerId: string) => ModelInfo[]
|
||||
getProviderInstance: (providerId: string) => Promise<ChatProvider>
|
||||
supportsModelListing: (providerId: string) => boolean
|
||||
}
|
||||
|
||||
interface ChatProviderFallbackOptions {
|
||||
activeModel: string
|
||||
activeProvider: string
|
||||
authenticated: boolean
|
||||
configuredProviderIds: string[]
|
||||
}
|
||||
|
||||
export interface ResolvedChatProviderRoute {
|
||||
/** Ordered candidates that the runtime resolves only after the primary provider fails. */
|
||||
fallbackCandidates: ChatOrchestratorProviderCandidateSource[]
|
||||
/** First usable provider and model for the chat turn. */
|
||||
primary: ChatOrchestratorProviderCandidate
|
||||
}
|
||||
|
||||
interface ChatProviderCandidateSpec {
|
||||
model?: string
|
||||
providerId: string
|
||||
selectFirstListedModel: boolean
|
||||
}
|
||||
|
||||
function buildCandidateSpecs(options: ChatProviderFallbackOptions): ChatProviderCandidateSpec[] {
|
||||
const specs: ChatProviderCandidateSpec[] = []
|
||||
const addedProviderIds = new Set<string>()
|
||||
|
||||
const append = (providerId: string, model: string | undefined, selectFirstListedModel: boolean) => {
|
||||
if (!providerId || addedProviderIds.has(providerId))
|
||||
return
|
||||
if (providerId === OFFICIAL_PROVIDER_ID && !options.authenticated)
|
||||
return
|
||||
|
||||
addedProviderIds.add(providerId)
|
||||
specs.push({ providerId, model, selectFirstListedModel })
|
||||
}
|
||||
|
||||
append(options.activeProvider, options.activeModel, false)
|
||||
if (options.authenticated)
|
||||
append(OFFICIAL_PROVIDER_ID, OFFICIAL_MODEL_ID, false)
|
||||
for (const providerId of options.configuredProviderIds)
|
||||
append(providerId, undefined, true)
|
||||
|
||||
return specs
|
||||
}
|
||||
|
||||
async function resolveCandidate(
|
||||
spec: ChatProviderCandidateSpec,
|
||||
dependencies: ChatProviderFallbackDependencies,
|
||||
): Promise<ChatOrchestratorProviderCandidate | undefined> {
|
||||
try {
|
||||
let model = spec.providerId === OFFICIAL_PROVIDER_ID ? OFFICIAL_MODEL_ID : spec.model?.trim()
|
||||
if (!model && !spec.selectFirstListedModel)
|
||||
return undefined
|
||||
|
||||
if (dependencies.supportsModelListing(spec.providerId)) {
|
||||
const cachedModels = dependencies.getCachedModels(spec.providerId)
|
||||
const models = cachedModels.length > 0
|
||||
? cachedModels
|
||||
: await dependencies.fetchModels(spec.providerId)
|
||||
|
||||
if (model && !models.some(item => item.id === model))
|
||||
return undefined
|
||||
|
||||
if (!model && spec.selectFirstListedModel)
|
||||
model = models.find(item => !!item.id)?.id
|
||||
}
|
||||
|
||||
if (!model)
|
||||
return undefined
|
||||
|
||||
const chatProvider = await dependencies.getProviderInstance(spec.providerId)
|
||||
return {
|
||||
providerId: spec.providerId,
|
||||
model,
|
||||
chatProvider,
|
||||
}
|
||||
}
|
||||
catch (error) {
|
||||
console.warn(
|
||||
`[chat] Provider "${spec.providerId}" is not available for fallback: ${errorMessageFrom(error) ?? 'Unknown error'}`,
|
||||
)
|
||||
return undefined
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves the first usable chat provider and keeps the remaining fixed-order candidates lazy.
|
||||
*
|
||||
* The active provider has first priority. The official provider has second priority after login.
|
||||
* Other configured providers keep registry order and use the first listed model.
|
||||
*/
|
||||
export async function resolveChatProviderRoute(
|
||||
options: ChatProviderFallbackOptions,
|
||||
dependencies: ChatProviderFallbackDependencies,
|
||||
): Promise<ResolvedChatProviderRoute> {
|
||||
const sources = buildCandidateSpecs(options)
|
||||
.map(spec => () => resolveCandidate(spec, dependencies))
|
||||
|
||||
for (const [index, source] of sources.entries()) {
|
||||
const primary = await source()
|
||||
if (!primary)
|
||||
continue
|
||||
|
||||
return {
|
||||
primary,
|
||||
fallbackCandidates: sources.slice(index + 1),
|
||||
}
|
||||
}
|
||||
|
||||
throw new Error('No available chat provider or model found')
|
||||
}
|
||||
Reference in New Issue
Block a user