fix(stage-ui,server-*): implement handling for input:text from anywhere, support for Discord (#928)

---------

Co-authored-by: Neko <neko@ayaka.moe>
This commit is contained in:
skyline624
2026-01-11 03:35:32 +08:00
committed by GitHub
co-authored by Neko
parent a7d7c1631e
commit 5d9040e430
15 changed files with 553 additions and 122 deletions
+3
View File
@@ -119,3 +119,6 @@ plugins/local
plugins/development
plugins-local
plugins-development
#ia
GEMINI.md
@@ -17,9 +17,8 @@ export async function setupServerChannel() {
const app = serverRuntime.setupApp()
const serverInstance = serve(app, {
// TODO: fix types
// @ts-expect-error - the .crossws property wasn't extended in types
plugins: [ws({ resolve: async req => (await app.fetch(req)).crossws })],
// TODO: add proper crossws typing upstream
plugins: [ws({ resolve: async req => (await app.fetch(req) as any).crossws })],
port: env.PORT ? Number(env.PORT) : 6121,
hostname: env.SERVER_RUNTIME_HOSTNAME || 'localhost',
reusePort: true,
+1 -1
View File
@@ -11,7 +11,7 @@
"license": "MIT",
"scripts": {
"build": "vite build",
"dev": "vite",
"dev": "vite --host",
"lint": "eslint .",
"preview": "vite preview",
"typecheck": "vue-tsc --noEmit"
@@ -1,4 +1,4 @@
import type { AssistantMessage, Message, ToolMessage, UserMessage } from '@xsai/shared-chat'
import type { AssistantMessage, CommonContentPart, Message, ToolMessage, UserMessage } from '@xsai/shared-chat'
export interface DiscordGuildMember {
nickname: string
@@ -13,9 +13,25 @@ export interface Discord {
}
export interface MetadataEventSource {
/**
* Stable module/plugin identifier (shared across instances).
* Example: "telegram-bot", "stage-tamagotchi".
*/
plugin: string
/**
* Unique instance id for this module run (per process/deployment).
* Example: "telegram-01", "stage-ui-2f7c9".
*/
instanceId: string
/**
* Optional semantic version for the module/plugin.
* Example: "0.8.1-beta.7".
*/
version?: string
/**
* K8s-style labels for routing and policy selectors.
* Example: { env: "prod", app: "telegram", devtools: "true" }.
*/
labels?: Record<string, string>
}
@@ -59,9 +75,10 @@ interface InputSource {
interface OutputSource {
'gen-ai:chat': {
input: UserMessage
contexts: Record<string, ContextUpdate[]>
message: UserMessage
contexts: Record<string, ContextUpdate<Record<string, any>, string | CommonContentPart[]>[]>
composedMessage: Array<Message>
input?: WebSocketEventInputs
}
}
@@ -104,6 +121,45 @@ export interface ContextUpdate<
metadata?: Metadata
}
export interface InputMessageOverrides {
sessionId?: string
messagePrefix?: string
}
export type InputContextUpdate
= Omit<ContextUpdate<Record<string, unknown>, string | CommonContentPart[]>, 'id' | 'contextId'>
& Partial<Pick<ContextUpdate<Record<string, unknown>, string | CommonContentPart[]>, 'id' | 'contextId'>>
export interface WebSocketEventInputTextBase {
text: string
textRaw?: string
overrides?: InputMessageOverrides
contextUpdates?: InputContextUpdate[]
}
export type WebSocketEventInputText = WebSocketEventInputTextBase & Partial<WithInputSource<'stage-web' | 'stage-tamagotchi' | 'discord'>>
export interface WebSocketEventInputTextVoiceBase {
transcription: string
textRaw?: string
overrides?: InputMessageOverrides
contextUpdates?: InputContextUpdate[]
}
export type WebSocketEventInputTextVoice = WebSocketEventInputTextVoiceBase & Partial<WithInputSource<'stage-web' | 'stage-tamagotchi' | 'discord'>>
export interface WebSocketEventInputVoiceBase {
audio: ArrayBuffer
overrides?: InputMessageOverrides
contextUpdates?: InputContextUpdate[]
}
export type WebSocketEventInputVoice = WebSocketEventInputVoiceBase & Partial<WithInputSource<'stage-web' | 'stage-tamagotchi' | 'discord'>>
export type WebSocketEventDataInputs = WebSocketEventInputText | WebSocketEventInputTextVoice | WebSocketEventInputVoice
export type WebSocketEventInputs = WebSocketBaseEvent<'input:text' | 'input:text:voice' | 'input:voice', WebSocketEventInputText | WebSocketEventInputTextVoice | WebSocketEventInputVoice>
export interface WebSocketBaseEvent<T, D, S extends string = string> {
type: T
data: D
@@ -155,15 +211,9 @@ export interface WebSocketEvents<C = undefined> {
config: C | Record<string, unknown>
}
'input:text': {
text: string
} & Partial<WithInputSource<'stage-web' | 'stage-tamagotchi' | 'discord'>>
'input:text:voice': {
transcription: string
} & Partial<WithInputSource<'stage-web' | 'stage-tamagotchi' | 'discord'>>
'input:voice': {
audio: ArrayBuffer
} & Partial<WithInputSource<'stage-web' | 'stage-tamagotchi' | 'discord'>>
'input:text': WebSocketEventInputText
'input:text:voice': WebSocketEventInputTextVoice
'input:voice': WebSocketEventInputVoice
'output:gen-ai:chat:tool-call': {
toolCalls: ToolMessage[]
@@ -9,6 +9,7 @@ import { ContextUpdateStrategy } from '@proj-airi/server-sdk'
import { useCharacterOrchestratorStore, useCharacterStore } from '@proj-airi/stage-ui/stores/character'
import { CHAT_STREAM_CHANNEL_NAME, CONTEXT_CHANNEL_NAME, useChatStore } from '@proj-airi/stage-ui/stores/chat'
import { useModsServerChannelStore } from '@proj-airi/stage-ui/stores/mods/api/channel-server'
import { getEventSourceKey } from '@proj-airi/stage-ui/utils'
import { Callout } from '@proj-airi/ui'
import { useBroadcastChannel } from '@vueuse/core'
import { nanoid } from 'nanoid'
@@ -331,7 +332,7 @@ onMounted(() => {
channel: 'server',
type: event.type,
summary: [
`source=${event.source}`,
`source=${getEventSourceKey(event)}`,
`strategy=${event.data.strategy}`,
summarizeContextUpdate(event.data),
].filter(Boolean).join(' '),
@@ -484,7 +485,7 @@ watch(incomingContext, (event) => {
channel: 'broadcast',
type: 'context:broadcast',
summary: [
`source=${event.source}`,
`source=${getEventSourceKey(event)}`,
`strategy=${event.strategy}`,
summarizeContextUpdate(event),
].filter(Boolean).join(' '),
@@ -10,6 +10,8 @@ import { nanoid } from 'nanoid'
import { validate } from 'xsschema'
import { z } from 'zod'
import { getEventSourceKey } from '../../../../../utils'
export interface SparkNotifyCommandDraft {
destinations: string[]
interrupt?: 'force' | 'soft' | boolean
@@ -172,7 +174,7 @@ export function setupAgentSparkNotifyHandler(deps: SparkNotifyAgentDeps) {
role: 'system',
content: [
deps.getSystemPrompt(),
getSparkNotifyHandlingAgentInstruction(event.source),
getSparkNotifyHandlingAgentInstruction(getEventSourceKey(event)),
].filter(Boolean).join('\n\n'),
}
+57 -26
View File
@@ -1,3 +1,4 @@
import type { WebSocketEventInputs } from '@proj-airi/server-sdk'
import type { ChatProvider } from '@xsai-ext/providers/utils'
import type { CommonContentPart, Message, SystemMessage, ToolMessage } from '@xsai/shared-chat'
@@ -14,6 +15,7 @@ import { useAnalytics } from '../composables'
import { useLlmmarkerParser } from '../composables/llm-marker-parser'
import { categorizeResponse, createStreamingCategorizer } from '../composables/response-categoriser'
import { useLLM } from '../stores/llm'
import { getEventSourceKey } from '../utils/event-source'
import { useCharacterStore } from './character'
import { useConsciousnessStore } from './modules/consciousness'
@@ -44,6 +46,7 @@ export const useChatStore = defineStore('chat', () => {
providerConfig?: Record<string, unknown>
attachments?: { type: 'image', data: string, mimeType: string }[]
tools?: StreamOptions['tools']
input?: WebSocketEventInputs
}
interface QueuedSend {
@@ -331,15 +334,16 @@ export const useChatStore = defineStore('chat', () => {
}, { immediate: true })
function ingestContextMessage(envelope: ContextMessage) {
if (!activeContexts.value[envelope.source]) {
activeContexts.value[envelope.source] = []
const sourceKey = getEventSourceKey(envelope)
if (!activeContexts.value[sourceKey]) {
activeContexts.value[sourceKey] = []
}
if (envelope.strategy === ContextUpdateStrategy.ReplaceSelf) {
activeContexts.value[envelope.source] = [envelope]
activeContexts.value[sourceKey] = [envelope]
}
else if (envelope.strategy === ContextUpdateStrategy.AppendSelf) {
activeContexts.value[envelope.source].push(envelope)
activeContexts.value[sourceKey].push(envelope)
}
}
@@ -357,9 +361,10 @@ export const useChatStore = defineStore('chat', () => {
const sendingCreatedAt = Date.now()
const streamingMessageContext: ChatStreamEventContext = {
input: { role: 'user', content: sendingMessage, createdAt: sendingCreatedAt },
message: { role: 'user', content: sendingMessage, createdAt: sendingCreatedAt },
contexts: toRaw(activeContexts.value),
composedMessage: [],
input: options.input,
}
const isStaleGeneration = () => getSessionGeneration(sessionId) !== generation
@@ -369,7 +374,21 @@ export const useChatStore = defineStore('chat', () => {
sending.value = true
streamingMessage.value = { role: 'assistant', content: '', slices: [], tool_results: [], createdAt: Date.now() }
const isForegroundSession = () => sessionId === activeSessionId.value
// Use a local object for building the message to avoid polluting the UI for background sessions
const buildingMessage: StreamingAssistantMessage = { role: 'assistant', content: '', slices: [], tool_results: [], createdAt: Date.now() }
// NOTICE: Clone into reactive state only for the foreground session
// to avoid background streams mutating UI.
const updateUI = () => {
if (isForegroundSession()) {
streamingMessage.value = JSON.parse(JSON.stringify(buildingMessage))
}
}
// Initialize UI if foreground
updateUI()
trackFirstMessage()
try {
@@ -391,7 +410,14 @@ export const useChatStore = defineStore('chat', () => {
}
const finalContent = contentParts.length > 1 ? contentParts : sendingMessage
streamingMessageContext.input.content = finalContent
if (!streamingMessageContext.input) {
streamingMessageContext.input = {
type: 'input:text',
data: {
text: sendingMessage,
},
}
}
if (shouldAbort())
return
@@ -408,7 +434,6 @@ export const useChatStore = defineStore('chat', () => {
if (shouldAbort())
return
console.log('literal', literal)
// Feed to categorizer first
categorizer.consume(literal)
@@ -420,24 +445,24 @@ export const useChatStore = defineStore('chat', () => {
// Only process non-empty speech content (filter empty/whitespace-only chunks)
// Preserve spacing in chunks with content for proper word boundaries
if (speechOnly.trim()) {
streamingMessage.value.content += speechOnly
buildingMessage.content += speechOnly
console.log('speechOnly', speechOnly)
// Emit TTS only for speech parts, not reasoning (clean data, no empty chunks)
await emitTokenLiteralHooks(speechOnly, streamingMessageContext)
// Add speech content to slices for rendering
// merge text slices for markdown
const lastSlice = streamingMessage.value.slices.at(-1)
const lastSlice = buildingMessage.slices.at(-1)
if (lastSlice?.type === 'text') {
lastSlice.text += speechOnly
return
}
streamingMessage.value.slices.push({
type: 'text',
text: speechOnly,
})
else {
buildingMessage.slices.push({
type: 'text',
text: speechOnly,
})
}
updateUI()
}
},
onSpecial: async (special) => {
@@ -454,10 +479,11 @@ export const useChatStore = defineStore('chat', () => {
const finalCategorization = categorizeResponse(fullText, activeProvider.value)
// Always store categorization (even if empty) for consistency and memory features
streamingMessage.value.categorization = {
buildingMessage.categorization = {
speech: finalCategorization.speech,
reasoning: finalCategorization.reasoning,
}
updateUI()
},
minLiteralEmitLength: 24, // Avoid emitting literals too fast. This is a magic number and can be changed later.
})
@@ -468,12 +494,14 @@ export const useChatStore = defineStore('chat', () => {
if (shouldAbort())
return
if (ctx.data.type === 'tool-call') {
streamingMessage.value.slices.push(ctx.data)
buildingMessage.slices.push(ctx.data)
updateUI()
return
}
if (ctx.data.type === 'tool-call-result') {
streamingMessage.value.tool_results.push(ctx.data)
buildingMessage.tool_results.push(ctx.data)
updateUI()
}
},
],
@@ -565,8 +593,8 @@ export const useChatStore = defineStore('chat', () => {
await parser.end()
// Add the completed message to the history only if it has content
if (!isStaleGeneration() && streamingMessage.value.slices.length > 0) {
sessionMessagesForSend.push(toRaw(streamingMessage.value))
if (!isStaleGeneration() && buildingMessage.slices.length > 0) {
sessionMessagesForSend.push(toRaw(buildingMessage))
}
// Call the end-of-stream hooks
@@ -576,15 +604,17 @@ export const useChatStore = defineStore('chat', () => {
await emitAssistantResponseEndHooks(fullText, streamingMessageContext)
await emitAfterSendHooks(sendingMessage, streamingMessageContext)
await emitAssistantMessageHooks({ ...streamingMessage.value }, fullText, streamingMessageContext)
await emitAssistantMessageHooks({ ...buildingMessage }, fullText, streamingMessageContext)
await emitChatTurnCompleteHooks({
output: { ...streamingMessage.value },
output: { ...buildingMessage },
outputText: fullText,
toolCalls: sessionMessagesForSend.filter(msg => msg.role === 'tool') as ToolMessage[],
}, streamingMessageContext)
// Reset the streaming message for the next turn
streamingMessage.value = { role: 'assistant', content: '', slices: [], tool_results: [] }
if (isForegroundSession()) {
streamingMessage.value = { role: 'assistant', content: '', slices: [], tool_results: [] }
}
}
catch (error) {
console.error('Error sending message:', error)
@@ -628,8 +658,9 @@ export const useChatStore = defineStore('chat', () => {
async function send(
sendingMessage: string,
options: SendOptions,
targetSessionId?: string,
) {
const sessionId = activeSessionId.value
const sessionId = targetSessionId || activeSessionId.value
const generation = getSessionGeneration(sessionId)
return new Promise<void>((resolve, reject) => {
@@ -1,3 +1,4 @@
import type { ChatProvider } from '@xsai-ext/providers/utils'
import type { UserMessage } from '@xsai/shared-chat'
import type { ChatStreamEvent, ContextMessage } from '../../../types/chat'
@@ -5,10 +6,13 @@ import type { ChatStreamEvent, ContextMessage } from '../../../types/chat'
import { isStageTamagotchi, isStageWeb } from '@proj-airi/stage-shared'
import { useBroadcastChannel } from '@vueuse/core'
import { Mutex } from 'es-toolkit'
import { defineStore } from 'pinia'
import { nanoid } from 'nanoid'
import { defineStore, storeToRefs } from 'pinia'
import { ref, toRaw, watch } from 'vue'
import { CHAT_STREAM_CHANNEL_NAME, CONTEXT_CHANNEL_NAME, useChatStore } from '../../chat'
import { useConsciousnessStore } from '../../modules/consciousness'
import { useProvidersStore } from '../../providers'
import { useModsServerChannelStore } from './channel-server'
export const useContextBridgeStore = defineStore('mods:api:context-bridge', () => {
@@ -16,6 +20,9 @@ export const useContextBridgeStore = defineStore('mods:api:context-bridge', () =
const chatStore = useChatStore()
const serverChannelStore = useModsServerChannelStore()
const consciousnessStore = useConsciousnessStore()
const providersStore = useProvidersStore()
const { activeProvider, activeModel } = storeToRefs(consciousnessStore)
const { post: broadcastContext, data: incomingContext } = useBroadcastChannel<ContextMessage, ContextMessage>({ name: CONTEXT_CHANNEL_NAME })
const { post: broadcastStreamEvent, data: incomingStreamEvent } = useBroadcastChannel<ChatStreamEvent, ChatStreamEvent>({ name: CHAT_STREAM_CHANNEL_NAME })
@@ -36,8 +43,65 @@ export const useContextBridgeStore = defineStore('mods:api:context-bridge', () =
disposeHookFns.value.push(stop)
disposeHookFns.value.push(serverChannelStore.onContextUpdate((event) => {
chatStore.ingestContextMessage({ source: event.source, createdAt: Date.now(), ...event.data })
broadcastContext(toRaw(event.data) as ContextMessage)
const contextMessage: ContextMessage = {
...event.data,
metadata: event.metadata,
createdAt: Date.now(),
}
chatStore.ingestContextMessage(contextMessage)
broadcastContext(toRaw(contextMessage))
}))
disposeHookFns.value.push(serverChannelStore.onEvent('input:text', async (event) => {
const {
text,
textRaw,
overrides,
contextUpdates,
} = event.data
const normalizedContextUpdates = contextUpdates?.map((update) => {
const id = update.id ?? nanoid()
const contextId = update.contextId ?? id
return {
...update,
id,
contextId,
}
})
if (normalizedContextUpdates?.length) {
const createdAt = Date.now()
for (const update of normalizedContextUpdates) {
chatStore.ingestContextMessage({
...update,
metadata: event.metadata,
createdAt,
})
}
}
if (activeProvider.value && activeModel.value) {
const chatProvider = await providersStore.getProviderInstance<ChatProvider>(activeProvider.value)
let messageText = text
if (overrides?.messagePrefix)
messageText = `${overrides.messagePrefix}${text}`
await chatStore.send(messageText, {
model: activeModel.value,
chatProvider,
input: {
type: 'input:text',
data: {
text,
textRaw,
overrides,
contextUpdates: normalizedContextUpdates,
},
},
}, overrides?.sessionId)
}
}))
disposeHookFns.value.push(
@@ -45,49 +109,49 @@ export const useContextBridgeStore = defineStore('mods:api:context-bridge', () =
if (isProcessingRemoteStream)
return
broadcastStreamEvent({ type: 'before-compose', message, sessionId: chatStore.activeSessionId, context: toRaw(context) })
broadcastStreamEvent({ type: 'before-compose', message, sessionId: chatStore.activeSessionId, context: structuredClone(toRaw(context)) })
}),
chatStore.onAfterMessageComposed(async (message, context) => {
if (isProcessingRemoteStream)
return
broadcastStreamEvent({ type: 'after-compose', message, sessionId: chatStore.activeSessionId, context })
broadcastStreamEvent({ type: 'after-compose', message, sessionId: chatStore.activeSessionId, context: structuredClone(toRaw(context)) })
}),
chatStore.onBeforeSend(async (message, context) => {
if (isProcessingRemoteStream)
return
broadcastStreamEvent({ type: 'before-send', message, sessionId: chatStore.activeSessionId, context })
broadcastStreamEvent({ type: 'before-send', message, sessionId: chatStore.activeSessionId, context: structuredClone(toRaw(context)) })
}),
chatStore.onAfterSend(async (message, context) => {
if (isProcessingRemoteStream)
return
broadcastStreamEvent({ type: 'after-send', message, sessionId: chatStore.activeSessionId, context })
broadcastStreamEvent({ type: 'after-send', message, sessionId: chatStore.activeSessionId, context: structuredClone(toRaw(context)) })
}),
chatStore.onTokenLiteral(async (literal, context) => {
if (isProcessingRemoteStream)
return
broadcastStreamEvent({ type: 'token-literal', literal, sessionId: chatStore.activeSessionId, context })
broadcastStreamEvent({ type: 'token-literal', literal, sessionId: chatStore.activeSessionId, context: structuredClone(toRaw(context)) })
}),
chatStore.onTokenSpecial(async (special, context) => {
if (isProcessingRemoteStream)
return
broadcastStreamEvent({ type: 'token-special', special, sessionId: chatStore.activeSessionId, context })
broadcastStreamEvent({ type: 'token-special', special, sessionId: chatStore.activeSessionId, context: structuredClone(toRaw(context)) })
}),
chatStore.onStreamEnd(async (context) => {
if (isProcessingRemoteStream)
return
broadcastStreamEvent({ type: 'stream-end', sessionId: chatStore.activeSessionId, context })
broadcastStreamEvent({ type: 'stream-end', sessionId: chatStore.activeSessionId, context: structuredClone(toRaw(context)) })
}),
chatStore.onAssistantResponseEnd(async (message, context) => {
if (isProcessingRemoteStream)
return
broadcastStreamEvent({ type: 'assistant-end', message, sessionId: chatStore.activeSessionId, context })
broadcastStreamEvent({ type: 'assistant-end', message, sessionId: chatStore.activeSessionId, context: structuredClone(toRaw(context)) })
}),
chatStore.onAssistantMessage(async (message, _messageText, context) => {
@@ -95,12 +159,14 @@ export const useContextBridgeStore = defineStore('mods:api:context-bridge', () =
type: 'output:gen-ai:chat:message',
data: {
message,
...context.input?.metadata?.source,
'stage-web': isStageWeb(),
'stage-tamagotchi': isStageTamagotchi(),
'gen-ai:chat': {
input: context.input as UserMessage,
message: context.message as UserMessage,
composedMessage: context.composedMessage,
contexts: context.contexts,
input: context.input,
},
},
})
@@ -111,6 +177,7 @@ export const useContextBridgeStore = defineStore('mods:api:context-bridge', () =
type: 'output:gen-ai:chat:complete',
data: {
'message': chat.output,
...context.input?.metadata?.source,
'toolCalls': [],
'stage-web': isStageWeb(),
'stage-tamagotchi': isStageTamagotchi(),
@@ -122,9 +189,10 @@ export const useContextBridgeStore = defineStore('mods:api:context-bridge', () =
source: 'estimate-based',
},
'gen-ai:chat': {
input: context.input as UserMessage,
message: context.message as UserMessage,
composedMessage: context.composedMessage,
contexts: context.contexts,
input: context.input,
},
},
})
+8 -5
View File
@@ -1,4 +1,4 @@
import type { ContextUpdate, WebSocketEventSource } from '@proj-airi/server-sdk'
import type { ContextUpdate, MetadataEventSource, WebSocketEventInputs } from '@proj-airi/server-sdk'
import type { AssistantMessage, CommonContentPart, CompletionToolCall, Message, SystemMessage, ToolMessage, UserMessage } from '@xsai/shared-chat'
export interface ChatSlicesText {
@@ -38,17 +38,20 @@ export interface ErrorMessage {
content: string
}
export interface ContextMessage extends ContextUpdate {
source: WebSocketEventSource | string
export interface ContextMessage extends ContextUpdate<Record<string, unknown>, string | CommonContentPart[]> {
metadata?: {
source: MetadataEventSource
}
createdAt: number
}
export type ChatHistoryItem = (ChatMessage | ErrorMessage) & { context?: ContextMessage } & { createdAt?: number }
export interface ChatStreamEventContext {
input: ChatHistoryItem
message: ChatHistoryItem
contexts: Record<string, ContextMessage[]>
composedMessage: Message[]
composedMessage: Array<Message>
input?: WebSocketEventInputs
}
export type ChatStreamEvent
@@ -0,0 +1,23 @@
import type { MetadataEventSource } from '@proj-airi/server-sdk'
interface EventSourcePayload {
source?: string
metadata?: { source?: MetadataEventSource }
eventMetadata?: { source?: MetadataEventSource }
}
function formatMetadataSource(source?: MetadataEventSource) {
if (!source?.plugin)
return undefined
return source.instanceId ? `${source.plugin}:${source.instanceId}` : source.plugin
}
export function getEventSourceKey(event: EventSourcePayload, fallback = 'unknown') {
return (
formatMetadataSource(event.eventMetadata?.source)
?? formatMetadataSource(event.metadata?.source)
?? event.source
?? fallback
)
}
+1
View File
@@ -1 +1,2 @@
export { getEventSourceKey } from './event-source'
export { randomSaccadeInterval } from '@proj-airi/stage-ui-live2d/utils/eye-motions'
+151
View File
@@ -2947,6 +2947,9 @@ importers:
'@proj-airi/server-shared':
specifier: workspace:^
version: link:../../packages/server-shared
'@snazzah/davey':
specifier: ^0.1.9
version: 0.1.9
'@xsai-ext/providers':
specifier: 'catalog:'
version: 0.4.0
@@ -7912,6 +7915,93 @@ packages:
resolution: {integrity: sha512-LtoMMhxAlorcGhmFYI+LhPgbPZCkgP6ra1YL604EeF6U98pLlQ3iWIGMdWSC+vWmPBWBNgmDBAhnAobLROJmwg==}
engines: {node: '>=18'}
'@snazzah/davey-android-arm-eabi@0.1.9':
resolution: {integrity: sha512-Dq0WyeVGBw+uQbisV/6PeCQV2ndJozfhZqiNIfQxu6ehIdXB7iHILv+oY+AQN2n+qxiFmLh/MOX9RF+pIWdPbA==}
engines: {node: '>= 10'}
cpu: [arm]
os: [android]
'@snazzah/davey-android-arm64@0.1.9':
resolution: {integrity: sha512-OE16OZjv7F/JrD7Mzw5eL2gY2vXRPC8S7ZrmkcMyz/sHHJsGHlT+L7X5s56Bec1YDTVmzAsH4UBuvVBoXuIWEQ==}
engines: {node: '>= 10'}
cpu: [arm64]
os: [android]
'@snazzah/davey-darwin-arm64@0.1.9':
resolution: {integrity: sha512-z7oORvAPExikFkH6tvHhbUdZd77MYZp9VqbCpKEiI+sisWFVXgHde7F7iH3G4Bz6gUYJfgvKhWXiDRc+0SC4dg==}
engines: {node: '>= 10'}
cpu: [arm64]
os: [darwin]
'@snazzah/davey-darwin-x64@0.1.9':
resolution: {integrity: sha512-f1LzGyRGlM414KpXml3OgWVSd7CgylcdYaFj/zDBb8bvWjxyvsI9iMeuPfe/cduloxRj8dELde/yCDZtFR6PdQ==}
engines: {node: '>= 10'}
cpu: [x64]
os: [darwin]
'@snazzah/davey-freebsd-x64@0.1.9':
resolution: {integrity: sha512-k6p3JY2b8rD6j0V9Ql7kBUMR4eJdcpriNwiHltLzmtGuz/nK5RGQdkEP68gTLc+Uj3xs5Cy0jRKmv2xJQBR4sA==}
engines: {node: '>= 10'}
cpu: [x64]
os: [freebsd]
'@snazzah/davey-linux-arm-gnueabihf@0.1.9':
resolution: {integrity: sha512-xDaAFUC/1+n/YayNwKsqKOBMuW0KI6F0SjgWU+krYTQTVmAKNjOM80IjemrVoqTpBOxBsT80zEtct2wj11CE3Q==}
engines: {node: '>= 10'}
cpu: [arm]
os: [linux]
'@snazzah/davey-linux-arm64-gnu@0.1.9':
resolution: {integrity: sha512-t1VxFBzWExPNpsNY/9oStdAAuHqFvwZvIO2YPYyVNstxfi2KmAbHMweHUW7xb2ppXuhVQZ4VGmmeXiXcXqhPBw==}
engines: {node: '>= 10'}
cpu: [arm64]
os: [linux]
'@snazzah/davey-linux-arm64-musl@0.1.9':
resolution: {integrity: sha512-Xvlr+nBPzuFV4PXHufddlt08JsEyu0p8mX2DpqdPxdpysYIH4I8V86yJiS4tk04a6pLBDd8IxTbBwvXJKqd/LQ==}
engines: {node: '>= 10'}
cpu: [arm64]
os: [linux]
'@snazzah/davey-linux-x64-gnu@0.1.9':
resolution: {integrity: sha512-6Uunc/NxiEkg1reroAKZAGfOtjl1CGa7hfTTVClb2f+DiA8ZRQWBh+3lgkq/0IeL262B4F14X8QRv5Bsv128qw==}
engines: {node: '>= 10'}
cpu: [x64]
os: [linux]
'@snazzah/davey-linux-x64-musl@0.1.9':
resolution: {integrity: sha512-fFQ/n3aWt1lXhxSdy+Ge3gi5bR3VETMVsWhH0gwBALUKrbo3ZzgSktm4lNrXE9i0ncMz/CDpZ5i0wt/N3XphEQ==}
engines: {node: '>= 10'}
cpu: [x64]
os: [linux]
'@snazzah/davey-wasm32-wasi@0.1.9':
resolution: {integrity: sha512-xWvzej8YCVlUvzlpmqJMIf0XmLlHqulKZ2e7WNe2TxQmsK+o0zTZqiQYs2MwaEbrNXBhYlHDkdpuwoXkJdscNQ==}
engines: {node: '>=14.0.0'}
cpu: [wasm32]
'@snazzah/davey-win32-arm64-msvc@0.1.9':
resolution: {integrity: sha512-sTqry/DfltX2OdW1CTLKa3dFYN5FloAEb2yhGsY1i5+Bms6OhwByXfALvyMHYVo61Th2+sD+9BJpQffHFKDA3w==}
engines: {node: '>= 10'}
cpu: [arm64]
os: [win32]
'@snazzah/davey-win32-ia32-msvc@0.1.9':
resolution: {integrity: sha512-twD3LwlkGnSwphsCtpGb5ztpBIWEvGdc0iujoVkdzZ6nJiq5p8iaLjJMO4hBm9h3s28fc+1Qd7AMVnagiOasnA==}
engines: {node: '>= 10'}
cpu: [ia32]
os: [win32]
'@snazzah/davey-win32-x64-msvc@0.1.9':
resolution: {integrity: sha512-eMnXbv4GoTngWYY538i/qHz2BS+RgSXFsvKltPzKqnqzPzhQZIY7TemEJn3D5yWGfW4qHve9u23rz93FQqnQMA==}
engines: {node: '>= 10'}
cpu: [x64]
os: [win32]
'@snazzah/davey@0.1.9':
resolution: {integrity: sha512-vNZk5y+IsxjwzTAXikvzz5pqMLb35YytC64nVF2MAFVhjpXu9ITOKUriZ0JG/llwzCAi56jb5x0cXDRIyE2A2A==}
engines: {node: '>= 10'}
'@socket.io/component-emitter@3.1.2':
resolution: {integrity: sha512-9BCxFwvbGg/RsZK9tjXd8s4UcwR0MWeFQ1XEKIQVVvAGJyINdrqKMcTRyLoK8Rse1GjzLV9cwjWV1olXRWEXVA==}
@@ -21439,6 +21529,67 @@ snapshots:
'@sindresorhus/merge-streams@2.3.0': {}
'@snazzah/davey-android-arm-eabi@0.1.9':
optional: true
'@snazzah/davey-android-arm64@0.1.9':
optional: true
'@snazzah/davey-darwin-arm64@0.1.9':
optional: true
'@snazzah/davey-darwin-x64@0.1.9':
optional: true
'@snazzah/davey-freebsd-x64@0.1.9':
optional: true
'@snazzah/davey-linux-arm-gnueabihf@0.1.9':
optional: true
'@snazzah/davey-linux-arm64-gnu@0.1.9':
optional: true
'@snazzah/davey-linux-arm64-musl@0.1.9':
optional: true
'@snazzah/davey-linux-x64-gnu@0.1.9':
optional: true
'@snazzah/davey-linux-x64-musl@0.1.9':
optional: true
'@snazzah/davey-wasm32-wasi@0.1.9':
dependencies:
'@napi-rs/wasm-runtime': 1.1.0
optional: true
'@snazzah/davey-win32-arm64-msvc@0.1.9':
optional: true
'@snazzah/davey-win32-ia32-msvc@0.1.9':
optional: true
'@snazzah/davey-win32-x64-msvc@0.1.9':
optional: true
'@snazzah/davey@0.1.9':
optionalDependencies:
'@snazzah/davey-android-arm-eabi': 0.1.9
'@snazzah/davey-android-arm64': 0.1.9
'@snazzah/davey-darwin-arm64': 0.1.9
'@snazzah/davey-darwin-x64': 0.1.9
'@snazzah/davey-freebsd-x64': 0.1.9
'@snazzah/davey-linux-arm-gnueabihf': 0.1.9
'@snazzah/davey-linux-arm64-gnu': 0.1.9
'@snazzah/davey-linux-arm64-musl': 0.1.9
'@snazzah/davey-linux-x64-gnu': 0.1.9
'@snazzah/davey-linux-x64-musl': 0.1.9
'@snazzah/davey-wasm32-wasi': 0.1.9
'@snazzah/davey-win32-arm64-msvc': 0.1.9
'@snazzah/davey-win32-ia32-msvc': 0.1.9
'@snazzah/davey-win32-x64-msvc': 0.1.9
'@socket.io/component-emitter@3.1.2': {}
'@standard-schema/spec@1.1.0': {}
+1
View File
@@ -25,6 +25,7 @@
"@proj-airi/audio": "workspace:^",
"@proj-airi/server-sdk": "workspace:^",
"@proj-airi/server-shared": "workspace:^",
"@snazzah/davey": "^0.1.9",
"@xsai-ext/providers": "catalog:",
"@xsai/generate-speech": "catalog:",
"@xsai/generate-text": "catalog:",
+154 -54
View File
@@ -1,9 +1,11 @@
import type { Discord } from '@proj-airi/server-shared/types'
import type { Interaction } from 'discord.js'
import { env } from 'node:process'
import { useLogg } from '@guiiai/logg'
import { Client as AiriClient } from '@proj-airi/server-sdk'
import { ContextUpdateStrategy } from '@proj-airi/server-shared/types'
import { Client, Events, GatewayIntentBits } from 'discord.js'
import { handlePing, registerCommands, VoiceManager } from '../bots/discord/commands'
@@ -31,6 +33,25 @@ function isDiscordConfig(config: unknown): config is DiscordConfig {
&& (typeof c.enabled === 'boolean' || typeof c.enabled === 'undefined')
}
function normalizeDiscordMetadata(discord?: Discord): Discord | undefined {
if (!discord)
return undefined
if (!discord.guildMember)
return discord
const { guildMember } = discord
return {
...discord,
guildMember: {
id: guildMember.id ?? guildMember.displayName ?? guildMember.nickname ?? '',
nickname: guildMember.nickname ?? guildMember.displayName ?? '',
displayName: guildMember.displayName ?? guildMember.nickname ?? '',
},
}
}
export class DiscordAdapter {
private airiClient: AiriClient
private discordClient: Client
@@ -43,17 +64,23 @@ export class DiscordAdapter {
// Initialize Discord client
this.discordClient = new Client({
intents: [GatewayIntentBits.Guilds, GatewayIntentBits.GuildVoiceStates],
intents: [
GatewayIntentBits.Guilds,
GatewayIntentBits.GuildVoiceStates,
GatewayIntentBits.GuildMessages,
GatewayIntentBits.MessageContent,
],
})
// Initialize AIRI client
this.airiClient = new AiriClient({
name: 'discord-bot',
name: 'discord',
possibleEvents: [
'input:text',
'input:text:voice',
'input:voice',
'ui:configure',
'module:configure',
'output:gen-ai:chat:message',
],
token: config.airiToken,
url: config.airiUrl,
@@ -66,60 +93,58 @@ export class DiscordAdapter {
private setupEventHandlers(): void {
// Handle configuration from UI
this.airiClient.onEvent('ui:configure', async (event) => {
if (event.data.moduleName === 'discord') {
if (this.isReconnecting) {
log.warn('A reconnect is already in progress, skipping this configuration event.')
return
}
this.isReconnecting = true
try {
log.log('Received Discord configuration:', event.data.config)
this.airiClient.onEvent('module:configure', async (event) => {
if (this.isReconnecting) {
log.warn('A reconnect is already in progress, skipping this configuration event.')
return
}
this.isReconnecting = true
try {
log.log('Received Discord configuration:', event.data.config)
if (isDiscordConfig(event.data.config)) {
const config = event.data.config as DiscordConfig
const { token, enabled } = config
if (isDiscordConfig(event.data.config)) {
const config = event.data.config as DiscordConfig
const { token, enabled } = config
if (enabled === false) {
if (this.discordClient.isReady) {
log.log('Disabling Discord bot as per configuration...')
await this.discordClient.destroy()
}
return
}
// If enabled, but no token is provided, stop the bot if it's running.
if (!token) {
log.warn('Discord bot enabled, but no token provided. Stopping bot.')
if (this.discordClient.isReady) {
await this.discordClient.destroy()
}
return
}
// Connect or reconnect if token changed or client is not ready.
if (this.discordToken !== token || !this.discordClient.isReady) {
this.discordToken = token
if (this.discordClient.isReady) {
log.log('Reconnecting Discord client with new token...')
await this.discordClient.destroy()
}
log.log('Connecting Discord client...')
await this.discordClient.login(this.discordToken)
log.log('Discord client connected.')
if (enabled === false) {
if (this.discordClient.isReady) {
log.log('Disabling Discord bot as per configuration...')
await this.discordClient.destroy()
}
return
}
else {
log.warn('Invalid Discord configuration received, skipping...')
// If enabled, but no token is provided, stop the bot if it's running.
if (!token) {
log.warn('Discord bot enabled, but no token provided. Stopping bot.')
if (this.discordClient.isReady) {
await this.discordClient.destroy()
}
return
}
// Connect or reconnect if token changed or client is not ready.
if (this.discordToken !== token || !this.discordClient.isReady) {
this.discordToken = token
if (this.discordClient.isReady) {
log.log('Reconnecting Discord client with new token...')
await this.discordClient.destroy()
}
log.log('Connecting Discord client...')
await this.discordClient.login(this.discordToken)
log.log('Discord client connected.')
}
}
catch (error) {
log.withError(error as Error).error('Failed to apply Discord configuration.')
}
finally {
this.isReconnecting = false
else {
log.warn('Invalid Discord configuration received, skipping...')
}
}
catch (error) {
log.withError(error as Error).error('Failed to apply Discord configuration.')
}
finally {
this.isReconnecting = false
}
})
// Handle input from AIRI system
@@ -129,16 +154,94 @@ export class DiscordAdapter {
// For now, we'll just log the input
})
// Handle output from AIRI system (IA response)
this.airiClient.onEvent('output:gen-ai:chat:message', async (event) => {
try {
const { message, discord } = event.data as {
message: { content: string }
discord?: { channelId: string }
}
if (discord?.channelId) {
const channel = await this.discordClient.channels.fetch(discord.channelId)
if (channel?.isTextBased() && 'send' in channel && typeof channel.send === 'function') {
await channel.send(message.content)
}
}
}
catch (error) {
log.withError(error as Error).error('Failed to send response to Discord')
}
})
// Set up Discord event handlers
this.discordClient.once(Events.ClientReady, (readyClient) => {
this.discordClient.once(Events.ClientReady, async (readyClient) => {
log.log(`Discord bot ready! User: ${readyClient.user.tag}`)
// Register commands dynamically using the authenticated client's ID and token
await registerCommands(this.discordToken, readyClient.user.id)
})
// Handle text messages from Discord
this.discordClient.on(Events.MessageCreate, async (message) => {
if (message.author.bot)
return
// Respond if the bot is mentioned
if (this.discordClient.user && message.mentions.has(this.discordClient.user)) {
const rawContent = message.content
const content = rawContent.replace(/<@!?\d+>/g, '').trim()
if (!content)
return
log.log(`Received text mention from ${message.author.tag} in ${message.channelId}`)
const discordContext: Discord = {
channelId: message.channelId,
guildId: message.guildId ?? undefined,
guildMember: {
id: message.author.id,
displayName: message.member?.displayName ?? message.author.username,
nickname: message.member?.nickname ?? message.author.username,
},
}
const normalizedDiscord = normalizeDiscordMetadata(discordContext)
const displayName = normalizedDiscord?.guildMember?.displayName
const discordNotice = normalizedDiscord
? `The input is coming from Discord channel ${normalizedDiscord.channelId} (Guild: ${normalizedDiscord.guildId ?? 'unknown'}).`
: undefined
this.airiClient.send({
type: 'input:text',
data: {
text: content,
textRaw: rawContent,
overrides: displayName
? {
messagePrefix: `(From Discord user ${displayName}): `,
sessionId: 'discord',
}
: undefined,
contextUpdates: discordNotice
? [{
strategy: ContextUpdateStrategy.AppendSelf,
text: discordNotice,
content: discordNotice,
metadata: {
discord: normalizedDiscord,
},
}]
: undefined,
discord: normalizedDiscord,
},
})
}
})
this.discordClient.on(Events.InteractionCreate, async (interaction: Interaction) => {
if (!interaction.isChatInputCommand())
return
log.log('Interaction received:', interaction)
log.log(`Interaction received: /${interaction.commandName} from ${interaction.user.tag}`)
switch (interaction.commandName) {
case 'ping':
@@ -155,9 +258,6 @@ export class DiscordAdapter {
log.log('Starting Discord adapter...')
try {
// Register commands
await registerCommands()
// Log in to Discord if token is available
if (this.discordToken) {
await this.discordClient.login(this.discordToken)
@@ -1,16 +1,14 @@
import { env } from 'node:process'
import { REST, Routes, SlashCommandBuilder } from 'discord.js'
export * from './ping'
export * from './summon'
export async function registerCommands() {
export async function registerCommands(token: string, clientId: string) {
const rest = new REST()
rest.setToken(env.DISCORD_TOKEN)
rest.setToken(token)
rest.put(
Routes.applicationCommands(env.DISCORD_BOT_CLIENT_ID),
Routes.applicationCommands(clientId),
{ body: [
new SlashCommandBuilder().setName('ping').setDescription('Replies with Pong!'),
new SlashCommandBuilder().setName('summon').setDescription('Summons the bot to your voice channel'),