From 5d9040e4302b2c0d7a53357c8688e51589a172bf Mon Sep 17 00:00:00 2001 From: skyline624 <35170939+skyline624@users.noreply.github.com> Date: Sat, 10 Jan 2026 20:35:32 +0100 Subject: [PATCH] fix(stage-ui,server-*): implement handling for `input:text` from anywhere, support for Discord (#928) --------- Co-authored-by: Neko --- .gitignore | 3 + .../services/airi/channel-server/index.ts | 5 +- apps/stage-web/package.json | 2 +- .../src/types/websocket/events.ts | 74 ++++++- .../src/pages/devtools/context-flow/index.vue | 5 +- .../event-handler-spark-notify/index.ts | 4 +- packages/stage-ui/src/stores/chat.ts | 83 ++++--- .../src/stores/mods/api/context-bridge.ts | 94 ++++++-- packages/stage-ui/src/types/chat.ts | 13 +- packages/stage-ui/src/utils/event-source.ts | 23 ++ packages/stage-ui/src/utils/index.ts | 1 + pnpm-lock.yaml | 151 +++++++++++++ services/discord-bot/package.json | 1 + .../discord-bot/src/adapters/airi-adapter.ts | 208 +++++++++++++----- .../src/bots/discord/commands/index.ts | 8 +- 15 files changed, 553 insertions(+), 122 deletions(-) create mode 100644 packages/stage-ui/src/utils/event-source.ts diff --git a/.gitignore b/.gitignore index 86b3591d5..026d97c30 100644 --- a/.gitignore +++ b/.gitignore @@ -119,3 +119,6 @@ plugins/local plugins/development plugins-local plugins-development + +#ia +GEMINI.md \ No newline at end of file diff --git a/apps/stage-tamagotchi/src/main/services/airi/channel-server/index.ts b/apps/stage-tamagotchi/src/main/services/airi/channel-server/index.ts index 21b1d38bb..bc1515e40 100644 --- a/apps/stage-tamagotchi/src/main/services/airi/channel-server/index.ts +++ b/apps/stage-tamagotchi/src/main/services/airi/channel-server/index.ts @@ -17,9 +17,8 @@ export async function setupServerChannel() { const app = serverRuntime.setupApp() const serverInstance = serve(app, { - // TODO: fix types - // @ts-expect-error - the .crossws property wasn't extended in types - plugins: [ws({ resolve: async req => (await app.fetch(req)).crossws })], + // TODO: add proper crossws typing upstream + plugins: [ws({ resolve: async req => (await app.fetch(req) as any).crossws })], port: env.PORT ? Number(env.PORT) : 6121, hostname: env.SERVER_RUNTIME_HOSTNAME || 'localhost', reusePort: true, diff --git a/apps/stage-web/package.json b/apps/stage-web/package.json index cc86521fe..860c823d6 100644 --- a/apps/stage-web/package.json +++ b/apps/stage-web/package.json @@ -11,7 +11,7 @@ "license": "MIT", "scripts": { "build": "vite build", - "dev": "vite", + "dev": "vite --host", "lint": "eslint .", "preview": "vite preview", "typecheck": "vue-tsc --noEmit" diff --git a/packages/server-shared/src/types/websocket/events.ts b/packages/server-shared/src/types/websocket/events.ts index a4bf46480..0125e29b9 100644 --- a/packages/server-shared/src/types/websocket/events.ts +++ b/packages/server-shared/src/types/websocket/events.ts @@ -1,4 +1,4 @@ -import type { AssistantMessage, Message, ToolMessage, UserMessage } from '@xsai/shared-chat' +import type { AssistantMessage, CommonContentPart, Message, ToolMessage, UserMessage } from '@xsai/shared-chat' export interface DiscordGuildMember { nickname: string @@ -13,9 +13,25 @@ export interface Discord { } export interface MetadataEventSource { + /** + * Stable module/plugin identifier (shared across instances). + * Example: "telegram-bot", "stage-tamagotchi". + */ plugin: string + /** + * Unique instance id for this module run (per process/deployment). + * Example: "telegram-01", "stage-ui-2f7c9". + */ instanceId: string + /** + * Optional semantic version for the module/plugin. + * Example: "0.8.1-beta.7". + */ version?: string + /** + * K8s-style labels for routing and policy selectors. + * Example: { env: "prod", app: "telegram", devtools: "true" }. + */ labels?: Record } @@ -59,9 +75,10 @@ interface InputSource { interface OutputSource { 'gen-ai:chat': { - input: UserMessage - contexts: Record + message: UserMessage + contexts: Record, string | CommonContentPart[]>[]> composedMessage: Array + input?: WebSocketEventInputs } } @@ -104,6 +121,45 @@ export interface ContextUpdate< metadata?: Metadata } +export interface InputMessageOverrides { + sessionId?: string + messagePrefix?: string +} + +export type InputContextUpdate + = Omit, string | CommonContentPart[]>, 'id' | 'contextId'> + & Partial, string | CommonContentPart[]>, 'id' | 'contextId'>> + +export interface WebSocketEventInputTextBase { + text: string + textRaw?: string + overrides?: InputMessageOverrides + contextUpdates?: InputContextUpdate[] +} + +export type WebSocketEventInputText = WebSocketEventInputTextBase & Partial> + +export interface WebSocketEventInputTextVoiceBase { + transcription: string + textRaw?: string + overrides?: InputMessageOverrides + contextUpdates?: InputContextUpdate[] +} + +export type WebSocketEventInputTextVoice = WebSocketEventInputTextVoiceBase & Partial> + +export interface WebSocketEventInputVoiceBase { + audio: ArrayBuffer + overrides?: InputMessageOverrides + contextUpdates?: InputContextUpdate[] +} + +export type WebSocketEventInputVoice = WebSocketEventInputVoiceBase & Partial> + +export type WebSocketEventDataInputs = WebSocketEventInputText | WebSocketEventInputTextVoice | WebSocketEventInputVoice + +export type WebSocketEventInputs = WebSocketBaseEvent<'input:text' | 'input:text:voice' | 'input:voice', WebSocketEventInputText | WebSocketEventInputTextVoice | WebSocketEventInputVoice> + export interface WebSocketBaseEvent { type: T data: D @@ -155,15 +211,9 @@ export interface WebSocketEvents { config: C | Record } - 'input:text': { - text: string - } & Partial> - 'input:text:voice': { - transcription: string - } & Partial> - 'input:voice': { - audio: ArrayBuffer - } & Partial> + 'input:text': WebSocketEventInputText + 'input:text:voice': WebSocketEventInputTextVoice + 'input:voice': WebSocketEventInputVoice 'output:gen-ai:chat:tool-call': { toolCalls: ToolMessage[] diff --git a/packages/stage-pages/src/pages/devtools/context-flow/index.vue b/packages/stage-pages/src/pages/devtools/context-flow/index.vue index e86bd1a2e..bc060393e 100644 --- a/packages/stage-pages/src/pages/devtools/context-flow/index.vue +++ b/packages/stage-pages/src/pages/devtools/context-flow/index.vue @@ -9,6 +9,7 @@ import { ContextUpdateStrategy } from '@proj-airi/server-sdk' import { useCharacterOrchestratorStore, useCharacterStore } from '@proj-airi/stage-ui/stores/character' import { CHAT_STREAM_CHANNEL_NAME, CONTEXT_CHANNEL_NAME, useChatStore } from '@proj-airi/stage-ui/stores/chat' import { useModsServerChannelStore } from '@proj-airi/stage-ui/stores/mods/api/channel-server' +import { getEventSourceKey } from '@proj-airi/stage-ui/utils' import { Callout } from '@proj-airi/ui' import { useBroadcastChannel } from '@vueuse/core' import { nanoid } from 'nanoid' @@ -331,7 +332,7 @@ onMounted(() => { channel: 'server', type: event.type, summary: [ - `source=${event.source}`, + `source=${getEventSourceKey(event)}`, `strategy=${event.data.strategy}`, summarizeContextUpdate(event.data), ].filter(Boolean).join(' '), @@ -484,7 +485,7 @@ watch(incomingContext, (event) => { channel: 'broadcast', type: 'context:broadcast', summary: [ - `source=${event.source}`, + `source=${getEventSourceKey(event)}`, `strategy=${event.strategy}`, summarizeContextUpdate(event), ].filter(Boolean).join(' '), diff --git a/packages/stage-ui/src/stores/character/orchestrator/agents/event-handler-spark-notify/index.ts b/packages/stage-ui/src/stores/character/orchestrator/agents/event-handler-spark-notify/index.ts index 44e944bd8..93e181d8d 100644 --- a/packages/stage-ui/src/stores/character/orchestrator/agents/event-handler-spark-notify/index.ts +++ b/packages/stage-ui/src/stores/character/orchestrator/agents/event-handler-spark-notify/index.ts @@ -10,6 +10,8 @@ import { nanoid } from 'nanoid' import { validate } from 'xsschema' import { z } from 'zod' +import { getEventSourceKey } from '../../../../../utils' + export interface SparkNotifyCommandDraft { destinations: string[] interrupt?: 'force' | 'soft' | boolean @@ -172,7 +174,7 @@ export function setupAgentSparkNotifyHandler(deps: SparkNotifyAgentDeps) { role: 'system', content: [ deps.getSystemPrompt(), - getSparkNotifyHandlingAgentInstruction(event.source), + getSparkNotifyHandlingAgentInstruction(getEventSourceKey(event)), ].filter(Boolean).join('\n\n'), } diff --git a/packages/stage-ui/src/stores/chat.ts b/packages/stage-ui/src/stores/chat.ts index 28b090bb1..b354bc4bc 100644 --- a/packages/stage-ui/src/stores/chat.ts +++ b/packages/stage-ui/src/stores/chat.ts @@ -1,3 +1,4 @@ +import type { WebSocketEventInputs } from '@proj-airi/server-sdk' import type { ChatProvider } from '@xsai-ext/providers/utils' import type { CommonContentPart, Message, SystemMessage, ToolMessage } from '@xsai/shared-chat' @@ -14,6 +15,7 @@ import { useAnalytics } from '../composables' import { useLlmmarkerParser } from '../composables/llm-marker-parser' import { categorizeResponse, createStreamingCategorizer } from '../composables/response-categoriser' import { useLLM } from '../stores/llm' +import { getEventSourceKey } from '../utils/event-source' import { useCharacterStore } from './character' import { useConsciousnessStore } from './modules/consciousness' @@ -44,6 +46,7 @@ export const useChatStore = defineStore('chat', () => { providerConfig?: Record attachments?: { type: 'image', data: string, mimeType: string }[] tools?: StreamOptions['tools'] + input?: WebSocketEventInputs } interface QueuedSend { @@ -331,15 +334,16 @@ export const useChatStore = defineStore('chat', () => { }, { immediate: true }) function ingestContextMessage(envelope: ContextMessage) { - if (!activeContexts.value[envelope.source]) { - activeContexts.value[envelope.source] = [] + const sourceKey = getEventSourceKey(envelope) + if (!activeContexts.value[sourceKey]) { + activeContexts.value[sourceKey] = [] } if (envelope.strategy === ContextUpdateStrategy.ReplaceSelf) { - activeContexts.value[envelope.source] = [envelope] + activeContexts.value[sourceKey] = [envelope] } else if (envelope.strategy === ContextUpdateStrategy.AppendSelf) { - activeContexts.value[envelope.source].push(envelope) + activeContexts.value[sourceKey].push(envelope) } } @@ -357,9 +361,10 @@ export const useChatStore = defineStore('chat', () => { const sendingCreatedAt = Date.now() const streamingMessageContext: ChatStreamEventContext = { - input: { role: 'user', content: sendingMessage, createdAt: sendingCreatedAt }, + message: { role: 'user', content: sendingMessage, createdAt: sendingCreatedAt }, contexts: toRaw(activeContexts.value), composedMessage: [], + input: options.input, } const isStaleGeneration = () => getSessionGeneration(sessionId) !== generation @@ -369,7 +374,21 @@ export const useChatStore = defineStore('chat', () => { sending.value = true - streamingMessage.value = { role: 'assistant', content: '', slices: [], tool_results: [], createdAt: Date.now() } + const isForegroundSession = () => sessionId === activeSessionId.value + + // Use a local object for building the message to avoid polluting the UI for background sessions + const buildingMessage: StreamingAssistantMessage = { role: 'assistant', content: '', slices: [], tool_results: [], createdAt: Date.now() } + + // NOTICE: Clone into reactive state only for the foreground session + // to avoid background streams mutating UI. + const updateUI = () => { + if (isForegroundSession()) { + streamingMessage.value = JSON.parse(JSON.stringify(buildingMessage)) + } + } + + // Initialize UI if foreground + updateUI() trackFirstMessage() try { @@ -391,7 +410,14 @@ export const useChatStore = defineStore('chat', () => { } const finalContent = contentParts.length > 1 ? contentParts : sendingMessage - streamingMessageContext.input.content = finalContent + if (!streamingMessageContext.input) { + streamingMessageContext.input = { + type: 'input:text', + data: { + text: sendingMessage, + }, + } + } if (shouldAbort()) return @@ -408,7 +434,6 @@ export const useChatStore = defineStore('chat', () => { if (shouldAbort()) return - console.log('literal', literal) // Feed to categorizer first categorizer.consume(literal) @@ -420,24 +445,24 @@ export const useChatStore = defineStore('chat', () => { // Only process non-empty speech content (filter empty/whitespace-only chunks) // Preserve spacing in chunks with content for proper word boundaries if (speechOnly.trim()) { - streamingMessage.value.content += speechOnly + buildingMessage.content += speechOnly - console.log('speechOnly', speechOnly) // Emit TTS only for speech parts, not reasoning (clean data, no empty chunks) await emitTokenLiteralHooks(speechOnly, streamingMessageContext) // Add speech content to slices for rendering // merge text slices for markdown - const lastSlice = streamingMessage.value.slices.at(-1) + const lastSlice = buildingMessage.slices.at(-1) if (lastSlice?.type === 'text') { lastSlice.text += speechOnly - return } - - streamingMessage.value.slices.push({ - type: 'text', - text: speechOnly, - }) + else { + buildingMessage.slices.push({ + type: 'text', + text: speechOnly, + }) + } + updateUI() } }, onSpecial: async (special) => { @@ -454,10 +479,11 @@ export const useChatStore = defineStore('chat', () => { const finalCategorization = categorizeResponse(fullText, activeProvider.value) // Always store categorization (even if empty) for consistency and memory features - streamingMessage.value.categorization = { + buildingMessage.categorization = { speech: finalCategorization.speech, reasoning: finalCategorization.reasoning, } + updateUI() }, minLiteralEmitLength: 24, // Avoid emitting literals too fast. This is a magic number and can be changed later. }) @@ -468,12 +494,14 @@ export const useChatStore = defineStore('chat', () => { if (shouldAbort()) return if (ctx.data.type === 'tool-call') { - streamingMessage.value.slices.push(ctx.data) + buildingMessage.slices.push(ctx.data) + updateUI() return } if (ctx.data.type === 'tool-call-result') { - streamingMessage.value.tool_results.push(ctx.data) + buildingMessage.tool_results.push(ctx.data) + updateUI() } }, ], @@ -565,8 +593,8 @@ export const useChatStore = defineStore('chat', () => { await parser.end() // Add the completed message to the history only if it has content - if (!isStaleGeneration() && streamingMessage.value.slices.length > 0) { - sessionMessagesForSend.push(toRaw(streamingMessage.value)) + if (!isStaleGeneration() && buildingMessage.slices.length > 0) { + sessionMessagesForSend.push(toRaw(buildingMessage)) } // Call the end-of-stream hooks @@ -576,15 +604,17 @@ export const useChatStore = defineStore('chat', () => { await emitAssistantResponseEndHooks(fullText, streamingMessageContext) await emitAfterSendHooks(sendingMessage, streamingMessageContext) - await emitAssistantMessageHooks({ ...streamingMessage.value }, fullText, streamingMessageContext) + await emitAssistantMessageHooks({ ...buildingMessage }, fullText, streamingMessageContext) await emitChatTurnCompleteHooks({ - output: { ...streamingMessage.value }, + output: { ...buildingMessage }, outputText: fullText, toolCalls: sessionMessagesForSend.filter(msg => msg.role === 'tool') as ToolMessage[], }, streamingMessageContext) // Reset the streaming message for the next turn - streamingMessage.value = { role: 'assistant', content: '', slices: [], tool_results: [] } + if (isForegroundSession()) { + streamingMessage.value = { role: 'assistant', content: '', slices: [], tool_results: [] } + } } catch (error) { console.error('Error sending message:', error) @@ -628,8 +658,9 @@ export const useChatStore = defineStore('chat', () => { async function send( sendingMessage: string, options: SendOptions, + targetSessionId?: string, ) { - const sessionId = activeSessionId.value + const sessionId = targetSessionId || activeSessionId.value const generation = getSessionGeneration(sessionId) return new Promise((resolve, reject) => { diff --git a/packages/stage-ui/src/stores/mods/api/context-bridge.ts b/packages/stage-ui/src/stores/mods/api/context-bridge.ts index e498f8f0c..ca143111d 100644 --- a/packages/stage-ui/src/stores/mods/api/context-bridge.ts +++ b/packages/stage-ui/src/stores/mods/api/context-bridge.ts @@ -1,3 +1,4 @@ +import type { ChatProvider } from '@xsai-ext/providers/utils' import type { UserMessage } from '@xsai/shared-chat' import type { ChatStreamEvent, ContextMessage } from '../../../types/chat' @@ -5,10 +6,13 @@ import type { ChatStreamEvent, ContextMessage } from '../../../types/chat' import { isStageTamagotchi, isStageWeb } from '@proj-airi/stage-shared' import { useBroadcastChannel } from '@vueuse/core' import { Mutex } from 'es-toolkit' -import { defineStore } from 'pinia' +import { nanoid } from 'nanoid' +import { defineStore, storeToRefs } from 'pinia' import { ref, toRaw, watch } from 'vue' import { CHAT_STREAM_CHANNEL_NAME, CONTEXT_CHANNEL_NAME, useChatStore } from '../../chat' +import { useConsciousnessStore } from '../../modules/consciousness' +import { useProvidersStore } from '../../providers' import { useModsServerChannelStore } from './channel-server' export const useContextBridgeStore = defineStore('mods:api:context-bridge', () => { @@ -16,6 +20,9 @@ export const useContextBridgeStore = defineStore('mods:api:context-bridge', () = const chatStore = useChatStore() const serverChannelStore = useModsServerChannelStore() + const consciousnessStore = useConsciousnessStore() + const providersStore = useProvidersStore() + const { activeProvider, activeModel } = storeToRefs(consciousnessStore) const { post: broadcastContext, data: incomingContext } = useBroadcastChannel({ name: CONTEXT_CHANNEL_NAME }) const { post: broadcastStreamEvent, data: incomingStreamEvent } = useBroadcastChannel({ name: CHAT_STREAM_CHANNEL_NAME }) @@ -36,8 +43,65 @@ export const useContextBridgeStore = defineStore('mods:api:context-bridge', () = disposeHookFns.value.push(stop) disposeHookFns.value.push(serverChannelStore.onContextUpdate((event) => { - chatStore.ingestContextMessage({ source: event.source, createdAt: Date.now(), ...event.data }) - broadcastContext(toRaw(event.data) as ContextMessage) + const contextMessage: ContextMessage = { + ...event.data, + metadata: event.metadata, + createdAt: Date.now(), + } + chatStore.ingestContextMessage(contextMessage) + broadcastContext(toRaw(contextMessage)) + })) + + disposeHookFns.value.push(serverChannelStore.onEvent('input:text', async (event) => { + const { + text, + textRaw, + overrides, + contextUpdates, + } = event.data + + const normalizedContextUpdates = contextUpdates?.map((update) => { + const id = update.id ?? nanoid() + const contextId = update.contextId ?? id + return { + ...update, + id, + contextId, + } + }) + + if (normalizedContextUpdates?.length) { + const createdAt = Date.now() + for (const update of normalizedContextUpdates) { + chatStore.ingestContextMessage({ + ...update, + metadata: event.metadata, + createdAt, + }) + } + } + + if (activeProvider.value && activeModel.value) { + const chatProvider = await providersStore.getProviderInstance(activeProvider.value) + + let messageText = text + if (overrides?.messagePrefix) + messageText = `${overrides.messagePrefix}${text}` + + await chatStore.send(messageText, { + model: activeModel.value, + chatProvider, + input: { + type: 'input:text', + data: { + text, + textRaw, + overrides, + contextUpdates: normalizedContextUpdates, + }, + }, + }, overrides?.sessionId) + } })) disposeHookFns.value.push( @@ -45,49 +109,49 @@ export const useContextBridgeStore = defineStore('mods:api:context-bridge', () = if (isProcessingRemoteStream) return - broadcastStreamEvent({ type: 'before-compose', message, sessionId: chatStore.activeSessionId, context: toRaw(context) }) + broadcastStreamEvent({ type: 'before-compose', message, sessionId: chatStore.activeSessionId, context: structuredClone(toRaw(context)) }) }), chatStore.onAfterMessageComposed(async (message, context) => { if (isProcessingRemoteStream) return - broadcastStreamEvent({ type: 'after-compose', message, sessionId: chatStore.activeSessionId, context }) + broadcastStreamEvent({ type: 'after-compose', message, sessionId: chatStore.activeSessionId, context: structuredClone(toRaw(context)) }) }), chatStore.onBeforeSend(async (message, context) => { if (isProcessingRemoteStream) return - broadcastStreamEvent({ type: 'before-send', message, sessionId: chatStore.activeSessionId, context }) + broadcastStreamEvent({ type: 'before-send', message, sessionId: chatStore.activeSessionId, context: structuredClone(toRaw(context)) }) }), chatStore.onAfterSend(async (message, context) => { if (isProcessingRemoteStream) return - broadcastStreamEvent({ type: 'after-send', message, sessionId: chatStore.activeSessionId, context }) + broadcastStreamEvent({ type: 'after-send', message, sessionId: chatStore.activeSessionId, context: structuredClone(toRaw(context)) }) }), chatStore.onTokenLiteral(async (literal, context) => { if (isProcessingRemoteStream) return - broadcastStreamEvent({ type: 'token-literal', literal, sessionId: chatStore.activeSessionId, context }) + broadcastStreamEvent({ type: 'token-literal', literal, sessionId: chatStore.activeSessionId, context: structuredClone(toRaw(context)) }) }), chatStore.onTokenSpecial(async (special, context) => { if (isProcessingRemoteStream) return - broadcastStreamEvent({ type: 'token-special', special, sessionId: chatStore.activeSessionId, context }) + broadcastStreamEvent({ type: 'token-special', special, sessionId: chatStore.activeSessionId, context: structuredClone(toRaw(context)) }) }), chatStore.onStreamEnd(async (context) => { if (isProcessingRemoteStream) return - broadcastStreamEvent({ type: 'stream-end', sessionId: chatStore.activeSessionId, context }) + broadcastStreamEvent({ type: 'stream-end', sessionId: chatStore.activeSessionId, context: structuredClone(toRaw(context)) }) }), chatStore.onAssistantResponseEnd(async (message, context) => { if (isProcessingRemoteStream) return - broadcastStreamEvent({ type: 'assistant-end', message, sessionId: chatStore.activeSessionId, context }) + broadcastStreamEvent({ type: 'assistant-end', message, sessionId: chatStore.activeSessionId, context: structuredClone(toRaw(context)) }) }), chatStore.onAssistantMessage(async (message, _messageText, context) => { @@ -95,12 +159,14 @@ export const useContextBridgeStore = defineStore('mods:api:context-bridge', () = type: 'output:gen-ai:chat:message', data: { message, + ...context.input?.metadata?.source, 'stage-web': isStageWeb(), 'stage-tamagotchi': isStageTamagotchi(), 'gen-ai:chat': { - input: context.input as UserMessage, + message: context.message as UserMessage, composedMessage: context.composedMessage, contexts: context.contexts, + input: context.input, }, }, }) @@ -111,6 +177,7 @@ export const useContextBridgeStore = defineStore('mods:api:context-bridge', () = type: 'output:gen-ai:chat:complete', data: { 'message': chat.output, + ...context.input?.metadata?.source, 'toolCalls': [], 'stage-web': isStageWeb(), 'stage-tamagotchi': isStageTamagotchi(), @@ -122,9 +189,10 @@ export const useContextBridgeStore = defineStore('mods:api:context-bridge', () = source: 'estimate-based', }, 'gen-ai:chat': { - input: context.input as UserMessage, + message: context.message as UserMessage, composedMessage: context.composedMessage, contexts: context.contexts, + input: context.input, }, }, }) diff --git a/packages/stage-ui/src/types/chat.ts b/packages/stage-ui/src/types/chat.ts index 348352df4..b0669c033 100644 --- a/packages/stage-ui/src/types/chat.ts +++ b/packages/stage-ui/src/types/chat.ts @@ -1,4 +1,4 @@ -import type { ContextUpdate, WebSocketEventSource } from '@proj-airi/server-sdk' +import type { ContextUpdate, MetadataEventSource, WebSocketEventInputs } from '@proj-airi/server-sdk' import type { AssistantMessage, CommonContentPart, CompletionToolCall, Message, SystemMessage, ToolMessage, UserMessage } from '@xsai/shared-chat' export interface ChatSlicesText { @@ -38,17 +38,20 @@ export interface ErrorMessage { content: string } -export interface ContextMessage extends ContextUpdate { - source: WebSocketEventSource | string +export interface ContextMessage extends ContextUpdate, string | CommonContentPart[]> { + metadata?: { + source: MetadataEventSource + } createdAt: number } export type ChatHistoryItem = (ChatMessage | ErrorMessage) & { context?: ContextMessage } & { createdAt?: number } export interface ChatStreamEventContext { - input: ChatHistoryItem + message: ChatHistoryItem contexts: Record - composedMessage: Message[] + composedMessage: Array + input?: WebSocketEventInputs } export type ChatStreamEvent diff --git a/packages/stage-ui/src/utils/event-source.ts b/packages/stage-ui/src/utils/event-source.ts new file mode 100644 index 000000000..cc59a2eea --- /dev/null +++ b/packages/stage-ui/src/utils/event-source.ts @@ -0,0 +1,23 @@ +import type { MetadataEventSource } from '@proj-airi/server-sdk' + +interface EventSourcePayload { + source?: string + metadata?: { source?: MetadataEventSource } + eventMetadata?: { source?: MetadataEventSource } +} + +function formatMetadataSource(source?: MetadataEventSource) { + if (!source?.plugin) + return undefined + + return source.instanceId ? `${source.plugin}:${source.instanceId}` : source.plugin +} + +export function getEventSourceKey(event: EventSourcePayload, fallback = 'unknown') { + return ( + formatMetadataSource(event.eventMetadata?.source) + ?? formatMetadataSource(event.metadata?.source) + ?? event.source + ?? fallback + ) +} diff --git a/packages/stage-ui/src/utils/index.ts b/packages/stage-ui/src/utils/index.ts index cdbf8cb0b..419e761cb 100644 --- a/packages/stage-ui/src/utils/index.ts +++ b/packages/stage-ui/src/utils/index.ts @@ -1 +1,2 @@ +export { getEventSourceKey } from './event-source' export { randomSaccadeInterval } from '@proj-airi/stage-ui-live2d/utils/eye-motions' diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 248693b31..6091c8422 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -2947,6 +2947,9 @@ importers: '@proj-airi/server-shared': specifier: workspace:^ version: link:../../packages/server-shared + '@snazzah/davey': + specifier: ^0.1.9 + version: 0.1.9 '@xsai-ext/providers': specifier: 'catalog:' version: 0.4.0 @@ -7912,6 +7915,93 @@ packages: resolution: {integrity: sha512-LtoMMhxAlorcGhmFYI+LhPgbPZCkgP6ra1YL604EeF6U98pLlQ3iWIGMdWSC+vWmPBWBNgmDBAhnAobLROJmwg==} engines: {node: '>=18'} + '@snazzah/davey-android-arm-eabi@0.1.9': + resolution: {integrity: sha512-Dq0WyeVGBw+uQbisV/6PeCQV2ndJozfhZqiNIfQxu6ehIdXB7iHILv+oY+AQN2n+qxiFmLh/MOX9RF+pIWdPbA==} + engines: {node: '>= 10'} + cpu: [arm] + os: [android] + + '@snazzah/davey-android-arm64@0.1.9': + resolution: {integrity: sha512-OE16OZjv7F/JrD7Mzw5eL2gY2vXRPC8S7ZrmkcMyz/sHHJsGHlT+L7X5s56Bec1YDTVmzAsH4UBuvVBoXuIWEQ==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [android] + + '@snazzah/davey-darwin-arm64@0.1.9': + resolution: {integrity: sha512-z7oORvAPExikFkH6tvHhbUdZd77MYZp9VqbCpKEiI+sisWFVXgHde7F7iH3G4Bz6gUYJfgvKhWXiDRc+0SC4dg==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [darwin] + + '@snazzah/davey-darwin-x64@0.1.9': + resolution: {integrity: sha512-f1LzGyRGlM414KpXml3OgWVSd7CgylcdYaFj/zDBb8bvWjxyvsI9iMeuPfe/cduloxRj8dELde/yCDZtFR6PdQ==} + engines: {node: '>= 10'} + cpu: [x64] + os: [darwin] + + '@snazzah/davey-freebsd-x64@0.1.9': + resolution: {integrity: sha512-k6p3JY2b8rD6j0V9Ql7kBUMR4eJdcpriNwiHltLzmtGuz/nK5RGQdkEP68gTLc+Uj3xs5Cy0jRKmv2xJQBR4sA==} + engines: {node: '>= 10'} + cpu: [x64] + os: [freebsd] + + '@snazzah/davey-linux-arm-gnueabihf@0.1.9': + resolution: {integrity: sha512-xDaAFUC/1+n/YayNwKsqKOBMuW0KI6F0SjgWU+krYTQTVmAKNjOM80IjemrVoqTpBOxBsT80zEtct2wj11CE3Q==} + engines: {node: '>= 10'} + cpu: [arm] + os: [linux] + + '@snazzah/davey-linux-arm64-gnu@0.1.9': + resolution: {integrity: sha512-t1VxFBzWExPNpsNY/9oStdAAuHqFvwZvIO2YPYyVNstxfi2KmAbHMweHUW7xb2ppXuhVQZ4VGmmeXiXcXqhPBw==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + + '@snazzah/davey-linux-arm64-musl@0.1.9': + resolution: {integrity: sha512-Xvlr+nBPzuFV4PXHufddlt08JsEyu0p8mX2DpqdPxdpysYIH4I8V86yJiS4tk04a6pLBDd8IxTbBwvXJKqd/LQ==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [linux] + + '@snazzah/davey-linux-x64-gnu@0.1.9': + resolution: {integrity: sha512-6Uunc/NxiEkg1reroAKZAGfOtjl1CGa7hfTTVClb2f+DiA8ZRQWBh+3lgkq/0IeL262B4F14X8QRv5Bsv128qw==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + + '@snazzah/davey-linux-x64-musl@0.1.9': + resolution: {integrity: sha512-fFQ/n3aWt1lXhxSdy+Ge3gi5bR3VETMVsWhH0gwBALUKrbo3ZzgSktm4lNrXE9i0ncMz/CDpZ5i0wt/N3XphEQ==} + engines: {node: '>= 10'} + cpu: [x64] + os: [linux] + + '@snazzah/davey-wasm32-wasi@0.1.9': + resolution: {integrity: sha512-xWvzej8YCVlUvzlpmqJMIf0XmLlHqulKZ2e7WNe2TxQmsK+o0zTZqiQYs2MwaEbrNXBhYlHDkdpuwoXkJdscNQ==} + engines: {node: '>=14.0.0'} + cpu: [wasm32] + + '@snazzah/davey-win32-arm64-msvc@0.1.9': + resolution: {integrity: sha512-sTqry/DfltX2OdW1CTLKa3dFYN5FloAEb2yhGsY1i5+Bms6OhwByXfALvyMHYVo61Th2+sD+9BJpQffHFKDA3w==} + engines: {node: '>= 10'} + cpu: [arm64] + os: [win32] + + '@snazzah/davey-win32-ia32-msvc@0.1.9': + resolution: {integrity: sha512-twD3LwlkGnSwphsCtpGb5ztpBIWEvGdc0iujoVkdzZ6nJiq5p8iaLjJMO4hBm9h3s28fc+1Qd7AMVnagiOasnA==} + engines: {node: '>= 10'} + cpu: [ia32] + os: [win32] + + '@snazzah/davey-win32-x64-msvc@0.1.9': + resolution: {integrity: sha512-eMnXbv4GoTngWYY538i/qHz2BS+RgSXFsvKltPzKqnqzPzhQZIY7TemEJn3D5yWGfW4qHve9u23rz93FQqnQMA==} + engines: {node: '>= 10'} + cpu: [x64] + os: [win32] + + '@snazzah/davey@0.1.9': + resolution: {integrity: sha512-vNZk5y+IsxjwzTAXikvzz5pqMLb35YytC64nVF2MAFVhjpXu9ITOKUriZ0JG/llwzCAi56jb5x0cXDRIyE2A2A==} + engines: {node: '>= 10'} + '@socket.io/component-emitter@3.1.2': resolution: {integrity: sha512-9BCxFwvbGg/RsZK9tjXd8s4UcwR0MWeFQ1XEKIQVVvAGJyINdrqKMcTRyLoK8Rse1GjzLV9cwjWV1olXRWEXVA==} @@ -21439,6 +21529,67 @@ snapshots: '@sindresorhus/merge-streams@2.3.0': {} + '@snazzah/davey-android-arm-eabi@0.1.9': + optional: true + + '@snazzah/davey-android-arm64@0.1.9': + optional: true + + '@snazzah/davey-darwin-arm64@0.1.9': + optional: true + + '@snazzah/davey-darwin-x64@0.1.9': + optional: true + + '@snazzah/davey-freebsd-x64@0.1.9': + optional: true + + '@snazzah/davey-linux-arm-gnueabihf@0.1.9': + optional: true + + '@snazzah/davey-linux-arm64-gnu@0.1.9': + optional: true + + '@snazzah/davey-linux-arm64-musl@0.1.9': + optional: true + + '@snazzah/davey-linux-x64-gnu@0.1.9': + optional: true + + '@snazzah/davey-linux-x64-musl@0.1.9': + optional: true + + '@snazzah/davey-wasm32-wasi@0.1.9': + dependencies: + '@napi-rs/wasm-runtime': 1.1.0 + optional: true + + '@snazzah/davey-win32-arm64-msvc@0.1.9': + optional: true + + '@snazzah/davey-win32-ia32-msvc@0.1.9': + optional: true + + '@snazzah/davey-win32-x64-msvc@0.1.9': + optional: true + + '@snazzah/davey@0.1.9': + optionalDependencies: + '@snazzah/davey-android-arm-eabi': 0.1.9 + '@snazzah/davey-android-arm64': 0.1.9 + '@snazzah/davey-darwin-arm64': 0.1.9 + '@snazzah/davey-darwin-x64': 0.1.9 + '@snazzah/davey-freebsd-x64': 0.1.9 + '@snazzah/davey-linux-arm-gnueabihf': 0.1.9 + '@snazzah/davey-linux-arm64-gnu': 0.1.9 + '@snazzah/davey-linux-arm64-musl': 0.1.9 + '@snazzah/davey-linux-x64-gnu': 0.1.9 + '@snazzah/davey-linux-x64-musl': 0.1.9 + '@snazzah/davey-wasm32-wasi': 0.1.9 + '@snazzah/davey-win32-arm64-msvc': 0.1.9 + '@snazzah/davey-win32-ia32-msvc': 0.1.9 + '@snazzah/davey-win32-x64-msvc': 0.1.9 + '@socket.io/component-emitter@3.1.2': {} '@standard-schema/spec@1.1.0': {} diff --git a/services/discord-bot/package.json b/services/discord-bot/package.json index 183b57dcf..283448928 100644 --- a/services/discord-bot/package.json +++ b/services/discord-bot/package.json @@ -25,6 +25,7 @@ "@proj-airi/audio": "workspace:^", "@proj-airi/server-sdk": "workspace:^", "@proj-airi/server-shared": "workspace:^", + "@snazzah/davey": "^0.1.9", "@xsai-ext/providers": "catalog:", "@xsai/generate-speech": "catalog:", "@xsai/generate-text": "catalog:", diff --git a/services/discord-bot/src/adapters/airi-adapter.ts b/services/discord-bot/src/adapters/airi-adapter.ts index b61b2a9d3..a126bc7b9 100644 --- a/services/discord-bot/src/adapters/airi-adapter.ts +++ b/services/discord-bot/src/adapters/airi-adapter.ts @@ -1,9 +1,11 @@ +import type { Discord } from '@proj-airi/server-shared/types' import type { Interaction } from 'discord.js' import { env } from 'node:process' import { useLogg } from '@guiiai/logg' import { Client as AiriClient } from '@proj-airi/server-sdk' +import { ContextUpdateStrategy } from '@proj-airi/server-shared/types' import { Client, Events, GatewayIntentBits } from 'discord.js' import { handlePing, registerCommands, VoiceManager } from '../bots/discord/commands' @@ -31,6 +33,25 @@ function isDiscordConfig(config: unknown): config is DiscordConfig { && (typeof c.enabled === 'boolean' || typeof c.enabled === 'undefined') } +function normalizeDiscordMetadata(discord?: Discord): Discord | undefined { + if (!discord) + return undefined + + if (!discord.guildMember) + return discord + + const { guildMember } = discord + + return { + ...discord, + guildMember: { + id: guildMember.id ?? guildMember.displayName ?? guildMember.nickname ?? '', + nickname: guildMember.nickname ?? guildMember.displayName ?? '', + displayName: guildMember.displayName ?? guildMember.nickname ?? '', + }, + } +} + export class DiscordAdapter { private airiClient: AiriClient private discordClient: Client @@ -43,17 +64,23 @@ export class DiscordAdapter { // Initialize Discord client this.discordClient = new Client({ - intents: [GatewayIntentBits.Guilds, GatewayIntentBits.GuildVoiceStates], + intents: [ + GatewayIntentBits.Guilds, + GatewayIntentBits.GuildVoiceStates, + GatewayIntentBits.GuildMessages, + GatewayIntentBits.MessageContent, + ], }) // Initialize AIRI client this.airiClient = new AiriClient({ - name: 'discord-bot', + name: 'discord', possibleEvents: [ 'input:text', 'input:text:voice', 'input:voice', - 'ui:configure', + 'module:configure', + 'output:gen-ai:chat:message', ], token: config.airiToken, url: config.airiUrl, @@ -66,60 +93,58 @@ export class DiscordAdapter { private setupEventHandlers(): void { // Handle configuration from UI - this.airiClient.onEvent('ui:configure', async (event) => { - if (event.data.moduleName === 'discord') { - if (this.isReconnecting) { - log.warn('A reconnect is already in progress, skipping this configuration event.') - return - } - this.isReconnecting = true - try { - log.log('Received Discord configuration:', event.data.config) + this.airiClient.onEvent('module:configure', async (event) => { + if (this.isReconnecting) { + log.warn('A reconnect is already in progress, skipping this configuration event.') + return + } + this.isReconnecting = true + try { + log.log('Received Discord configuration:', event.data.config) - if (isDiscordConfig(event.data.config)) { - const config = event.data.config as DiscordConfig - const { token, enabled } = config + if (isDiscordConfig(event.data.config)) { + const config = event.data.config as DiscordConfig + const { token, enabled } = config - if (enabled === false) { - if (this.discordClient.isReady) { - log.log('Disabling Discord bot as per configuration...') - await this.discordClient.destroy() - } - return - } - - // If enabled, but no token is provided, stop the bot if it's running. - if (!token) { - log.warn('Discord bot enabled, but no token provided. Stopping bot.') - if (this.discordClient.isReady) { - await this.discordClient.destroy() - } - return - } - - // Connect or reconnect if token changed or client is not ready. - if (this.discordToken !== token || !this.discordClient.isReady) { - this.discordToken = token - if (this.discordClient.isReady) { - log.log('Reconnecting Discord client with new token...') - await this.discordClient.destroy() - } - log.log('Connecting Discord client...') - await this.discordClient.login(this.discordToken) - log.log('Discord client connected.') + if (enabled === false) { + if (this.discordClient.isReady) { + log.log('Disabling Discord bot as per configuration...') + await this.discordClient.destroy() } + return } - else { - log.warn('Invalid Discord configuration received, skipping...') + + // If enabled, but no token is provided, stop the bot if it's running. + if (!token) { + log.warn('Discord bot enabled, but no token provided. Stopping bot.') + if (this.discordClient.isReady) { + await this.discordClient.destroy() + } + return + } + + // Connect or reconnect if token changed or client is not ready. + if (this.discordToken !== token || !this.discordClient.isReady) { + this.discordToken = token + if (this.discordClient.isReady) { + log.log('Reconnecting Discord client with new token...') + await this.discordClient.destroy() + } + log.log('Connecting Discord client...') + await this.discordClient.login(this.discordToken) + log.log('Discord client connected.') } } - catch (error) { - log.withError(error as Error).error('Failed to apply Discord configuration.') - } - finally { - this.isReconnecting = false + else { + log.warn('Invalid Discord configuration received, skipping...') } } + catch (error) { + log.withError(error as Error).error('Failed to apply Discord configuration.') + } + finally { + this.isReconnecting = false + } }) // Handle input from AIRI system @@ -129,16 +154,94 @@ export class DiscordAdapter { // For now, we'll just log the input }) + // Handle output from AIRI system (IA response) + this.airiClient.onEvent('output:gen-ai:chat:message', async (event) => { + try { + const { message, discord } = event.data as { + message: { content: string } + discord?: { channelId: string } + } + + if (discord?.channelId) { + const channel = await this.discordClient.channels.fetch(discord.channelId) + if (channel?.isTextBased() && 'send' in channel && typeof channel.send === 'function') { + await channel.send(message.content) + } + } + } + catch (error) { + log.withError(error as Error).error('Failed to send response to Discord') + } + }) + // Set up Discord event handlers - this.discordClient.once(Events.ClientReady, (readyClient) => { + this.discordClient.once(Events.ClientReady, async (readyClient) => { log.log(`Discord bot ready! User: ${readyClient.user.tag}`) + // Register commands dynamically using the authenticated client's ID and token + await registerCommands(this.discordToken, readyClient.user.id) + }) + + // Handle text messages from Discord + this.discordClient.on(Events.MessageCreate, async (message) => { + if (message.author.bot) + return + + // Respond if the bot is mentioned + if (this.discordClient.user && message.mentions.has(this.discordClient.user)) { + const rawContent = message.content + const content = rawContent.replace(/<@!?\d+>/g, '').trim() + if (!content) + return + + log.log(`Received text mention from ${message.author.tag} in ${message.channelId}`) + + const discordContext: Discord = { + channelId: message.channelId, + guildId: message.guildId ?? undefined, + guildMember: { + id: message.author.id, + displayName: message.member?.displayName ?? message.author.username, + nickname: message.member?.nickname ?? message.author.username, + }, + } + const normalizedDiscord = normalizeDiscordMetadata(discordContext) + const displayName = normalizedDiscord?.guildMember?.displayName + const discordNotice = normalizedDiscord + ? `The input is coming from Discord channel ${normalizedDiscord.channelId} (Guild: ${normalizedDiscord.guildId ?? 'unknown'}).` + : undefined + + this.airiClient.send({ + type: 'input:text', + data: { + text: content, + textRaw: rawContent, + overrides: displayName + ? { + messagePrefix: `(From Discord user ${displayName}): `, + sessionId: 'discord', + } + : undefined, + contextUpdates: discordNotice + ? [{ + strategy: ContextUpdateStrategy.AppendSelf, + text: discordNotice, + content: discordNotice, + metadata: { + discord: normalizedDiscord, + }, + }] + : undefined, + discord: normalizedDiscord, + }, + }) + } }) this.discordClient.on(Events.InteractionCreate, async (interaction: Interaction) => { if (!interaction.isChatInputCommand()) return - log.log('Interaction received:', interaction) + log.log(`Interaction received: /${interaction.commandName} from ${interaction.user.tag}`) switch (interaction.commandName) { case 'ping': @@ -155,9 +258,6 @@ export class DiscordAdapter { log.log('Starting Discord adapter...') try { - // Register commands - await registerCommands() - // Log in to Discord if token is available if (this.discordToken) { await this.discordClient.login(this.discordToken) diff --git a/services/discord-bot/src/bots/discord/commands/index.ts b/services/discord-bot/src/bots/discord/commands/index.ts index b984d02e7..9e2953977 100644 --- a/services/discord-bot/src/bots/discord/commands/index.ts +++ b/services/discord-bot/src/bots/discord/commands/index.ts @@ -1,16 +1,14 @@ -import { env } from 'node:process' - import { REST, Routes, SlashCommandBuilder } from 'discord.js' export * from './ping' export * from './summon' -export async function registerCommands() { +export async function registerCommands(token: string, clientId: string) { const rest = new REST() - rest.setToken(env.DISCORD_TOKEN) + rest.setToken(token) rest.put( - Routes.applicationCommands(env.DISCORD_BOT_CLIENT_ID), + Routes.applicationCommands(clientId), { body: [ new SlashCommandBuilder().setName('ping').setDescription('Replies with Pong!'), new SlashCommandBuilder().setName('summon').setDescription('Summons the bot to your voice channel'),