faet(stage-ui,stage-web,stage-tamagotchi): ingest and process messages & contexts from sevrer channel

This commit is contained in:
Neko Ayaka
2025-12-02 02:46:15 +08:00
parent 6eaa3679ed
commit d666e28955
7 changed files with 477 additions and 29 deletions
+8 -1
View File
@@ -2,10 +2,11 @@
import { defineInvoke, defineInvokeHandler } from '@moeru/eventa'
import { useDisplayModelsStore } from '@proj-airi/stage-ui/stores/display-models'
import { useOnboardingStore } from '@proj-airi/stage-ui/stores/onboarding'
import { installChatContextBridge } from '@proj-airi/stage-ui/stores/plugins/chat-context-bridge'
import { useSettings } from '@proj-airi/stage-ui/stores/settings'
import { useTheme } from '@proj-airi/ui'
import { storeToRefs } from 'pinia'
import { onMounted, watch } from 'vue'
import { onMounted, onUnmounted, watch } from 'vue'
import { useI18n } from 'vue-i18n'
import { RouterView, useRoute, useRouter } from 'vue-router'
@@ -21,6 +22,7 @@ const { language, themeColorsHue, themeColorsHueDynamic } = storeToRefs(settings
const onboardingStore = useOnboardingStore()
const router = useRouter()
const route = useRoute()
let disposeChatBridge: (() => void) | undefined
watch(language, () => {
i18n.locale.value = language.value
@@ -38,6 +40,9 @@ onMounted(async () => {
await displayModelsStore.loadDisplayModelsFromIndexedDB()
await settingsStore.initializeStageModel()
const bridge = installChatContextBridge()
disposeChatBridge = bridge.dispose
const context = useElectronEventaContext()
const startTrackingCursorPoint = defineInvoke(context.value, electronStartTrackMousePosition)
await startTrackingCursorPoint()
@@ -53,6 +58,8 @@ watch(themeColorsHue, () => {
watch(themeColorsHueDynamic, () => {
document.documentElement.classList.toggle('dynamic-hue', themeColorsHueDynamic.value)
}, { immediate: true })
onUnmounted(() => disposeChatBridge?.())
</script>
<template>
+5 -4
View File
@@ -1,8 +1,8 @@
<script setup lang="ts">
import { OnboardingDialog, ToasterRoot } from '@proj-airi/stage-ui/components'
import { useDisplayModelsStore } from '@proj-airi/stage-ui/stores/display-models'
import { useModsChannelServerStore } from '@proj-airi/stage-ui/stores/mods/api/channel-server'
import { useOnboardingStore } from '@proj-airi/stage-ui/stores/onboarding'
import { installChatContextBridge } from '@proj-airi/stage-ui/stores/plugins/chat-context-bridge'
import { useSettings } from '@proj-airi/stage-ui/stores/settings'
import { useTheme } from '@proj-airi/ui'
import { StageTransitionGroup } from '@proj-airi/ui-transitions'
@@ -24,7 +24,7 @@ const settings = storeToRefs(settingsStore)
const onboardingStore = useOnboardingStore()
const { shouldShowSetup } = storeToRefs(onboardingStore)
const { isDark } = useTheme()
const channelServerStore = useModsChannelServerStore()
let disposeChatBridge: (() => void) | undefined
const primaryColor = computed(() => {
return isDark.value
@@ -63,14 +63,15 @@ watch(settings.themeColorsHueDynamic, () => {
// Initialize first-time setup check when app mounts
onMounted(async () => {
onboardingStore.initializeSetupCheck()
channelServerStore.initialize()
const bridge = installChatContextBridge()
disposeChatBridge = bridge.dispose
await displayModelsStore.loadDisplayModelsFromIndexedDB()
await settingsStore.initializeStageModel()
})
onUnmounted(() => {
channelServerStore.dispose()
disposeChatBridge?.()
})
// Handle first-time setup events
@@ -15,6 +15,34 @@ interface InputSource {
discord: Discord
}
export type ContextSource =
| 'text'
| 'stt'
| 'vision'
| 'llm'
| 'server-channel'
| 'plugin'
| 'system'
export interface ContextMessage<Payload = unknown, Meta = Record<string, unknown>> {
/**
* Session identifier so UIs can group conversations from multiple windows/devices.
*/
sessionId: string
/**
* Unix timestamp in milliseconds.
*/
ts: number
role: 'user' | 'assistant' | 'system' | 'error'
source: ContextSource
/**
* The actual payload being carried. Keep this generic so different inputs (text, stt, vision)
* can share the same envelope.
*/
payload: Payload
meta?: Meta
}
export interface WebSocketBaseEvent<T, D> {
type: T
data: D
@@ -60,6 +88,7 @@ export interface WebSocketEvents<C = undefined> {
audio: ArrayBuffer
} & Partial<WithInputSource<'browser' | 'discord'>>
'vscode:context': C
'context:update': ContextMessage
}
export type WebSocketEvent<C = undefined> = {
+121
View File
@@ -0,0 +1,121 @@
import type { ContextMessage } from '@proj-airi/server-sdk'
import type { ContextPayload } from './chat'
import { createPinia, setActivePinia } from 'pinia'
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
import { ref } from 'vue'
import { useChatStore } from './chat'
import { installChatContextBridge } from './plugins/chat-context-bridge'
const mockSendContextUpdate = vi.fn()
const mockInitialize = vi.fn().mockResolvedValue(undefined)
let contextUpdateHandler: ((event: { type: 'context:update', data: ContextMessage }) => void | Promise<void>) | null = null
let broadcastPosts: ContextMessage[] = []
let bridge: { dispose: () => void } | null = null
const localStorageMap = new Map<string, unknown>()
vi.mock('@vueuse/core', () => {
return {
useLocalStorage: <T>(key: string, defaultValue: T) => {
if (!localStorageMap.has(key))
localStorageMap.set(key, ref(defaultValue))
return localStorageMap.get(key) as ReturnType<typeof ref<T>>
},
useBroadcastChannel: () => {
const data = ref<ContextMessage | undefined>()
const post = (value: ContextMessage) => {
broadcastPosts.push(value)
data.value = value
}
return { data, post }
},
}
})
vi.mock('./llm', () => ({
useLLM: () => ({
stream: vi.fn(),
discoverToolsCompatibility: vi.fn(),
}),
}))
vi.mock('./modules', () => ({
useAiriCardStore: () => ({
systemPrompt: ref(''),
}),
}))
vi.mock('./mods/api/channel-server', () => ({
useModsChannelServerStore: () => ({
connected: ref(true),
initialize: mockInitialize,
onContextUpdate: (cb: typeof contextUpdateHandler) => {
contextUpdateHandler = cb
return () => { contextUpdateHandler = null }
},
sendContextUpdate: mockSendContextUpdate,
}),
}))
describe('chat store', () => {
beforeEach(() => {
setActivePinia(createPinia())
broadcastPosts = []
localStorageMap.clear()
mockSendContextUpdate.mockClear()
mockInitialize.mockClear()
contextUpdateHandler = null
bridge = null
})
afterEach(() => {
bridge?.dispose()
bridge = null
vi.clearAllMocks()
})
it('ingests assistant context updates from the channel server', async () => {
const store = useChatStore()
bridge = installChatContextBridge()
expect(mockInitialize).toHaveBeenCalled()
expect(contextUpdateHandler).toBeTruthy()
const envelope: ContextMessage = {
sessionId: 'session-ctx',
ts: 123,
role: 'assistant',
source: 'llm',
payload: { content: 'hello from server' },
}
await contextUpdateHandler?.({ type: 'context:update', data: envelope })
store.setActiveSession('session-ctx')
const last = store.messages.at(-1)
expect(last?.role).toBe('assistant')
expect(last?.content).toBe('hello from server')
})
it('publishes local context updates through the shared channel server store', () => {
const store = useChatStore()
bridge = installChatContextBridge()
const envelope: ContextMessage = {
sessionId: 'session-local',
ts: 456,
role: 'assistant',
source: 'system',
payload: { content: 'local broadcast' },
}
store.publishContextMessage(envelope as ContextMessage<ContextPayload, Record<string, unknown>>, 'local')
expect(mockSendContextUpdate).toHaveBeenCalledWith(envelope)
expect(broadcastPosts).toContain(envelope)
})
})
+199 -17
View File
@@ -1,3 +1,4 @@
import type { ContextMessage, ContextSource } from '@proj-airi/server-sdk'
import type { ChatProvider } from '@xsai-ext/shared-providers'
import type { CommonContentPart, Message, SystemMessage } from '@xsai/shared-chat'
@@ -6,7 +7,7 @@ import type { ChatAssistantMessage, ChatMessage, ChatSlices } from '../types/cha
import { useLocalStorage } from '@vueuse/core'
import { defineStore, storeToRefs } from 'pinia'
import { ref, toRaw, watch } from 'vue'
import { computed, ref, toRaw, watch } from 'vue'
import { useLlmmarkerParser } from '../composables/llmmarkerParser'
import { useLLM } from '../stores/llm'
@@ -19,12 +20,36 @@ export interface ErrorMessage {
content: string
}
interface MessageContext {
sessionId: string
source: ContextSource
ts: number
meta?: Record<string, unknown>
}
type ChatEntry = (ChatMessage | ErrorMessage) & { context?: MessageContext }
export interface ContextPayload {
content?: unknown
slices?: ChatSlices[]
tool_results?: ChatAssistantMessage['tool_results']
text?: string
}
const CHAT_STORAGE_KEY = 'chat/messages/v2'
const ACTIVE_SESSION_STORAGE_KEY = 'chat/active-session'
export const CONTEXT_CHANNEL_NAME = 'airi-context-update'
export const useChatStore = defineStore('chat', () => {
const { stream, discoverToolsCompatibility } = useLLM()
const { systemPrompt } = storeToRefs(useAiriCardStore())
const activeSessionId = useLocalStorage<string>(ACTIVE_SESSION_STORAGE_KEY, 'default')
const sessionMessages = useLocalStorage<Record<string, ChatEntry[]>>(CHAT_STORAGE_KEY, {})
const sending = ref(false)
// ----- Hooks (UI callbacks) -----
const onBeforeMessageComposedHooks = ref<Array<(message: string) => Promise<void>>>([])
const onAfterMessageComposedHooks = ref<Array<(message: string) => Promise<void>>>([])
const onBeforeSendHooks = ref<Array<(message: string) => Promise<void>>>([])
@@ -33,6 +58,7 @@ export const useChatStore = defineStore('chat', () => {
const onTokenSpecialHooks = ref<Array<(special: string) => Promise<void>>>([])
const onStreamEndHooks = ref<Array<() => Promise<void>>>([])
const onAssistantResponseEndHooks = ref<Array<(message: string) => Promise<void>>>([])
const onContextPublishHooks = ref<Array<(envelope: ContextMessage<ContextPayload>, origin: 'local' | 'ws' | 'broadcast') => Promise<void> | void>>([])
function onBeforeMessageComposed(cb: (message: string) => Promise<void>) {
onBeforeMessageComposedHooks.value.push(cb)
@@ -66,6 +92,14 @@ export const useChatStore = defineStore('chat', () => {
onAssistantResponseEndHooks.value.push(cb)
}
function onContextPublish(cb: (envelope: ContextMessage<ContextPayload>, origin: 'local' | 'ws' | 'broadcast') => Promise<void> | void) {
onContextPublishHooks.value.push(cb)
return () => {
onContextPublishHooks.value = onContextPublishHooks.value.filter(hook => hook !== cb)
}
}
function clearHooks() {
onBeforeMessageComposedHooks.value = []
onAfterMessageComposedHooks.value = []
@@ -75,8 +109,10 @@ export const useChatStore = defineStore('chat', () => {
onTokenSpecialHooks.value = []
onStreamEndHooks.value = []
onAssistantResponseEndHooks.value = []
onContextPublishHooks.value = []
}
// ----- Session state helpers -----
// I know this nu uh, better than loading all language on rehypeShiki
const codeBlockSystemPrompt = '- For any programming code block, always specify the programming language that supported on @shikijs/rehype on the rendered markdown, eg. ```python ... ```\n'
const mathSyntaxSystemPrompt = '- For any math equation, use LaTeX format, eg: $ x^3 $, always escape dollar sign outside math equation\n'
@@ -89,20 +125,122 @@ export const useChatStore = defineStore('chat', () => {
} satisfies SystemMessage
}
const messages = useLocalStorage<Array<ChatMessage | ErrorMessage>>('chat/messages', [generateInitialMessage()])
function ensureSession(sessionId: string) {
if (!sessionMessages.value[sessionId] || sessionMessages.value[sessionId].length === 0) {
sessionMessages.value[sessionId] = [{
...generateInitialMessage(),
context: {
sessionId,
source: 'system',
ts: Date.now(),
},
}]
}
}
function cleanupMessages() {
messages.value = [generateInitialMessage()]
ensureSession(activeSessionId.value)
const messages = computed<ChatEntry[]>({
get: () => {
ensureSession(activeSessionId.value)
return sessionMessages.value[activeSessionId.value]
},
set: (value) => {
sessionMessages.value[activeSessionId.value] = value
},
})
function setActiveSession(sessionId: string) {
activeSessionId.value = sessionId
ensureSession(sessionId)
}
function cleanupMessages(sessionId = activeSessionId.value) {
sessionMessages.value[sessionId] = [{
...generateInitialMessage(),
context: {
sessionId,
source: 'system',
ts: Date.now(),
},
}]
}
watch(systemPrompt, () => {
if (messages.value.length > 0 && messages.value[0].role === 'system') {
messages.value[0] = generateInitialMessage()
for (const [sessionId, history] of Object.entries(sessionMessages.value)) {
if (history.length > 0 && history[0].role === 'system') {
sessionMessages.value[sessionId][0] = {
...generateInitialMessage(),
context: {
sessionId,
source: 'system',
ts: Date.now(),
},
}
}
}
}, {
immediate: true,
})
}, { immediate: true })
// ----- Context bridge (WS + BroadcastChannel) -----
function normalizePayload(payload?: ContextPayload) {
const baseContent = payload?.content ?? payload?.text ?? ''
const normalizedContent = typeof baseContent === 'string' || Array.isArray(baseContent)
? baseContent
: JSON.stringify(baseContent)
return {
content: normalizedContent,
slices: payload?.slices ?? [],
tool_results: payload?.tool_results ?? [],
}
}
function ingestContextMessage(envelope: ContextMessage<ContextPayload>, origin: 'local' | 'ws' | 'broadcast' = 'local') {
ensureSession(envelope.sessionId)
const { content, slices, tool_results } = normalizePayload(envelope.payload)
const context: MessageContext = {
sessionId: envelope.sessionId,
source: envelope.source,
ts: envelope.ts,
meta: envelope.meta,
}
const nextHistory = sessionMessages.value[envelope.sessionId]
if (envelope.role === 'assistant') {
nextHistory.push({
role: 'assistant',
content,
slices,
tool_results,
context,
})
}
else if (envelope.role === 'error') {
nextHistory.push({
role: 'error',
content: typeof content === 'string' ? content : JSON.stringify(content),
context,
})
}
else {
nextHistory.push({
role: envelope.role,
content,
context,
} as ChatEntry)
}
}
function publishContextMessage(envelope: ContextMessage<ContextPayload>, origin: 'local' | 'ws' | 'broadcast' = 'local') {
for (const hook of onContextPublishHooks.value)
void hook(envelope, origin)
}
// ----- Send flow (user -> LLM -> assistant) -----
const streamingMessage = ref<ChatAssistantMessage>({ role: 'assistant', content: '', slices: [], tool_results: [] })
async function send(
@@ -141,7 +279,16 @@ export const useChatStore = defineStore('chat', () => {
const finalContent = contentParts.length > 1 ? contentParts : sendingMessage
messages.value.push({ role: 'user', content: finalContent })
const userContext: MessageContext = { sessionId: activeSessionId.value, source: 'text', ts: Date.now() }
messages.value.push({ role: 'user', content: finalContent, context: userContext })
publishContextMessage({
sessionId: userContext.sessionId,
ts: userContext.ts,
role: 'user',
source: userContext.source,
payload: { content: finalContent },
}, 'local')
const parser = useLlmmarkerParser({
onLiteral: async (literal) => {
@@ -187,13 +334,19 @@ export const useChatStore = defineStore('chat', () => {
})
streamingMessage.value = { role: 'assistant', content: '', slices: [], tool_results: [] }
const newMessages = messages.value.map((msg) => {
if (msg.role === 'assistant') {
const { slices: _, ...rest } = msg // exclude slices
rest.tool_results = toRaw(rest.tool_results)
return toRaw(rest)
const { context: _context, ...withoutContext } = msg
const rawMessage = toRaw(withoutContext)
if (rawMessage.role === 'assistant') {
const { slices: _, tool_results, ...rest } = rawMessage as ChatAssistantMessage
return {
...toRaw(rest),
tool_results: toRaw(tool_results),
}
}
return toRaw(msg)
return rawMessage
})
for (const hook of onAfterMessageComposedHooks.value) {
@@ -241,8 +394,32 @@ export const useChatStore = defineStore('chat', () => {
await parser.end()
// Add the completed message to the history only if it has content
if (streamingMessage.value.slices.length > 0)
messages.value.push(toRaw(streamingMessage.value))
if (streamingMessage.value.slices.length > 0) {
const assistantContext: MessageContext = {
sessionId: activeSessionId.value,
source: 'llm',
ts: Date.now(),
}
const assistantMessage: ChatEntry = {
...(toRaw(streamingMessage.value) as ChatAssistantMessage),
context: assistantContext,
}
messages.value.push(assistantMessage)
publishContextMessage({
sessionId: assistantContext.sessionId,
ts: assistantContext.ts,
role: 'assistant',
source: assistantContext.source,
payload: {
content: assistantMessage.content,
slices: assistantMessage.slices,
tool_results: assistantMessage.tool_results,
},
}, 'local')
}
// Reset the streaming message for the next turn
streamingMessage.value = { role: 'assistant', content: '', slices: [], tool_results: [] }
@@ -278,12 +455,16 @@ export const useChatStore = defineStore('chat', () => {
return {
sending,
activeSessionId,
messages,
streamingMessage,
discoverToolsCompatibility,
send,
setActiveSession,
ingestContextMessage,
publishContextMessage,
cleanupMessages,
clearHooks,
@@ -295,5 +476,6 @@ export const useChatStore = defineStore('chat', () => {
onTokenSpecial,
onStreamEnd,
onAssistantResponseEnd,
onContextPublish,
}
})
@@ -1,4 +1,4 @@
import type { WebSocketEvent } from '@proj-airi/server-sdk'
import type { ContextMessage, WebSocketBaseEvent, WebSocketEvent, WebSocketEvents } from '@proj-airi/server-sdk'
import { Client } from '@proj-airi/server-sdk'
import { defineStore } from 'pinia'
@@ -7,35 +7,67 @@ import { ref } from 'vue'
export const useModsChannelServerStore = defineStore('mods:channels:proj-airi:server', () => {
const connected = ref(false)
const client = ref<Client>()
const initializing = ref<Promise<void> | null>(null)
const pendingSend = ref<Array<WebSocketEvent>>([])
function initialize(options?: { token?: string }) {
return new Promise<void>((resolve, reject) => {
function initialize(options?: { token?: string, possibleEvents?: Array<keyof WebSocketEvents> }) {
if (connected.value && client.value)
return Promise.resolve()
if (initializing.value)
return initializing.value
const possibleEvents = Array.from(new Set<keyof WebSocketEvents>([
'ui:configure',
'context:update',
...(options?.possibleEvents ?? []),
]))
initializing.value = new Promise<void>((resolve, reject) => {
client.value = new Client({
name: 'proj-airi:ui:stage',
url: import.meta.env.VITE_AIRI_WS_URL || 'ws://localhost:6121/ws',
token: options?.token,
possibleEvents: [
'ui:configure',
'module:authenticated',
],
possibleEvents,
onError: (error) => {
client.value = undefined
connected.value = false
initializing.value = null
reject(error)
},
onClose: () => {
connected.value = false
initializing.value = null
},
})
client.value.onEvent('module:authenticated', (event) => {
if (event.data.authenticated) {
connected.value = true
flush()
initializeListeners()
resolve()
return
}
connected.value = false
})
})
return initializing.value
}
function initializeListeners() {
if (!client.value)
// No-op for now; keep placeholder for future shared listeners.
// eslint-disable-next-line no-useless-return
return
}
function send(data: WebSocketEvent) {
if (!client.value && !initializing.value)
void initialize()
if (client.value && connected.value) {
client.value.send(data)
}
@@ -54,12 +86,31 @@ export const useModsChannelServerStore = defineStore('mods:channels:proj-airi:se
}
}
function onContextUpdate(callback: (event: WebSocketBaseEvent<'context:update', ContextMessage>) => void | Promise<void>) {
if (!client.value && !initializing.value)
void initialize()
client.value?.onEvent('context:update', callback as any)
return () => {
client.value?.offEvent('context:update', callback as any)
}
}
function sendContextUpdate(message: ContextMessage) {
send({
type: 'context:update',
data: message,
})
}
function dispose() {
flush()
client.value?.close()
connected.value = false
client.value = undefined
initializing.value = null
}
return {
@@ -67,6 +118,8 @@ export const useModsChannelServerStore = defineStore('mods:channels:proj-airi:se
initialize,
send,
sendContextUpdate,
onContextUpdate,
dispose,
}
})
@@ -0,0 +1,55 @@
import type { ContextMessage } from '@proj-airi/server-sdk'
import { useBroadcastChannel } from '@vueuse/core'
import { watch } from 'vue'
import type { ContextPayload } from '../chat'
import { CONTEXT_CHANNEL_NAME, useChatStore } from '../chat'
import { useModsChannelServerStore } from '../mods/api/channel-server'
let installed = false
export function installChatContextBridge() {
if (installed) {
return {
dispose: () => {},
}
}
const chatStore = useChatStore()
const modsChannelServer = useModsChannelServerStore()
const { post: broadcastContext, data: incomingContext } = useBroadcastChannel<ContextMessage<ContextPayload>, ContextMessage<ContextPayload>>({
name: CONTEXT_CHANNEL_NAME,
})
const stopIncomingWatch = watch(incomingContext, (event) => {
if (event)
chatStore.ingestContextMessage(event, 'broadcast')
})
const offPublish = chatStore.onContextPublish((envelope, origin) => {
if (origin !== 'broadcast')
broadcastContext(envelope)
if (origin === 'local')
modsChannelServer.sendContextUpdate(envelope)
})
modsChannelServer.initialize({ possibleEvents: ['context:update'] }).catch(error => console.error('Context bridge init error:', error))
const offWs = modsChannelServer.onContextUpdate((event) => {
const envelope = event.data as ContextMessage<ContextPayload, Record<string, unknown>>
chatStore.ingestContextMessage(envelope, 'ws')
broadcastContext(envelope)
})
installed = true
return {
dispose: () => {
stopIncomingWatch()
offPublish()
offWs?.()
installed = false
},
}
}