From 27b39900cf8067daa4104bd72d14cc0bd5ef8847 Mon Sep 17 00:00:00 2001 From: Neko Ayaka Date: Sun, 30 Nov 2025 21:36:40 +0800 Subject: [PATCH] feat(stage-tamagotchi): enable widgets calling as tool --- apps/stage-tamagotchi/package.json | 1 + .../renderer/components/InteractiveArea.vue | 14 +- .../stores/tools/builtin/widgets.test.ts | 121 ++++++++++++ .../renderer/stores/tools/builtin/widgets.ts | 176 ++++++++++++++++++ apps/stage-tamagotchi/vitest.config.ts | 8 + packages/stage-ui/src/stores/chat.ts | 4 +- packages/stage-ui/src/stores/llm.ts | 18 +- pnpm-lock.yaml | 3 + 8 files changed, 336 insertions(+), 9 deletions(-) create mode 100644 apps/stage-tamagotchi/src/renderer/stores/tools/builtin/widgets.test.ts create mode 100644 apps/stage-tamagotchi/src/renderer/stores/tools/builtin/widgets.ts create mode 100644 apps/stage-tamagotchi/vitest.config.ts diff --git a/apps/stage-tamagotchi/package.json b/apps/stage-tamagotchi/package.json index 9263a3988..da5d9c996 100644 --- a/apps/stage-tamagotchi/package.json +++ b/apps/stage-tamagotchi/package.json @@ -65,6 +65,7 @@ "@xsai/shared-chat": "catalog:", "@xsai/stream-text": "catalog:", "@xsai/stream-transcription": "0.4.0-beta.8", + "@xsai/tool": "catalog:", "@xsai/utils-chat": "catalog:", "animejs": "^4.2.2", "colorjs.io": "^0.5.2", diff --git a/apps/stage-tamagotchi/src/renderer/components/InteractiveArea.vue b/apps/stage-tamagotchi/src/renderer/components/InteractiveArea.vue index 6212b72df..9f192ea49 100644 --- a/apps/stage-tamagotchi/src/renderer/components/InteractiveArea.vue +++ b/apps/stage-tamagotchi/src/renderer/components/InteractiveArea.vue @@ -13,6 +13,8 @@ import { useI18n } from 'vue-i18n' import TamagotchiChatHistory from './ChatHistory.vue' +import { widgetsTools } from '../stores/tools/builtin/widgets' + const messageInput = ref('') const listening = ref(false) const attachments = ref<{ type: 'image', data: string, mimeType: string, url: string }[]>([]) @@ -27,6 +29,8 @@ const { activeModel, activeProvider } = storeToRefs(useConsciousnessStore()) const isComposing = ref(false) async function handleSend() { + debugger + if (isComposing.value) { return } @@ -43,6 +47,7 @@ async function handleSend() { chatProvider: await providersStore.getProviderInstance(activeProvider.value), providerConfig, attachments: attachmentsToSend, + tools: widgetsTools, }) // clear after sending @@ -132,7 +137,7 @@ watch([activeProvider, activeModel], async () => { if (activeProvider.value && activeModel.value) { await discoverToolsCompatibility(activeModel.value, await providersStore.getProviderInstance(activeProvider.value), []) } -}) +}, { immediate: true }) onAfterMessageComposed(async () => { messageInput.value = '' @@ -170,9 +175,10 @@ onAfterMessageComposed(async () => { { + describe('normalizeComponentProps', () => { + it('parses JSON strings into objects', () => { + const result = normalizeComponentProps('{"city":"Tokyo","temp":15}') + expect(result).toEqual({ city: 'Tokyo', temp: 15 }) + }) + + it('returns empty object for empty or undefined', () => { + expect(normalizeComponentProps(' ')).toEqual({}) + expect(normalizeComponentProps(undefined)).toEqual({}) + expect(normalizeComponentProps(null as any)).toEqual({}) + }) + + it('passes through object inputs', () => { + const payload = { foo: 'bar', nested: { a: 1 } } + expect(normalizeComponentProps(payload)).toBe(payload) + }) + + it('throws on invalid JSON', () => { + expect(() => normalizeComponentProps('{ bad json ')).toThrow() + }) + }) + describe('executeWidgetAction with mocked invokers', () => { + const makeInvokers = (): WidgetInvokers => ({ + prepareWindow: vi.fn(), + openWindow: vi.fn(), + addWidget: vi.fn(), + updateWidget: vi.fn(), + removeWidget: vi.fn(), + clearWidgets: vi.fn(), + }) + + it('spawns with ttl conversion and parsed props', async () => { + const invokers = makeInvokers() + vi.mocked(invokers.addWidget).mockResolvedValue('abc123') + + const result = await executeWidgetAction({ + action: 'spawn', + id: ' abc123 ', + componentName: 'weather', + componentProps: '{"city":"Tokyo"}', + size: 'm', + ttlSeconds: 2, + }, { invokers }) + + expect(result).toContain('abc123') + expect(invokers.addWidget).toHaveBeenCalledTimes(1) + expect(invokers.addWidget).toHaveBeenCalledWith({ + id: 'abc123', + componentName: 'weather', + componentProps: { city: 'Tokyo' }, + size: 'm', + ttlMs: 2000, + }) + }) + + it('updates props and trims id', async () => { + const invokers = makeInvokers() + await executeWidgetAction({ + action: 'update', + id: ' xyz ', + componentName: '', + componentProps: '{"foo":1}', + size: 'm', + ttlSeconds: 0, + }, { invokers }) + + expect(invokers.updateWidget).toHaveBeenCalledWith({ id: 'xyz', componentProps: { foo: 1 } }) + }) + + it('removes when id provided', async () => { + const invokers = makeInvokers() + await executeWidgetAction({ + action: 'remove', + id: 'rem-id', + componentName: '', + componentProps: '{}', + size: 's', + ttlSeconds: 0, + }, { invokers }) + + expect(invokers.removeWidget).toHaveBeenCalledWith({ id: 'rem-id' }) + }) + + it('opens window with prepared id', async () => { + const invokers = makeInvokers() + vi.mocked(invokers.prepareWindow).mockResolvedValue('prepared-id') + await executeWidgetAction({ + action: 'open', + id: ' prepared-id ', + componentName: '', + componentProps: '{}', + size: 'l', + ttlSeconds: 0, + }, { invokers }) + + expect(invokers.prepareWindow).toHaveBeenCalledWith({ id: 'prepared-id' }) + expect(invokers.openWindow).toHaveBeenCalledWith({ id: 'prepared-id' }) + }) + + it('clears widgets', async () => { + const invokers = makeInvokers() + await executeWidgetAction({ + action: 'clear', + id: '', + componentName: '', + componentProps: '{}', + size: 'm', + ttlSeconds: 0, + }, { invokers }) + + expect(invokers.clearWidgets).toHaveBeenCalledTimes(1) + }) + }) +}) diff --git a/apps/stage-tamagotchi/src/renderer/stores/tools/builtin/widgets.ts b/apps/stage-tamagotchi/src/renderer/stores/tools/builtin/widgets.ts new file mode 100644 index 000000000..f391e8c8c --- /dev/null +++ b/apps/stage-tamagotchi/src/renderer/stores/tools/builtin/widgets.ts @@ -0,0 +1,176 @@ +import { defineInvoke } from '@moeru/eventa' +import { createContext } from '@moeru/eventa/adapters/electron/renderer' +import { tool } from '@xsai/tool' +import { z } from 'zod' + +import { widgetsAdd, widgetsClear, widgetsOpenWindow, widgetsPrepareWindow, widgetsRemove, widgetsUpdate } from '../../../../shared/eventa' + +type SizePreset = 's' | 'm' | 'l' + +type WidgetActionInput + = | { + action: 'spawn' + id: string + componentName: string + componentProps: string | Record + size: SizePreset + ttlSeconds: number + } + | { + action: 'update' + id: string + componentProps: string | Record + componentName?: string + size?: SizePreset + ttlSeconds?: number + } + | { + action: 'remove' + id: string + componentName?: string + componentProps?: string | Record + size?: SizePreset + ttlSeconds?: number + } + | { + action: 'clear' + id: string + componentName?: string + componentProps?: string | Record + size?: SizePreset + ttlSeconds?: number + } + | { + action: 'open' + id: string + componentName?: string + componentProps?: string | Record + size?: SizePreset + ttlSeconds?: number + } + +export type WidgetInvokers = ReturnType + +let cachedInvokers: WidgetInvokers | undefined + +function createInvokers() { + const ipcRenderer = typeof window !== 'undefined' ? (window as any)?.electron?.ipcRenderer : undefined + if (!ipcRenderer) + throw new Error('Widget tools are only available in the desktop app.') + + const { context } = createContext(ipcRenderer) + + return { + prepareWindow: defineInvoke(context, widgetsPrepareWindow), + openWindow: defineInvoke(context, widgetsOpenWindow), + addWidget: defineInvoke(context, widgetsAdd), + updateWidget: defineInvoke(context, widgetsUpdate), + removeWidget: defineInvoke(context, widgetsRemove), + clearWidgets: defineInvoke(context, widgetsClear), + } +} + +function resolveInvokers(override?: WidgetInvokers): WidgetInvokers { + if (override) + return override + if (!cachedInvokers) + cachedInvokers = createInvokers() + return cachedInvokers +} + +const widgetParams = z.object({ + action: z.enum(['spawn', 'update', 'remove', 'clear', 'open']).describe('Choose one: spawn, update, remove, clear, open'), + id: z.string().default('').describe('Widget id; required for update/remove, optional for spawn/open'), + componentName: z.string().default('').describe('Widget component to render, e.g. weather (required for spawn)'), + componentProps: z.string().default('{}').describe('Widget props as JSON string (e.g. {"city":"Tokyo"})'), + size: z.enum(['s', 'm', 'l']).default('m'), + ttlSeconds: z.number().int().nonnegative().default(0).describe('Auto-close timer in seconds (spawn only)'), +}).strict() + +export function normalizeComponentProps(raw?: string | Record) { + if (raw === undefined || raw === null) + return {} + + if (typeof raw === 'string') { + const payload = raw.trim() + if (!payload) + return {} + try { + const parsed = JSON.parse(payload) + return typeof parsed === 'object' && parsed !== null ? parsed : {} + } + catch (error) { + throw new Error(`Invalid JSON for componentProps: ${(error as Error).message}`) + } + } + + if (typeof raw === 'object') + return raw + + return {} +} + +export async function executeWidgetAction(input: WidgetActionInput, deps?: { invokers?: WidgetInvokers }) { + const invokers = resolveInvokers(deps?.invokers) + const normalizedId = input.id?.trim() || undefined + + switch (input.action) { + case 'spawn': { + if (!input.componentName?.trim()) + throw new Error('componentName is required to spawn a widget.') + + const componentProps = normalizeComponentProps(input.componentProps) + const ttlMs = input.ttlSeconds ? Math.floor(input.ttlSeconds * 1000) : 0 + const id = await invokers.addWidget({ + id: normalizedId, + componentName: input.componentName, + componentProps, + size: input.size ?? 'm', + ttlMs, + }) + + return `Spawned widget${id ? ` (${id})` : ''}.` + } + case 'update': { + if (!normalizedId) + throw new Error('id is required to update a widget.') + + const componentProps = normalizeComponentProps(input.componentProps) + await invokers.updateWidget({ + id: normalizedId, + componentProps, + }) + + return `Updated widget (${normalizedId}).` + } + case 'remove': { + if (!normalizedId) + throw new Error('id is required to remove a widget.') + + await invokers.removeWidget({ id: normalizedId }) + return `Removed widget (${normalizedId}).` + } + case 'clear': { + await invokers.clearWidgets() + return 'Cleared all widgets.' + } + case 'open': { + const id = await invokers.prepareWindow(normalizedId ? { id: normalizedId } : {}) + await invokers.openWindow(normalizedId ? { id: normalizedId } : {}) + return `Opened widget window${id ? ` (${id})` : ''}.` + } + default: + return 'No action performed.' + } +} + +const tools = [ + tool({ + name: 'stage_widgets', + description: 'Manage overlay widgets in the Stage desktop app (spawn, update, remove, clear, or open the widgets window).', + execute: params => executeWidgetAction(params as WidgetActionInput), + parameters: widgetParams, + }), +] + +export const widgetsTools = async () => Promise.all(tools) diff --git a/apps/stage-tamagotchi/vitest.config.ts b/apps/stage-tamagotchi/vitest.config.ts new file mode 100644 index 000000000..5d54d7722 --- /dev/null +++ b/apps/stage-tamagotchi/vitest.config.ts @@ -0,0 +1,8 @@ +import { defineConfig } from 'vitest/config' + +export default defineConfig({ + test: { + include: ['src/**/*.test.ts'], + exclude: ['**/node_modules/**', '**/.git/**'], + }, +}) diff --git a/packages/stage-ui/src/stores/chat.ts b/packages/stage-ui/src/stores/chat.ts index 7ddf31105..bde9caa16 100644 --- a/packages/stage-ui/src/stores/chat.ts +++ b/packages/stage-ui/src/stores/chat.ts @@ -1,7 +1,7 @@ import type { ChatProvider } from '@xsai-ext/shared-providers' import type { CommonContentPart, Message, SystemMessage } from '@xsai/shared-chat' -import type { StreamEvent } from '../stores/llm' +import type { StreamEvent, StreamOptions } from '../stores/llm' import type { ChatAssistantMessage, ChatMessage, ChatSlices } from '../types/chat' import { useLocalStorage } from '@vueuse/core' @@ -112,6 +112,7 @@ export const useChatStore = defineStore('chat', () => { chatProvider: ChatProvider providerConfig?: Record attachments?: { type: 'image', data: string, mimeType: string }[] + tools?: StreamOptions['tools'] }, ) { if (!sendingMessage && !options.attachments?.length) @@ -208,6 +209,7 @@ export const useChatStore = defineStore('chat', () => { await stream(options.model, options.chatProvider, newMessages as Message[], { headers, + tools: options.tools, onStreamEvent: async (event: StreamEvent) => { switch (event.type) { case 'tool-call': diff --git a/packages/stage-ui/src/stores/llm.ts b/packages/stage-ui/src/stores/llm.ts index 98d07f797..9c0d55c5d 100644 --- a/packages/stage-ui/src/stores/llm.ts +++ b/packages/stage-ui/src/stores/llm.ts @@ -1,5 +1,5 @@ import type { ChatProvider } from '@xsai-ext/shared-providers' -import type { CommonContentPart, CompletionToolCall, Message } from '@xsai/shared-chat' +import type { CommonContentPart, CompletionToolCall, Message, Tool } from '@xsai/shared-chat' import { listModels } from '@xsai/model' import { XSAIError } from '@xsai/shared' @@ -21,6 +21,7 @@ export interface StreamOptions { onStreamEvent?: (event: StreamEvent) => void | Promise toolsCompatibility?: Map supportsTools?: boolean + tools?: Tool[] | (() => Promise) } // TODO: proper format for other error messages. @@ -36,27 +37,36 @@ function sanitizeMessages(messages: unknown[]): Message[] { }) } -function streamOptionsToolsCompatibilityOk(model: string, chatProvider: ChatProvider, _: Message[], options?: StreamOptions, toolsCompatibility: Map = new Map()): boolean { - return !!(options?.supportsTools || toolsCompatibility.get(`${chatProvider.chat(model).baseURL}-${model}`)) +function streamOptionsToolsCompatibilityOk(model: string, chatProvider: ChatProvider, _: Message[], options?: StreamOptions): boolean { + return !!(options?.supportsTools || options?.toolsCompatibility?.get(`${chatProvider.chat(model).baseURL}-${model}`)) } async function streamFrom(model: string, chatProvider: ChatProvider, messages: Message[], options?: StreamOptions) { const headers = options?.headers const sanitized = sanitizeMessages(messages as unknown[]) + const resolveTools = async () => { + const tools = typeof options?.tools === 'function' + ? await options.tools() + : options?.tools + return tools ?? [] + } return new Promise(async (resolve, reject) => { try { + const supportedTools = streamOptionsToolsCompatibilityOk(model, chatProvider, messages, options) + await streamText({ ...chatProvider.chat(model), maxSteps: 10, messages: sanitized, headers, // TODO: we need Automatic tools discovery - tools: streamOptionsToolsCompatibilityOk(model, chatProvider, messages, options) + tools: supportedTools ? [ ...await mcp(), ...await debug(), + ...await resolveTools(), ] : undefined, async onEvent(event) { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 986351cb9..b5afedb29 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -365,6 +365,9 @@ importers: '@xsai/stream-transcription': specifier: 0.4.0-beta.8 version: 0.4.0-beta.8 + '@xsai/tool': + specifier: 'catalog:' + version: 0.4.0-beta.9(zod-to-json-schema@3.24.6(zod@4.1.12))(zod@4.1.12) '@xsai/utils-chat': specifier: 'catalog:' version: 0.4.0-beta.5