nanakura/moeka-project
1
0
Fork 0
Code Issues Pull Requests 1 Actions 1 Packages Projects Releases Wiki Activity

feat(stage-tamagotchi): enable widgets calling as tool

Browse Source
This commit is contained in:
Neko Ayaka
2025-11-30 21:37:19 +08:00
parent ac9a01a41f
commit 27b39900cf
8 changed files with 336 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/stage-tamagotchi/package.json
+1
View File
@@ -65,6 +65,7 @@
"@xsai/shared-chat": "catalog:",
"@xsai/stream-text": "catalog:",
"@xsai/stream-transcription": "0.4.0-beta.8",
"@xsai/tool": "catalog:",
"@xsai/utils-chat": "catalog:",
"animejs": "^4.2.2",
"colorjs.io": "^0.5.2",
apps/stage-tamagotchi/src/renderer/components/InteractiveArea.vue
+10 -4
View File
@@ -13,6 +13,8 @@ import { useI18n } from 'vue-i18n'
import TamagotchiChatHistory from './ChatHistory.vue'
import { widgetsTools } from '../stores/tools/builtin/widgets'
const messageInput = ref('')
const listening = ref(false)
const attachments = ref<{ type: 'image', data: string, mimeType: string, url: string }[]>([])
@@ -27,6 +29,8 @@ const { activeModel, activeProvider } = storeToRefs(useConsciousnessStore())
const isComposing = ref(false)
async function handleSend() {
debugger
if (isComposing.value) {
return
}
@@ -43,6 +47,7 @@ async function handleSend() {
chatProvider: await providersStore.getProviderInstance<ChatProvider>(activeProvider.value),
providerConfig,
attachments: attachmentsToSend,
tools: widgetsTools,
})
// clear after sending
@@ -132,7 +137,7 @@ watch([activeProvider, activeModel], async () => {
if (activeProvider.value && activeModel.value) {
await discoverToolsCompatibility(activeModel.value, await providersStore.getProviderInstance<ChatProvider>(activeProvider.value), [])
}
})
}, { immediate: true })
onAfterMessageComposed(async () => {
messageInput.value = ''
@@ -170,9 +175,10 @@ onAfterMessageComposed(async () => {
<BasicTextarea
v-model="messageInput"
:placeholder="t('stage.message')"
border="solid 2 primary-100"
text="primary-400 hover:primary-600 placeholder:primary-400 placeholder:hover:primary-600"
bg="primary-50 dark:primary-100" max-h="[10lh]" min-h="[1lh]"
text="primary-600 placeholder:primary-600"
border="solid 2 primary-200/20 dark:primary-400/20"
bg="primary-100/50 dark:primary-900/70"
max-h="[10lh]" min-h="[1lh]"
w-full shrink-0 resize-none overflow-y-scroll rounded-xl p-2 font-medium outline-none
transition="all duration-250 ease-in-out placeholder:all placeholder:duration-250 placeholder:ease-in-out"
@compositionstart="isComposing = true"
apps/stage-tamagotchi/src/renderer/stores/tools/builtin/widgets.test.ts
+121
View File
@@ -0,0 +1,121 @@
import type { WidgetInvokers } from './widgets'
import { describe, expect, it, vi } from 'vitest'
import { executeWidgetAction, normalizeComponentProps } from './widgets'
describe('widgets tool helpers', () => {
describe('normalizeComponentProps', () => {
it('parses JSON strings into objects', () => {
const result = normalizeComponentProps('{"city":"Tokyo","temp":15}')
expect(result).toEqual({ city: 'Tokyo', temp: 15 })
})
it('returns empty object for empty or undefined', () => {
expect(normalizeComponentProps(' ')).toEqual({})
expect(normalizeComponentProps(undefined)).toEqual({})
expect(normalizeComponentProps(null as any)).toEqual({})
})
it('passes through object inputs', () => {
const payload = { foo: 'bar', nested: { a: 1 } }
expect(normalizeComponentProps(payload)).toBe(payload)
})
it('throws on invalid JSON', () => {
expect(() => normalizeComponentProps('{ bad json ')).toThrow()
})
})
describe('executeWidgetAction with mocked invokers', () => {
const makeInvokers = (): WidgetInvokers => ({
prepareWindow: vi.fn(),
openWindow: vi.fn(),
addWidget: vi.fn(),
updateWidget: vi.fn(),
removeWidget: vi.fn(),
clearWidgets: vi.fn(),
})
it('spawns with ttl conversion and parsed props', async () => {
const invokers = makeInvokers()
vi.mocked(invokers.addWidget).mockResolvedValue('abc123')
const result = await executeWidgetAction({
action: 'spawn',
id: ' abc123 ',
componentName: 'weather',
componentProps: '{"city":"Tokyo"}',
size: 'm',
ttlSeconds: 2,
}, { invokers })
expect(result).toContain('abc123')
expect(invokers.addWidget).toHaveBeenCalledTimes(1)
expect(invokers.addWidget).toHaveBeenCalledWith({
id: 'abc123',
componentName: 'weather',
componentProps: { city: 'Tokyo' },
size: 'm',
ttlMs: 2000,
})
})
it('updates props and trims id', async () => {
const invokers = makeInvokers()
await executeWidgetAction({
action: 'update',
id: ' xyz ',
componentName: '',
componentProps: '{"foo":1}',
size: 'm',
ttlSeconds: 0,
}, { invokers })
expect(invokers.updateWidget).toHaveBeenCalledWith({ id: 'xyz', componentProps: { foo: 1 } })
})
it('removes when id provided', async () => {
const invokers = makeInvokers()
await executeWidgetAction({
action: 'remove',
id: 'rem-id',
componentName: '',
componentProps: '{}',
size: 's',
ttlSeconds: 0,
}, { invokers })
expect(invokers.removeWidget).toHaveBeenCalledWith({ id: 'rem-id' })
})
it('opens window with prepared id', async () => {
const invokers = makeInvokers()
vi.mocked(invokers.prepareWindow).mockResolvedValue('prepared-id')
await executeWidgetAction({
action: 'open',
id: ' prepared-id ',
componentName: '',
componentProps: '{}',
size: 'l',
ttlSeconds: 0,
}, { invokers })
expect(invokers.prepareWindow).toHaveBeenCalledWith({ id: 'prepared-id' })
expect(invokers.openWindow).toHaveBeenCalledWith({ id: 'prepared-id' })
})
it('clears widgets', async () => {
const invokers = makeInvokers()
await executeWidgetAction({
action: 'clear',
id: '',
componentName: '',
componentProps: '{}',
size: 'm',
ttlSeconds: 0,
}, { invokers })
expect(invokers.clearWidgets).toHaveBeenCalledTimes(1)
})
})
})
apps/stage-tamagotchi/src/renderer/stores/tools/builtin/widgets.ts
+176
View File
@@ -0,0 +1,176 @@
import { defineInvoke } from '@moeru/eventa'
import { createContext } from '@moeru/eventa/adapters/electron/renderer'
import { tool } from '@xsai/tool'
import { z } from 'zod'
import { widgetsAdd, widgetsClear, widgetsOpenWindow, widgetsPrepareWindow, widgetsRemove, widgetsUpdate } from '../../../../shared/eventa'
type SizePreset = 's' | 'm' | 'l'
type WidgetActionInput
= | {
action: 'spawn'
id: string
componentName: string
componentProps: string | Record<string, any>
size: SizePreset
ttlSeconds: number
}
| {
action: 'update'
id: string
componentProps: string | Record<string, any>
componentName?: string
size?: SizePreset
ttlSeconds?: number
}
| {
action: 'remove'
id: string
componentName?: string
componentProps?: string | Record<string, any>
size?: SizePreset
ttlSeconds?: number
}
| {
action: 'clear'
id: string
componentName?: string
componentProps?: string | Record<string, any>
size?: SizePreset
ttlSeconds?: number
}
| {
action: 'open'
id: string
componentName?: string
componentProps?: string | Record<string, any>
size?: SizePreset
ttlSeconds?: number
}
export type WidgetInvokers = ReturnType<typeof createInvokers>
let cachedInvokers: WidgetInvokers | undefined
function createInvokers() {
const ipcRenderer = typeof window !== 'undefined' ? (window as any)?.electron?.ipcRenderer : undefined
if (!ipcRenderer)
throw new Error('Widget tools are only available in the desktop app.')
const { context } = createContext(ipcRenderer)
return {
prepareWindow: defineInvoke(context, widgetsPrepareWindow),
openWindow: defineInvoke(context, widgetsOpenWindow),
addWidget: defineInvoke(context, widgetsAdd),
updateWidget: defineInvoke(context, widgetsUpdate),
removeWidget: defineInvoke(context, widgetsRemove),
clearWidgets: defineInvoke(context, widgetsClear),
}
}
function resolveInvokers(override?: WidgetInvokers): WidgetInvokers {
if (override)
return override
if (!cachedInvokers)
cachedInvokers = createInvokers()
return cachedInvokers
}
const widgetParams = z.object({
action: z.enum(['spawn', 'update', 'remove', 'clear', 'open']).describe('Choose one: spawn, update, remove, clear, open'),
id: z.string().default('').describe('Widget id; required for update/remove, optional for spawn/open'),
componentName: z.string().default('').describe('Widget component to render, e.g. weather (required for spawn)'),
componentProps: z.string().default('{}').describe('Widget props as JSON string (e.g. {"city":"Tokyo"})'),
size: z.enum(['s', 'm', 'l']).default('m'),
ttlSeconds: z.number().int().nonnegative().default(0).describe('Auto-close timer in seconds (spawn only)'),
}).strict()
export function normalizeComponentProps(raw?: string | Record<string, any>) {
if (raw === undefined || raw === null)
return {}
if (typeof raw === 'string') {
const payload = raw.trim()
if (!payload)
return {}
try {
const parsed = JSON.parse(payload)
return typeof parsed === 'object' && parsed !== null ? parsed : {}
}
catch (error) {
throw new Error(`Invalid JSON for componentProps: ${(error as Error).message}`)
}
}
if (typeof raw === 'object')
return raw
return {}
}
export async function executeWidgetAction(input: WidgetActionInput, deps?: { invokers?: WidgetInvokers }) {
const invokers = resolveInvokers(deps?.invokers)
const normalizedId = input.id?.trim() || undefined
switch (input.action) {
case 'spawn': {
if (!input.componentName?.trim())
throw new Error('componentName is required to spawn a widget.')
const componentProps = normalizeComponentProps(input.componentProps)
const ttlMs = input.ttlSeconds ? Math.floor(input.ttlSeconds * 1000) : 0
const id = await invokers.addWidget({
id: normalizedId,
componentName: input.componentName,
componentProps,
size: input.size ?? 'm',
ttlMs,
})
return `Spawned widget${id ? ` (${id})` : ''}.`
}
case 'update': {
if (!normalizedId)
throw new Error('id is required to update a widget.')
const componentProps = normalizeComponentProps(input.componentProps)
await invokers.updateWidget({
id: normalizedId,
componentProps,
})
return `Updated widget (${normalizedId}).`
}
case 'remove': {
if (!normalizedId)
throw new Error('id is required to remove a widget.')
await invokers.removeWidget({ id: normalizedId })
return `Removed widget (${normalizedId}).`
}
case 'clear': {
await invokers.clearWidgets()
return 'Cleared all widgets.'
}
case 'open': {
const id = await invokers.prepareWindow(normalizedId ? { id: normalizedId } : {})
await invokers.openWindow(normalizedId ? { id: normalizedId } : {})
return `Opened widget window${id ? ` (${id})` : ''}.`
}
default:
return 'No action performed.'
}
}
const tools = [
tool({
name: 'stage_widgets',
description: 'Manage overlay widgets in the Stage desktop app (spawn, update, remove, clear, or open the widgets window).',
execute: params => executeWidgetAction(params as WidgetActionInput),
parameters: widgetParams,
}),
]
export const widgetsTools = async () => Promise.all(tools)
apps/stage-tamagotchi/vitest.config.ts
+8
View File
@@ -0,0 +1,8 @@
import { defineConfig } from 'vitest/config'
export default defineConfig({
test: {
include: ['src/**/*.test.ts'],
exclude: ['**/node_modules/**', '**/.git/**'],
},
})
packages/stage-ui/src/stores/chat.ts
+3 -1
View File
@@ -1,7 +1,7 @@
import type { ChatProvider } from '@xsai-ext/shared-providers'
import type { CommonContentPart, Message, SystemMessage } from '@xsai/shared-chat'
import type { StreamEvent } from '../stores/llm'
import type { StreamEvent, StreamOptions } from '../stores/llm'
import type { ChatAssistantMessage, ChatMessage, ChatSlices } from '../types/chat'
import { useLocalStorage } from '@vueuse/core'
@@ -112,6 +112,7 @@ export const useChatStore = defineStore('chat', () => {
chatProvider: ChatProvider
providerConfig?: Record<string, unknown>
attachments?: { type: 'image', data: string, mimeType: string }[]
tools?: StreamOptions['tools']
},
) {
if (!sendingMessage && !options.attachments?.length)
@@ -208,6 +209,7 @@ export const useChatStore = defineStore('chat', () => {
await stream(options.model, options.chatProvider, newMessages as Message[], {
headers,
tools: options.tools,
onStreamEvent: async (event: StreamEvent) => {
switch (event.type) {
case 'tool-call':
packages/stage-ui/src/stores/llm.ts
+14 -4
View File
@@ -1,5 +1,5 @@
import type { ChatProvider } from '@xsai-ext/shared-providers'
import type { CommonContentPart, CompletionToolCall, Message } from '@xsai/shared-chat'
import type { CommonContentPart, CompletionToolCall, Message, Tool } from '@xsai/shared-chat'
import { listModels } from '@xsai/model'
import { XSAIError } from '@xsai/shared'
@@ -21,6 +21,7 @@ export interface StreamOptions {
onStreamEvent?: (event: StreamEvent) => void | Promise<void>
toolsCompatibility?: Map<string, boolean>
supportsTools?: boolean
tools?: Tool[] | (() => Promise<Tool[] | undefined>)
}
// TODO: proper format for other error messages.
@@ -36,27 +37,36 @@ function sanitizeMessages(messages: unknown[]): Message[] {
})
}
function streamOptionsToolsCompatibilityOk(model: string, chatProvider: ChatProvider, _: Message[], options?: StreamOptions, toolsCompatibility: Map<string, boolean> = new Map()): boolean {
return !!(options?.supportsTools || toolsCompatibility.get(`${chatProvider.chat(model).baseURL}-${model}`))
function streamOptionsToolsCompatibilityOk(model: string, chatProvider: ChatProvider, _: Message[], options?: StreamOptions): boolean {
return !!(options?.supportsTools || options?.toolsCompatibility?.get(`${chatProvider.chat(model).baseURL}-${model}`))
}
async function streamFrom(model: string, chatProvider: ChatProvider, messages: Message[], options?: StreamOptions) {
const headers = options?.headers
const sanitized = sanitizeMessages(messages as unknown[])
const resolveTools = async () => {
const tools = typeof options?.tools === 'function'
? await options.tools()
: options?.tools
return tools ?? []
}
return new Promise<void>(async (resolve, reject) => {
try {
const supportedTools = streamOptionsToolsCompatibilityOk(model, chatProvider, messages, options)
await streamText({
...chatProvider.chat(model),
maxSteps: 10,
messages: sanitized,
headers,
// TODO: we need Automatic tools discovery
tools: streamOptionsToolsCompatibilityOk(model, chatProvider, messages, options)
tools: supportedTools
? [
...await mcp(),
...await debug(),
...await resolveTools(),
]
: undefined,
async onEvent(event) {
pnpm-lock.yaml
Generated
+3
View File
@@ -365,6 +365,9 @@ importers:
'@xsai/stream-transcription':
specifier: 0.4.0-beta.8
version: 0.4.0-beta.8
'@xsai/tool':
specifier: 'catalog:'
version: 0.4.0-beta.9(zod-to-json-schema@3.24.6(zod@4.1.12))(zod@4.1.12)
'@xsai/utils-chat':
specifier: 'catalog:'
version: 0.4.0-beta.5