diff --git a/apps/stage-tamagotchi/src/renderer/components/InteractiveArea.vue b/apps/stage-tamagotchi/src/renderer/components/InteractiveArea.vue index fa9e7f9ef..023b3f389 100644 --- a/apps/stage-tamagotchi/src/renderer/components/InteractiveArea.vue +++ b/apps/stage-tamagotchi/src/renderer/components/InteractiveArea.vue @@ -11,7 +11,7 @@ import { useConsciousnessStore } from '@proj-airi/stage-ui/stores/modules/consci import { useProvidersStore } from '@proj-airi/stage-ui/stores/providers' import { BasicTextarea } from '@proj-airi/ui' import { storeToRefs } from 'pinia' -import { computed, ref, watch } from 'vue' +import { computed, ref } from 'vue' import { useI18n } from 'vue-i18n' import { widgetsTools } from '../stores/tools/builtin/widgets' @@ -23,7 +23,7 @@ const chatOrchestrator = useChatOrchestratorStore() const chatSession = useChatSessionStore() const chatStream = useChatStreamStore() const { cleanupMessages } = useChatMaintenanceStore() -const { ingest, onAfterMessageComposed, discoverToolsCompatibility } = chatOrchestrator +const { ingest, onAfterMessageComposed } = chatOrchestrator const { messages } = storeToRefs(chatSession) const { streamingMessage } = storeToRefs(chatStream) const { sending } = storeToRefs(chatOrchestrator) @@ -103,12 +103,6 @@ function removeAttachment(index: number) { } } -watch([activeProvider, activeModel], async () => { - if (activeProvider.value && activeModel.value) { - await discoverToolsCompatibility(activeModel.value, await providersStore.getProviderInstance(activeProvider.value), []) - } -}, { immediate: true }) - onAfterMessageComposed(async () => { messageInput.value = '' attachments.value.forEach(att => URL.revokeObjectURL(att.url)) diff --git a/packages/stage-layouts/src/components/Layouts/MobileInteractiveArea.vue b/packages/stage-layouts/src/components/Layouts/MobileInteractiveArea.vue index cf94965dc..688561beb 100644 --- a/packages/stage-layouts/src/components/Layouts/MobileInteractiveArea.vue +++ b/packages/stage-layouts/src/components/Layouts/MobileInteractiveArea.vue @@ -52,7 +52,7 @@ useResizeObserver(document.documentElement, () => screenSafeArea.update()) const { themeColorsHueDynamic, stageViewControlsEnabled } = storeToRefs(useSettings()) const settingsAudioDevice = useSettingsAudioDevice() const { enabled, selectedAudioInput, stream, audioInputs } = storeToRefs(settingsAudioDevice) -const { ingest, onAfterMessageComposed, discoverToolsCompatibility } = chatOrchestrator +const { ingest, onAfterMessageComposed } = chatOrchestrator const { t } = useI18n() const { audioContext } = useAudioContext() const { startAnalyzer, stopAnalyzer, volumeLevel } = useAudioAnalyzer() @@ -130,12 +130,6 @@ watch(hearingDialogOpen, (value) => { onAfterMessageComposed(async () => { }) -watch([activeProvider, activeModel], async () => { - if (activeProvider.value && activeModel.value) { - await discoverToolsCompatibility(activeModel.value, await providersStore.getProviderInstance(activeProvider.value), []) - } -}, { immediate: true }) - onUnmounted(() => { teardownAnalyzer() }) diff --git a/packages/stage-layouts/src/components/Widgets/ChatArea.vue b/packages/stage-layouts/src/components/Widgets/ChatArea.vue index a587cd021..0e0e24621 100644 --- a/packages/stage-layouts/src/components/Widgets/ChatArea.vue +++ b/packages/stage-layouts/src/components/Widgets/ChatArea.vue @@ -32,7 +32,7 @@ const { askPermission, startStream } = useSettingsAudioDevice() const { enabled, selectedAudioInput, stream, audioInputs } = storeToRefs(useSettingsAudioDevice()) const chatOrchestrator = useChatOrchestratorStore() const chatSession = useChatSessionStore() -const { ingest, onAfterMessageComposed, discoverToolsCompatibility } = chatOrchestrator +const { ingest, onAfterMessageComposed } = chatOrchestrator const { messages } = storeToRefs(chatSession) const { audioContext } = useAudioContext() const { t } = useI18n() @@ -134,12 +134,6 @@ watch(hearingPopoverOpen, async (value) => { } }) -watch([activeProvider, activeModel], async () => { - if (activeProvider.value && activeModel.value) { - await discoverToolsCompatibility(activeModel.value, await providersStore.getProviderInstance(activeProvider.value), []) - } -}, { immediate: true }) - onAfterMessageComposed(async () => { }) diff --git a/packages/stage-ui/src/stores/chat.ts b/packages/stage-ui/src/stores/chat.ts index 310880dec..1bc740252 100644 --- a/packages/stage-ui/src/stores/chat.ts +++ b/packages/stage-ui/src/stores/chat.ts @@ -413,8 +413,6 @@ export const useChatOrchestratorStore = defineStore('chat-orchestrator', () => { return { sending, - discoverToolsCompatibility: llmStore.discoverToolsCompatibility, - ingest, ingestOnFork, cancelPendingSends, diff --git a/packages/stage-ui/src/stores/llm.test.ts b/packages/stage-ui/src/stores/llm.test.ts index d7754c615..52a1116af 100644 --- a/packages/stage-ui/src/stores/llm.test.ts +++ b/packages/stage-ui/src/stores/llm.test.ts @@ -1,37 +1,49 @@ -import { env } from 'node:process' - -import { createOpenRouter } from '@xsai-ext/providers/create' import { describe, expect, it } from 'vitest' -import { attemptForToolsCompatibilityDiscovery } from './llm' +import { isToolRelatedError } from './llm' -function doesHaveOpenRouterApiKey() { - const apiKey = env.LLM_API_OPENROUTER_API_KEY - if (!apiKey) { - console.warn('Skipping llm store tests, because LLM_API_OPENROUTER_API_KEY is not set') +describe('isToolRelatedError', () => { + const positives: [provider: string, msg: string][] = [ + ['ollama', 'llama3 does not support tools'], + ['ollama', 'phi does not support tools'], + ['openrouter', 'No endpoints found that support tool use'], + ['openai-compatible', 'Invalid schema for function \'myFunc\': \'dict\' is not valid under any of the given schemas'], + ['openai-compatible', 'invalid_function_parameters'], + ['openai-compatible', 'invalid function parameters'], + ['azure', 'Functions are not supported at this time'], + ['azure', 'Unrecognized request argument supplied: tools'], + ['azure', 'Unrecognized request arguments supplied: tool_choice, tools'], + ['google', 'Tool use with function calling is unsupported'], + ['groq', 'tool_use_failed'], + ['groq', 'Error code: tool_use_failed - Failed to call a function'], + ['anthropic', 'This model does not support function calling'], + ['anthropic', 'does not support function_calling'], + ['cloudflare', 'tools is not supported'], + ['cloudflare', 'tool is not supported for this model'], + ['cloudflare', 'tools are not supported'], + ] + + const negatives = [ + 'network error', + 'timeout', + 'rate limit exceeded', + 'invalid api key', + 'model not found', + 'context length exceeded', + '', + ] + + for (const [provider, msg] of positives) { + it(`matches [${provider}]: "${msg}"`, () => { + expect(isToolRelatedError(msg)).toBe(true) + expect(isToolRelatedError(new Error(msg))).toBe(true) + }) } - return !!apiKey -} - -const hasOpenRouterApiKey = doesHaveOpenRouterApiKey() - -describe.skipIf(!hasOpenRouterApiKey)('llm store', { timeout: 60000 }, async () => { - it('should be false for phi-4', async () => { - // TODO: base url should not be hardcoded, wait for https://github.com/moeru-ai/xsai/pull/194 - const res1 = await attemptForToolsCompatibilityDiscovery('microsoft/phi-4', createOpenRouter(env.LLM_API_OPENROUTER_API_KEY!, 'https://openrouter.ai/api/v1/'), []) - expect(res1).toBe(false) - }) - - it('should be false for gpt-4o-mini', async () => { - // TODO: base url should not be hardcoded, wait for https://github.com/moeru-ai/xsai/pull/194 - const res1 = await attemptForToolsCompatibilityDiscovery('openai/gpt-4o-mini', createOpenRouter(env.LLM_API_OPENROUTER_API_KEY!, 'https://openrouter.ai/api/v1/'), []) - expect(res1).toBe(false) - }) - - it('should be true for gpt-4o', async () => { - // TODO: base url should not be hardcoded, wait for https://github.com/moeru-ai/xsai/pull/194 - const res2 = await attemptForToolsCompatibilityDiscovery('openai/gpt-4o', createOpenRouter(env.LLM_API_OPENROUTER_API_KEY!, 'https://openrouter.ai/api/v1/'), []) - expect(res2).toBe(true) - }) + for (const msg of negatives) { + it(`rejects: "${msg}"`, () => { + expect(isToolRelatedError(msg)).toBe(false) + expect(isToolRelatedError(new Error(msg))).toBe(false) + }) + } }) diff --git a/packages/stage-ui/src/stores/llm.ts b/packages/stage-ui/src/stores/llm.ts index 7902ae893..b70696670 100644 --- a/packages/stage-ui/src/stores/llm.ts +++ b/packages/stage-ui/src/stores/llm.ts @@ -2,7 +2,6 @@ import type { ChatProvider } from '@xsai-ext/providers/utils' import type { CommonContentPart, CompletionToolCall, Message, Tool } from '@xsai/shared-chat' import { listModels } from '@xsai/model' -import { XSAIError } from '@xsai/shared' import { streamText } from '@xsai/stream-text' import { defineStore } from 'pinia' import { ref } from 'vue' @@ -22,11 +21,10 @@ export interface StreamOptions { onStreamEvent?: (event: StreamEvent) => void | Promise toolsCompatibility?: Map supportsTools?: boolean - waitForTools?: boolean // when true,won't resolve on finishReason=='tool_calls'; + waitForTools?: boolean tools?: Tool[] | (() => Promise) } -// TODO: proper format for other error messages. function sanitizeMessages(messages: unknown[]): Message[] { return messages.map((m: any) => { if (m && m.role === 'error') { @@ -35,10 +33,8 @@ function sanitizeMessages(messages: unknown[]): Message[] { content: `User encountered error: ${String(m.content ?? '')}`, } as Message } - // NOTICE: This block is critical for backward compatibility with LLM providers (e.g., DeepSeek) - // that expect message content to be a string, not an array of content parts. - // Failure to flatten array content (when no image_url is present) can lead to - // deserialization errors like "invalid type: sequence, expected a string". + // NOTICE: Flatten array content for providers (e.g. DeepSeek) that expect string, + // not content-part arrays. Skipped when image_url parts are present. if (m && Array.isArray(m.content)) { const contentParts = m.content as { type?: string, text?: string }[] if (!contentParts.some(p => p?.type === 'image_url')) { @@ -50,14 +46,16 @@ function sanitizeMessages(messages: unknown[]): Message[] { } function streamOptionsToolsCompatibilityOk(model: string, chatProvider: ChatProvider, _: Message[], options?: StreamOptions): boolean { - return !!(options?.supportsTools || options?.toolsCompatibility?.get(`${chatProvider.chat(model).baseURL}-${model}`)) + if (options?.supportsTools) + return true + const key = `${chatProvider.chat(model).baseURL}-${model}` + return options?.toolsCompatibility?.get(key) !== false } async function streamFrom(model: string, chatProvider: ChatProvider, messages: Message[], options?: StreamOptions) { - const headers = options?.headers const chatConfig = chatProvider.chat(model) - const sanitized = sanitizeMessages(messages as unknown[]) + const resolveTools = async () => { const tools = typeof options?.tools === 'function' ? await options.tools() @@ -98,8 +96,7 @@ async function streamFrom(model: string, chatProvider: ChatProvider, messages: M resolveOnce() } else if (event && (event as StreamEvent).type === 'error') { - const error = (event as any).error ?? new Error('Stream error') - rejectOnce(error) + rejectOnce((event as any).error ?? new Error('Stream error')) } } catch (err) { @@ -113,30 +110,20 @@ async function streamFrom(model: string, chatProvider: ChatProvider, messages: M abortSignal: options?.abortSignal, maxSteps: 10, messages: sanitized, - headers, - // TODO: we need Automatic tools discovery + headers: options?.headers, tools, onEvent, }) - // NOTICE: @xsai/stream-text rejects its result promises when the SSE parser - // encounters provider-side errors such as `{"error":{"message":"failed to process image"}}`. - // This wrapper resolves/rejects via `onEvent`, so we must also consume the - // underlying promises to prevent those internal rejections from surfacing as - // unhandled promise errors and leaving the app in a faulted state. + // NOTICE: Consume underlying promises to prevent unhandled rejections from + // @xsai/stream-text's SSE parser surfacing as faulted app state. void streamResult.steps.catch((err) => { rejectOnce(err) console.error('Stream steps error:', err) }) - void streamResult.messages.catch((err) => { - console.error('Stream messages error:', err) - }) - void streamResult.usage.catch((err) => { - console.error('Stream usage error:', err) - }) - void streamResult.totalUsage.catch((err) => { - console.error('Stream totalUsage error:', err) - }) + void streamResult.messages.catch(err => console.error('Stream messages error:', err)) + void streamResult.usage.catch(err => console.error('Stream usage error:', err)) + void streamResult.totalUsage.catch(err => console.error('Stream totalUsage error:', err)) } catch (err) { rejectOnce(err) @@ -144,97 +131,49 @@ async function streamFrom(model: string, chatProvider: ChatProvider, messages: M }) } -export async function attemptForToolsCompatibilityDiscovery(model: string, chatProvider: ChatProvider, _: Message[], options?: Omit): Promise { - async function attempt(enable: boolean) { - try { - await streamFrom(model, chatProvider, [{ role: 'user', content: 'Hello, world!' }], { ...options, supportsTools: enable }) - return true - } - catch (err) { - if (err instanceof Error && err.name === new XSAIError('').name) { - // TODO: if you encountered many more errors like these, please, add them here. +// Runtime auto-degrade: patterns that indicate the model/provider does not support tool calling. +const TOOLS_RELATED_ERROR_PATTERNS: RegExp[] = [ + /does not support tools/i, // Ollama + /no endpoints found that support tool use/i, // OpenRouter + /invalid schema for function/i, // OpenAI-compatible + /invalid.?function.?parameters/i, // OpenAI-compatible + /functions are not supported/i, // Azure AI Foundry + /unrecognized request argument.+tools/i, // Azure AI Foundry + /tool use with function calling is unsupported/i, // Google Generative AI + /tool_use_failed/i, // Groq + /does not support function.?calling/i, // Anthropic + /tools?\s+(is|are)\s+not\s+supported/i, // Cloudflare Workers AI +] - // Ollama - /** - * {"error":{"message":"registry.ollama.ai// does not support tools","type":"api_error","param":null,"code":null}} - */ - if (String(err).includes('does not support tools')) { - return false - } - // OpenRouter - /** - * {"error":{"message":"No endpoints found that support tool use. To learn more about provider routing, visit: https://openrouter.ai/docs/provider-routing","code":404}} - */ - if (String(err).includes('No endpoints found that support tool use.')) { - return false - } - } - - throw err - } - } - - function promiseAllWithInterval(promises: (() => Promise)[], interval: number): Promise<{ result?: T, error?: any }[]> { - return new Promise((resolve) => { - const results: { result?: T, error?: any }[] = [] - let completed = 0 - - promises.forEach((promiseFn, index) => { - setTimeout(() => { - promiseFn() - .then((result) => { - results[index] = { result } - }) - .catch((err) => { - results[index] = { error: err } - }) - .finally(() => { - completed++ - if (completed === promises.length) { - resolve(results) - } - }) - }, index * interval) - }) - }) - } - - const attempts = [ - () => attempt(true), - () => attempt(false), - ] - - const attemptsResults = await promiseAllWithInterval(attempts, 1000) - if (attemptsResults.some(res => res.error)) { - const err = new Error(`Error during tools compatibility discovery for model: ${model}. Errors: ${attemptsResults.map(res => res.error).filter(Boolean).join(', ')}`) - err.cause = attemptsResults.map(res => res.error).filter(Boolean) - throw err - } - - return attemptsResults[0].result === true && attemptsResults[1].result === true +export function isToolRelatedError(err: unknown): boolean { + const msg = String(err) + return TOOLS_RELATED_ERROR_PATTERNS.some(p => p.test(msg)) } export const useLLM = defineStore('llm', () => { const toolsCompatibility = ref>(new Map()) - async function discoverToolsCompatibility(model: string, chatProvider: ChatProvider, _: Message[], options?: Omit) { - // Cached, no need to discover again - if (toolsCompatibility.value.has(`${chatProvider.chat(model).baseURL}-${model}`)) { - return - } - - const res = await attemptForToolsCompatibilityDiscovery(model, chatProvider, _, { ...options, toolsCompatibility: toolsCompatibility.value }) - toolsCompatibility.value.set(`${chatProvider.chat(model).baseURL}-${model}`, res) + function modelKey(model: string, chatProvider: ChatProvider): string { + return `${chatProvider.chat(model).baseURL}-${model}` } - function stream(model: string, chatProvider: ChatProvider, messages: Message[], options?: StreamOptions) { - return streamFrom(model, chatProvider, messages, { ...options, toolsCompatibility: toolsCompatibility.value }) + async function stream(model: string, chatProvider: ChatProvider, messages: Message[], options?: StreamOptions) { + const key = modelKey(model, chatProvider) + try { + await streamFrom(model, chatProvider, messages, { ...options, toolsCompatibility: toolsCompatibility.value }) + } + catch (err) { + if (isToolRelatedError(err)) { + console.warn(`[llm] Auto-disabling tools for "${key}" due to tool-related error`) + toolsCompatibility.value.set(key, false) + } + throw err + } } async function models(apiUrl: string, apiKey: string) { - if (apiUrl === '') { + if (apiUrl === '') return [] - } try { return await listModels({ @@ -243,10 +182,8 @@ export const useLLM = defineStore('llm', () => { }) } catch (err) { - if (String(err).includes(`Failed to construct 'URL': Invalid URL`)) { + if (String(err).includes(`Failed to construct 'URL': Invalid URL`)) return [] - } - throw err } } @@ -254,6 +191,5 @@ export const useLLM = defineStore('llm', () => { return { models, stream, - discoverToolsCompatibility, } }) diff --git a/packages/stage-ui/src/tools/mcp.test.ts b/packages/stage-ui/src/tools/mcp.test.ts index c1906c3e0..26b3ce694 100644 --- a/packages/stage-ui/src/tools/mcp.test.ts +++ b/packages/stage-ui/src/tools/mcp.test.ts @@ -7,25 +7,20 @@ import { mcp } from './mcp' describe('tools mcp schema', () => { it('emits strict parameter objects', async () => { const tools = await mcp() - const toolNames = [ - 'mcp_list_tools', - 'mcp_call_tool', - ] - - for (const name of toolNames) { - const tool = tools.find(entry => entry.function.name === name) - expect(tool, `missing tool: ${name}`).toBeDefined() - expect(tool?.function.parameters.additionalProperties).toBe(false) + for (const name of ['mcp_list_tools', 'mcp_call_tool']) { + const t = tools.find(entry => entry.function.name === name) + expect(t, `missing tool: ${name}`).toBeDefined() + expect(t?.function.parameters.additionalProperties).toBe(false) } }) - it('keeps mcp_call_tool parameters items strict', async () => { + it('mcp_call_tool uses flat name+arguments schema', async () => { const tools = await mcp() const callTool = tools.find(entry => entry.function.name === 'mcp_call_tool') - expect(callTool).toBeDefined() - const items = ((callTool!.function.parameters as JsonSchema).properties?.parameters as JsonSchema)?.items as JsonSchema - expect(items).toBeDefined() - expect(items.additionalProperties).toBe(false) + + const props = (callTool!.function.parameters as JsonSchema).properties! + expect((props.name as JsonSchema).type).toBe('string') + expect((props.arguments as JsonSchema).type).toBe('string') }) }) diff --git a/packages/stage-ui/src/tools/mcp.ts b/packages/stage-ui/src/tools/mcp.ts index 8f6e437de..3bbe54e37 100644 --- a/packages/stage-ui/src/tools/mcp.ts +++ b/packages/stage-ui/src/tools/mcp.ts @@ -6,8 +6,8 @@ import { getMcpToolBridge } from '../stores/mcp-tool-bridge' const tools = [ tool({ name: 'mcp_list_tools', - description: 'List all tools available on the connected MCP servers', - execute: async (_, __) => { + description: 'List all available MCP tools. Call this first to discover tool names before calling mcp_call_tool.', + execute: async () => { try { return await getMcpToolBridge().listTools() } @@ -20,40 +20,24 @@ const tools = [ }), tool({ name: 'mcp_call_tool', - description: 'Call a tool on the MCP server. The result is a list of content and a boolean indicating whether the tool call is an error.', - execute: async ({ name, parameters }) => { + description: 'Call an MCP tool by name. Use mcp_list_tools first to get available tool names.', + execute: async ({ name, arguments: argsJson }) => { try { - const parametersObject = Object.fromEntries(parameters.map(({ name, value }) => [name, value])) - const result = await getMcpToolBridge().callTool({ - name, - arguments: parametersObject, - }) - return result satisfies { - content?: Record[] - isError?: boolean - structuredContent?: Record - toolResult?: unknown - } + const args = argsJson ? JSON.parse(argsJson) : {} + return await getMcpToolBridge().callTool({ name, arguments: args }) } catch (error) { - const message = error instanceof Error ? error.message : String(error) return { isError: true, - content: [ - { - type: 'text', - text: message, - }, - ], + content: [{ type: 'text', text: error instanceof Error ? error.message : String(error) }], } } }, + // NOTICE: `arguments` is z.string() (JSON) because z.unknown() produces `{}` (no `type` key) + // and z.record() emits `propertyNames`, both rejected by OpenAI. parameters: z.object({ - name: z.string().describe('The qualified tool name to call. Use format "::"'), - parameters: z.array(z.object({ - name: z.string().describe('The name of the parameter'), - value: z.unknown().describe('The value of the parameter'), - }).strict()).describe('The parameters to pass to the tool'), + name: z.string().describe('Tool name in "::" format'), + arguments: z.string().describe('JSON object of tool arguments, e.g. {"query":"hello","limit":10}'), }).strict(), }), ]