fix(stage-*): passed tools without check & discovery for provider (#296)

apps/stage-tamagotchi/src/components/InteractiveArea.vue

@@ -16,7 +16,7 @@ const listening = ref(false)
 // const { audioInputs } = useDevicesList({ constraints: { audio: true }, requestPermissions: true })
 // const { selectedAudioDevice, isAudioInputOn, selectedAudioDeviceId } = storeToRefs(useSettings())
 const { isAudioInputOn, selectedAudioDeviceId } = storeToRefs(useSettings())
-const { send, onAfterSend } = useChatStore()
+const { send, onAfterSend, discoverToolsCompatibility } = useChatStore()
 const { messages } = storeToRefs(useChatStore())
 const { t } = useI18n()
 const providersStore = useProvidersStore()
@@ -92,6 +92,12 @@ watch(isAudioInputOn, async (value) => {
   }
 })
 
+watch([activeProvider, activeModel], async () => {
+  if (activeProvider.value && activeModel.value) {
+    await discoverToolsCompatibility(activeModel.value, providersStore.getProviderInstance(activeProvider.value) as ChatProvider, [])
+  }
+})
+
 onAfterSend(async () => {
   messageInput.value = ''
 })

apps/stage-web/src/components/Layouts/InteractiveArea.vue

@@ -26,7 +26,7 @@ const { themeColorsHueDynamic } = storeToRefs(useSettings())
 
 const { audioInputs, ensurePermissions } = useDevicesList({ constraints: { audio: true } })
 const { selectedAudioDevice, isAudioInputOn, selectedAudioDeviceId } = storeToRefs(useSettings())
-const { send, onAfterSend } = useChatStore()
+const { send, onAfterSend, discoverToolsCompatibility } = useChatStore()
 const { messages } = storeToRefs(useChatStore())
 const { audioContext } = useAudioContext()
 const { t } = useI18n()
@@ -128,6 +128,12 @@ watch(showMicrophoneSelect, async (value) => {
   }
 })
 
+watch([activeProvider, activeModel], async () => {
+  if (activeProvider.value && activeModel.value) {
+    await discoverToolsCompatibility(activeModel.value, providersStore.getProviderInstance(activeProvider.value) as ChatProvider, [])
+  }
+})
+
 onMounted(() => {
   // loadWhisper()
   start()

apps/stage-web/src/components/Layouts/MobileInteractiveArea.vue

@@ -20,7 +20,7 @@ const { activeProvider, activeModel } = storeToRefs(useConsciousnessStore())
 // const { audioInputs } = useDevicesList({ constraints: { audio: true }, requestPermissions: true })
 // const { selectedAudioDevice, isAudioInputOn, selectedAudioDeviceId } = storeToRefs(useSettings())
 const { isAudioInputOn, selectedAudioDeviceId, themeColorsHueDynamic } = storeToRefs(useSettings())
-const { send, onAfterSend } = useChatStore()
+const { send, onAfterSend, discoverToolsCompatibility } = useChatStore()
 const { messages } = storeToRefs(useChatStore())
 const { t } = useI18n()
 
@@ -88,6 +88,12 @@ onAfterSend(async () => {
   messageInput.value = ''
 })
 
+watch([activeProvider, activeModel], async () => {
+  if (activeProvider.value && activeModel.value) {
+    await discoverToolsCompatibility(activeModel.value, providersStore.getProviderInstance(activeProvider.value) as ChatProvider, [])
+  }
+})
+
 onMounted(() => {
   start()
 })

packages/stage-ui/package.json

@@ -34,7 +34,9 @@
     "story:dev": "histoire dev",
     "story:build": "histoire build",
     "story:build-base": "histoire build --base /ui",
-    "story:preview": "histoire preview"
+    "story:preview": "histoire preview",
+    "test": "vitest",
+    "test:run": "vitest run"
   },
   "dependencies": {
     "@formkit/auto-animate": "^0.8.2",
@@ -79,6 +81,7 @@
     "@xsai/generate-speech": "catalog:",
     "@xsai/generate-transcription": "catalog:",
     "@xsai/model": "catalog:",
+    "@xsai/shared": "catalog:",
     "@xsai/shared-chat": "catalog:",
     "@xsai/stream-text": "catalog:",
     "@xsai/tool": "catalog:",

packages/stage-ui/src/stores/chat.ts

@@ -18,7 +18,7 @@ export interface ErrorMessage {
 }
 
 export const useChatStore = defineStore('chat', () => {
-  const { stream } = useLLM()
+  const { stream, discoverToolsCompatibility } = useLLM()
   const { systemPrompt } = storeToRefs(useAiriCardStore())
 
   const sending = ref(false)
@@ -209,7 +209,11 @@ export const useChatStore = defineStore('chat', () => {
     sending,
     messages,
     streamingMessage,
+
+    discoverToolsCompatibility,
+
     send,
+
     onBeforeMessageComposed,
     onAfterMessageComposed,
     onBeforeSend,

packages/stage-ui/src/stores/llm.test.ts

@@ -0,0 +1,37 @@
+import { env } from 'node:process'
+
+import { createOpenRouter } from '@xsai-ext/providers-cloud'
+import { describe, expect, it } from 'vitest'
+
+import { attemptForToolsCompatibilityDiscovery } from './llm'
+
+function doesHaveOpenRouterApiKey() {
+  const apiKey = env.LLM_API_OPENROUTER_API_KEY
+  if (!apiKey) {
+    console.warn('Skipping llm store tests, because LLM_API_OPENROUTER_API_KEY is not set')
+  }
+
+  return !!apiKey
+}
+
+const hasOpenRouterApiKey = doesHaveOpenRouterApiKey()
+
+describe.skipIf(!hasOpenRouterApiKey)('llm store', { timeout: 60000 }, async () => {
+  it('should be false for phi-4', async () => {
+    // TODO: base url should not be hardcoded, wait for https://github.com/moeru-ai/xsai/pull/194
+    const res1 = await attemptForToolsCompatibilityDiscovery('microsoft/phi-4', createOpenRouter(env.LLM_API_OPENROUTER_API_KEY!, 'https://openrouter.ai/api/v1/'), [])
+    expect(res1).toBe(false)
+  })
+
+  it('should be false for gpt-4o-mini', async () => {
+    // TODO: base url should not be hardcoded, wait for https://github.com/moeru-ai/xsai/pull/194
+    const res1 = await attemptForToolsCompatibilityDiscovery('openai/gpt-4o-mini', createOpenRouter(env.LLM_API_OPENROUTER_API_KEY!, 'https://openrouter.ai/api/v1/'), [])
+    expect(res1).toBe(false)
+  })
+
+  it('should be true for gpt-4o', async () => {
+    // TODO: base url should not be hardcoded, wait for https://github.com/moeru-ai/xsai/pull/194
+    const res2 = await attemptForToolsCompatibilityDiscovery('openai/gpt-4o', createOpenRouter(env.LLM_API_OPENROUTER_API_KEY!, 'https://openrouter.ai/api/v1/'), [])
+    expect(res2).toBe(true)
+  })
+})

packages/stage-ui/src/stores/llm.ts

@@ -1,44 +1,148 @@
 import type { ChatProvider } from '@xsai-ext/shared-providers'
 import type { Message, ToolCall, ToolMessagePart } from '@xsai/shared-chat'
 
+import { readableStreamToAsyncIterator } from '@moeru/std'
 import { listModels } from '@xsai/model'
+import { XSAIError } from '@xsai/shared'
 import { streamText } from '@xsai/stream-text'
 import { defineStore } from 'pinia'
+import { ref } from 'vue'
 
 import { debug, mcp } from '../tools'
 
-export const useLLM = defineStore('llm', () => {
-  async function stream(model: string, chatProvider: ChatProvider, messages: Message[], options?: {
-    headers?: Record<string, string>
-    onToolCall?: (toolCall: ToolCall) => void
-    onToolCallResult?: (toolCallResult: {
-      id: string
-      result?: string | ToolMessagePart[]
-    }) => void
-  }) {
-    const headers = options?.headers
+export interface StreamOptions {
+  headers?: Record<string, string>
+  onToolCall?: (toolCall: ToolCall) => void
+  onToolCallResult?: (toolCallResult: {
+    id: string
+    result?: string | ToolMessagePart[]
+  }) => void
+  toolsCompatibility?: Map<string, boolean>
+  supportsTools?: boolean
+}
 
-    return await streamText({
-      ...chatProvider.chat(model),
-      maxSteps: 10,
-      // TODO: proper format for other error messages.
-      messages: messages.map(msg => ({ ...msg, content: (msg.role as string === 'error' ? `User encountered error: ${msg.content}` : msg.content), role: (msg.role as string === 'error' ? 'user' : msg.role) } as Message)),
-      headers,
-      tools: [
-        ...await mcp(),
-        ...await debug(),
-      ],
-      onEvent(event) {
-        if (event.type === 'tool-call') {
-          options?.onToolCall?.(event.toolCall)
+function streamOptionsToolsCompatibilityOk(model: string, chatProvider: ChatProvider, _: Message[], options?: StreamOptions, toolsCompatibility: Map<string, boolean> = new Map()): boolean {
+  return !!(options?.supportsTools || toolsCompatibility.get(`${chatProvider.chat(model).baseURL}-${model}`))
+}
+
+async function streamFrom(model: string, chatProvider: ChatProvider, messages: Message[], options?: StreamOptions) {
+  const headers = options?.headers
+
+  return await streamText({
+    ...chatProvider.chat(model),
+    maxSteps: 10,
+    // TODO: proper format for other error messages.
+    messages: messages.map(msg => ({ ...msg, content: (msg.role as string === 'error' ? `User encountered error: ${msg.content}` : msg.content), role: (msg.role as string === 'error' ? 'user' : msg.role) } as Message)),
+    headers,
+    // TODO: we need Automatic tools discovery
+    tools: streamOptionsToolsCompatibilityOk(model, chatProvider, messages, options)
+      ? [
+          ...await mcp(),
+          ...await debug(),
+        ]
+      : undefined,
+    onEvent(event) {
+      if (event.type === 'tool-call') {
+        options?.onToolCall?.(event.toolCall)
+      }
+      else if (event.type === 'tool-call-result') {
+        options?.onToolCallResult?.({ id: event.id, result: event.result })
+      }
+    },
+  })
+}
+
+export async function attemptForToolsCompatibilityDiscovery(model: string, chatProvider: ChatProvider, _: Message[], options?: Omit<StreamOptions, 'supportsTools'>): Promise<boolean> {
+  async function attempt(enable: boolean) {
+    try {
+      const res = await streamFrom(model, chatProvider, [{ role: 'user', content: 'Hello, world!' }], { ...options, supportsTools: enable })
+      for await (const _ of readableStreamToAsyncIterator(res.textStream, async v => v)) {
+        // Drop
+      }
+
+      return true
+    }
+    catch (err) {
+      if (err instanceof Error && err.name === new XSAIError('').name) {
+        // TODO: if you encountered many more errors like these, please, add them here.
+
+        // Ollama
+        /**
+         * {"error":{"message":"registry.ollama.ai/<scope>/<model> does not support tools","type":"api_error","param":null,"code":null}}
+         */
+        if (String(err).includes('does not support tools')) {
+          return false
         }
-        else if (event.type === 'tool-call-result') {
-          options?.onToolCallResult?.({ id: event.id, result: event.result })
+        // OpenRouter
+        /**
+         * {"error":{"message":"No endpoints found that support tool use. To learn more about provider routing, visit: https://openrouter.ai/docs/provider-routing","code":404}}
+         */
+        if (String(err).includes('No endpoints found that support tool use.')) {
+          return false
         }
-      },
+      }
+
+      throw err
+    }
+  }
+
+  function promiseAllWithInterval<T>(promises: (() => Promise<T>)[], interval: number): Promise<{ result?: T, error?: any }[]> {
+    return new Promise((resolve) => {
+      const results: { result?: T, error?: any }[] = []
+      let completed = 0
+
+      promises.forEach((promiseFn, index) => {
+        setTimeout(() => {
+          promiseFn()
+            .then((result) => {
+              results[index] = { result }
+            })
+            .catch((err) => {
+              results[index] = { error: err }
+            })
+            .finally(() => {
+              completed++
+              if (completed === promises.length) {
+                resolve(results)
+              }
+            })
+        }, index * interval)
+      })
     })
   }
 
+  const attempts = [
+    () => attempt(true),
+    () => attempt(false),
+  ]
+
+  const attemptsResults = await promiseAllWithInterval<boolean | undefined>(attempts, 1000)
+  if (attemptsResults.some(res => res.error)) {
+    const err = new Error(`Error during tools compatibility discovery for model: ${model}. Errors: ${attemptsResults.map(res => res.error).filter(Boolean).join(', ')}`)
+    err.cause = attemptsResults.map(res => res.error).filter(Boolean)
+    throw err
+  }
+
+  return attemptsResults[0].result === true && attemptsResults[1].result === true
+}
+
+export const useLLM = defineStore('llm', () => {
+  const toolsCompatibility = ref<Map<string, boolean>>(new Map())
+
+  async function discoverToolsCompatibility(model: string, chatProvider: ChatProvider, _: Message[], options?: Omit<StreamOptions, 'supportsTools'>) {
+    // Cached, no need to discover again
+    if (toolsCompatibility.value.has(`${chatProvider.chat(model).baseURL}-${model}`)) {
+      return
+    }
+
+    const res = await attemptForToolsCompatibilityDiscovery(model, chatProvider, _, { ...options, toolsCompatibility: toolsCompatibility.value })
+    toolsCompatibility.value.set(`${chatProvider.chat(model).baseURL}-${model}`, res)
+  }
+
+  function stream(model: string, chatProvider: ChatProvider, messages: Message[], options?: StreamOptions) {
+    return streamFrom(model, chatProvider, messages, { ...options, toolsCompatibility: toolsCompatibility.value })
+  }
+
   async function models(apiUrl: string, apiKey: string) {
     if (apiUrl === '') {
       return []
@@ -62,5 +166,6 @@ export const useLLM = defineStore('llm', () => {
   return {
     models,
     stream,
+    discoverToolsCompatibility,
   }
 })

packages/stage-ui/vite.config.ts

@@ -1,3 +1,5 @@
+import type { Plugin } from 'vite'
+
 import { join, resolve } from 'node:path'
 
 import Vue from '@vitejs/plugin-vue'
@@ -20,9 +22,13 @@ export default defineConfig({
     },
   },
   plugins: [
-    Yaml(),
+    // TODO: Type wrong for `unplugin-yaml` in Histoire required
+    // Vite version, wait until Histoire updates to support Vite 7
+    Yaml() as Plugin,
     Vue(),
     Unocss(),
-    Inspect(),
+    // TODO: Type wrong for `unplugin-yaml` in Histoire required
+    // Vite version, wait until Histoire updates to support Vite 7
+    Inspect() as Plugin,
   ],
 })

packages/stage-ui/vitest.config.ts

@@ -0,0 +1,7 @@
+import { defineConfig } from 'vitest/config'
+
+export default defineConfig({
+  test: {
+    include: ['src/**/*.test.ts'],
+  },
+})

pnpm-lock.yaml

@@ -37,7 +37,7 @@ catalogs:
       specifier: ^0.3.0-beta.8
       version: 0.3.0-beta.8
     '@xsai/shared':
-      specifier: ^0.3.0-beta.8
+      specifier: 0.3.0-beta.8
       version: 0.3.0-beta.8
     '@xsai/shared-chat':
       specifier: ^0.3.0-beta.8
@@ -1247,6 +1247,9 @@ importers:
       '@xsai/model':
         specifier: 'catalog:'
         version: 0.3.0-beta.8
+      '@xsai/shared':
+        specifier: 'catalog:'
+        version: 0.3.0-beta.8
       '@xsai/shared-chat':
         specifier: 'catalog:'
         version: 0.3.0-beta.8
@@ -6340,15 +6343,12 @@ packages:
   '@xsai/shared@0.3.0':
     resolution: {integrity: sha512-fUtZY+P2uedrxg3oU/Jdv9K/v+K9SiS3DmZ/iO5WFqC3NK/fvU/eJMJARpPCqLhPiDbv0+5eHZ/Ok/WXdIWLqg==}
 
-  '@xsai/shared@0.3.0-beta.5':
-    resolution: {integrity: sha512-0vDW0fkEcFhfd3uG2OEMkoqq/ghh62KJUokMUwaAPKLNcVNAVcFRoslH9lKjdHJziSZ/b59f5rpUEjU3DHooZQ==}
-
-  '@xsai/shared@0.3.0-beta.7':
-    resolution: {integrity: sha512-V2zLH7NKAzp5dT+4fxhed+05svN5vtyxvuLcY/E+K/A+XrAHQW1epRuqXBHwXWcjGoSVGWbhHYe8MzW0PT1t6Q==}
-
   '@xsai/shared@0.3.0-beta.8':
     resolution: {integrity: sha512-S7BYvluU7aA2Ti7HEblV3ISDZbF89VP2dDC5WCrylYlQIYoFlZjlhJhJL5c/+KwX+og0pvh5Y0iSPKdcoMmLmw==}
 
+  '@xsai/shared@0.3.2':
+    resolution: {integrity: sha512-StLLjMwfDDyneIInJCaYYEYbzi9ipAEKp6/cSErQ3r7OmCvn9G7urLgxyT+ofopE+aGGBck22ryPYOaSWRyENQ==}
+
   '@xsai/stream-text@0.3.0-beta.6':
     resolution: {integrity: sha512-w1aQEqa1yGXjSckZzZcpFgRf3JzUuqHDB4tc3UZaKa4DLGmslr3e4xXpOXWuqgIeDMdj2YWI9163GC/bfF0zBA==}
 
@@ -17670,20 +17670,20 @@ snapshots:
   '@xsai-ext/providers-cloud@0.3.0-beta.8':
     dependencies:
       '@xsai-ext/shared-providers': 0.3.0-beta.8
-      '@xsai/shared': 0.3.0-beta.8
+      '@xsai/shared': 0.3.0
 
   '@xsai-ext/providers-local@0.3.0-beta.8':
     dependencies:
       '@xsai-ext/shared-providers': 0.3.0-beta.8
-      '@xsai/shared': 0.3.0-beta.8
+      '@xsai/shared': 0.3.0
 
   '@xsai-ext/shared-providers@0.2.2':
     dependencies:
-      '@xsai/shared': 0.3.0
+      '@xsai/shared': 0.3.2
 
   '@xsai-ext/shared-providers@0.3.0-beta.8':
     dependencies:
-      '@xsai/shared': 0.3.0-beta.8
+      '@xsai/shared': 0.3.0
 
   '@xsai-transformers/embed@0.0.7(web-worker@1.5.0)':
     dependencies:
@@ -17692,7 +17692,7 @@ snapshots:
       '@xsai-ext/shared-providers': 0.3.0-beta.8
       '@xsai-transformers/shared': 0.0.7
       '@xsai/embed': 0.3.0-beta.5
-      '@xsai/shared': 0.3.0-beta.5
+      '@xsai/shared': 0.3.0
       gpuu: 1.0.4
     optionalDependencies:
       web-worker: 1.5.0
@@ -17701,29 +17701,29 @@ snapshots:
     dependencies:
       '@huggingface/transformers': 3.6.1
       '@xsai-ext/shared-providers': 0.3.0-beta.8
-      '@xsai/shared': 0.3.0-beta.7
+      '@xsai/shared': 0.3.0
       onnxruntime-common: 1.22.0
 
   '@xsai/embed@0.3.0-beta.5':
     dependencies:
-      '@xsai/shared': 0.3.0-beta.7
+      '@xsai/shared': 0.3.0
 
   '@xsai/embed@0.3.0-beta.8':
     dependencies:
-      '@xsai/shared': 0.3.0-beta.8
+      '@xsai/shared': 0.3.0
 
   '@xsai/generate-speech@0.3.0-beta.8':
     dependencies:
-      '@xsai/shared': 0.3.0-beta.8
+      '@xsai/shared': 0.3.0
 
   '@xsai/generate-text@0.3.0-beta.6':
     dependencies:
-      '@xsai/shared': 0.3.0-beta.7
+      '@xsai/shared': 0.3.0
       '@xsai/shared-chat': 0.3.0-beta.6
 
   '@xsai/generate-text@0.3.0-beta.8':
     dependencies:
-      '@xsai/shared': 0.3.0-beta.8
+      '@xsai/shared': 0.3.0
       '@xsai/shared-chat': 0.3.0-beta.8
 
   '@xsai/generate-transcription@0.3.0-beta.8':
@@ -17732,26 +17732,24 @@ snapshots:
 
   '@xsai/model@0.3.0-beta.8':
     dependencies:
-      '@xsai/shared': 0.3.0-beta.8
+      '@xsai/shared': 0.3.0
 
   '@xsai/shared-chat@0.3.0-beta.6':
     dependencies:
-      '@xsai/shared': 0.3.0-beta.7
+      '@xsai/shared': 0.3.0
 
   '@xsai/shared-chat@0.3.0-beta.8':
     dependencies:
-      '@xsai/shared': 0.3.0-beta.8
+      '@xsai/shared': 0.3.0
 
   '@xsai/shared@0.2.2': {}
 
   '@xsai/shared@0.3.0': {}
 
-  '@xsai/shared@0.3.0-beta.5': {}
-
-  '@xsai/shared@0.3.0-beta.7': {}
-
   '@xsai/shared@0.3.0-beta.8': {}
 
+  '@xsai/shared@0.3.2': {}
+
   '@xsai/stream-text@0.3.0-beta.6':
     dependencies:
       '@xsai/shared-chat': 0.3.0-beta.6
@@ -17762,7 +17760,7 @@ snapshots:
 
   '@xsai/tool@0.3.0-beta.8(zod-to-json-schema@3.24.6(zod@3.25.74))(zod@3.25.74)':
     dependencies:
-      '@xsai/shared': 0.3.0-beta.8
+      '@xsai/shared': 0.3.0
       '@xsai/shared-chat': 0.3.0-beta.8
       xsschema: 0.3.0-beta.8(@valibot/to-json-schema@1.0.0-rc.0(valibot@1.0.0-beta.9(typescript@5.8.3)))(zod-to-json-schema@3.24.6(zod@3.25.74))(zod@3.25.74)
     transitivePeerDependencies:

pnpm-workspace.yaml

@@ -17,7 +17,7 @@ catalog:
   '@xsai/generate-text': ^0.3.0-beta.8
   '@xsai/generate-transcription': 0.3.0-beta.8
   '@xsai/model': ^0.3.0-beta.8
-  '@xsai/shared': ^0.3.0-beta.8
+  '@xsai/shared': 0.3.0-beta.8
   '@xsai/shared-chat': ^0.3.0-beta.8
   '@xsai/stream-text': ^0.3.0-beta.8
   '@xsai/tool': ^0.3.0-beta.8