diff --git a/packages/core-agent/src/index.ts b/packages/core-agent/src/index.ts index 8ec6dcdf8..9e9535880 100644 --- a/packages/core-agent/src/index.ts +++ b/packages/core-agent/src/index.ts @@ -8,10 +8,12 @@ export { createChatHooks } from './runtime/agent-hooks' export type { ContextHistoryEntry, ContextRegistry } from './runtime/context-registry' export { createContextRegistry } from './runtime/context-registry' export { + isContentArrayRelatedError, isToolRelatedError, modelKey, sanitizeMessages, streamFrom, + streamOptionsContentArrayCompatibilityOk, streamOptionsToolsCompatibilityOk, } from './runtime/llm-service' export { mergeLoadedSessionMessages } from './session/merge-loaded-session-messages' diff --git a/packages/core-agent/src/runtime/llm-service.test.ts b/packages/core-agent/src/runtime/llm-service.test.ts index 0ccad508c..8fc13b44b 100644 --- a/packages/core-agent/src/runtime/llm-service.test.ts +++ b/packages/core-agent/src/runtime/llm-service.test.ts @@ -3,7 +3,7 @@ import type { Message, Tool } from '@xsai/shared-chat' import { describe, expect, it, vi } from 'vitest' -import { streamFrom } from './llm-service' +import { isContentArrayRelatedError, sanitizeMessages, streamFrom } from './llm-service' const { streamTextMock } = vi.hoisted(() => ({ streamTextMock: vi.fn(), @@ -111,3 +111,147 @@ describe('streamFrom tool error capture', () => { })) }) }) + +describe('sanitizeMessages', () => { + it('rewrites internal `error`-role messages as user-role narrations', () => { + /** + * @example + * sanitizeMessages([{ role: 'error', content: 'Remote sent 400' }]) + * // -> [{ role: 'user', content: 'User encountered error: Remote sent 400' }] + */ + const out = sanitizeMessages([{ role: 'error', content: 'Remote sent 400' }]) + expect(out).toEqual([ + { role: 'user', content: 'User encountered error: Remote sent 400' }, + ]) + }) + + it('flattens text-only content arrays to a string by default', () => { + /** + * @example + * sanitizeMessages([{ + * role: 'user', + * content: [{ type: 'text', text: 'hi' }, { type: 'text', text: ' there' }], + * }]) + * // -> [{ role: 'user', content: 'hi there' }] + */ + const out = sanitizeMessages([{ + role: 'user', + content: [ + { type: 'text', text: 'hi' }, + { type: 'text', text: ' there' }, + ], + }]) + expect(out).toEqual([{ role: 'user', content: 'hi there' }]) + }) + + it('preserves multimodal arrays when supportsContentArray is true (default)', () => { + /** + * @example + * sanitizeMessages([{ role: 'user', content: [{type:'text',text:'see'},{type:'image_url',...}] }]) + * // -> unchanged: image_url part stays so vision-capable providers receive the image + */ + const message = { + role: 'user', + content: [ + { type: 'text', text: 'see this' }, + { type: 'image_url', image_url: { url: 'data:image/png;base64,AAA' } }, + ], + } + const out = sanitizeMessages([message]) + expect(out[0]).toEqual(message) + }) + + // ROOT CAUSE: + // + // Some Rust/serde-based OpenAI-compatible gateways only deserialize + // `messages[].content` as a plain string and reject content-part arrays + // with HTTP 400 "Failed to deserialize the JSON body into the target type: + // messages[N]: invalid type: sequence, expected a string". Before the fix, + // historical messages that contained an `image_url` part (uploaded image, + // vision capture, restored session) bypassed the existing flatten branch + // and stayed as arrays, so every subsequent send re-tripped the 400. + // + // We fixed this by adding a `supportsContentArray` flag — when the runtime + // auto-degrade has flipped it to `false`, we force-flatten arrays to a + // text-only string and drop non-text parts so the request shape matches + // what a string-only provider can deserialize. + // + // See: https://github.com/moeru-ai/airi/issues/1500 + it('issue #1500: drops image_url parts and flattens to string when supportsContentArray=false', () => { + /** + * @example + * sanitizeMessages([{ role:'user', content: [{type:'text',text:'hi'},{type:'image_url',...}] }], false) + * // -> [{ role: 'user', content: 'hi' }] + */ + const out = sanitizeMessages([ + { + role: 'user', + content: [ + { type: 'text', text: 'hi' }, + { type: 'image_url', image_url: { url: 'data:image/png;base64,AAA' } }, + ], + }, + ], false) + expect(out).toEqual([{ role: 'user', content: 'hi' }]) + }) + + it('issue #1500: drops audio/file parts when supportsContentArray=false', () => { + /** + * @example + * sanitizeMessages([{ role:'user', content: [{type:'text',text:'q'},{type:'input_audio',...},{type:'file',...}] }], false) + * // -> [{ role: 'user', content: 'q' }] + */ + const out = sanitizeMessages([ + { + role: 'user', + content: [ + { type: 'text', text: 'q' }, + { type: 'input_audio', input_audio: { data: 'AAA', format: 'wav' } }, + { type: 'file', file: { file_id: 'f_1' } }, + ], + }, + ], false) + expect(out).toEqual([{ role: 'user', content: 'q' }]) + }) + + it('passes string content through untouched regardless of the flag', () => { + expect(sanitizeMessages([{ role: 'user', content: 'plain' }], true)) + .toEqual([{ role: 'user', content: 'plain' }]) + expect(sanitizeMessages([{ role: 'user', content: 'plain' }], false)) + .toEqual([{ role: 'user', content: 'plain' }]) + }) +}) + +describe('isContentArrayRelatedError', () => { + it('issue #1500: detects the Rust/serde "expected a string" wire error', () => { + /** + * @example + * isContentArrayRelatedError( + * `Remote sent 400 response: {"error":{"message":"Failed to deserialize the JSON body into the target type: messages[7]: invalid type: sequence, expected a string at line 1 column 5603","code":"invalid_request_error"}}` + * ) + * // -> true + */ + const wire = 'Remote sent 400 response: {"error":{"message":"Failed to deserialize the JSON body into the target type: messages[7]: invalid type: sequence, expected a string at line 1 column 5603","type":"invalid_request_error","param":null,"code":"invalid_request_error"}}' + expect(isContentArrayRelatedError(wire)).toBe(true) + expect(isContentArrayRelatedError(new Error(wire))).toBe(true) + }) + + it('detects the Pydantic/Python "Input should be a valid string" variant', () => { + /** + * @example + * isContentArrayRelatedError('messages.0.content: Input should be a valid string') + * // -> true + */ + expect(isContentArrayRelatedError('messages.0.content: Input should be a valid string')) + .toBe(true) + expect(isContentArrayRelatedError('messages.3.content expected string, got list')) + .toBe(true) + }) + + it('does not false-positive on unrelated 400s', () => { + expect(isContentArrayRelatedError('Remote sent 400 response: model not found')).toBe(false) + expect(isContentArrayRelatedError('Remote sent 401 response: invalid api key')).toBe(false) + expect(isContentArrayRelatedError('Tool call failed: invalid schema for function')).toBe(false) + expect(isContentArrayRelatedError(undefined)).toBe(false) + }) +}) diff --git a/packages/core-agent/src/runtime/llm-service.ts b/packages/core-agent/src/runtime/llm-service.ts index 992fb8221..8ce0d82f0 100644 --- a/packages/core-agent/src/runtime/llm-service.ts +++ b/packages/core-agent/src/runtime/llm-service.ts @@ -7,7 +7,32 @@ import { errorMessageFrom } from '@moeru/std' import { stepCountAtLeast } from '@xsai/shared-chat' import { streamText } from '@xsai/stream-text' -export function sanitizeMessages(messages: unknown[]): Message[] { +/** + * Normalize chat messages so they match the wire format the active provider + * actually accepts, flattening content-part arrays back to plain strings when + * the provider can't deserialize arrays. + * + * Use when: + * - Composing the final message list right before handing it to the OpenAI- + * compatible chat SDK. + * + * Expects: + * - `role: 'error'` entries (AIRI-internal markers from the chat UI). They are + * rewritten as user-role narrations so the provider doesn't reject them. + * - `content` may be a string, a content-part array, or undefined. + * + * Returns: + * - A new array of `Message` values; original objects are not mutated. + * + * @param messages - Raw messages from the chat session, may include AIRI's + * `error` role. + * @param supportsContentArray - When `false`, force-flatten every array + * content (including text + `image_url` mixes) to a text-only string and + * drop non-text parts. Drives the runtime auto-degrade for strict providers. + * Defaults to `true` to preserve vision/multimodal payloads on capable + * providers. + */ +export function sanitizeMessages(messages: unknown[], supportsContentArray: boolean = true): Message[] { return messages.map((message: any) => { if (message && message.role === 'error') { return { @@ -16,11 +41,27 @@ export function sanitizeMessages(messages: unknown[]): Message[] { } as Message } - // NOTICE: Flatten array content for providers (e.g. DeepSeek) that expect string, - // not content-part arrays. Skipped when image_url parts are present. + // NOTICE: + // Flatten array content for providers (e.g. DeepSeek and other Rust/serde- + // strict OpenAI-compatible gateways) that only accept `messages[].content` + // as a plain string and reject arrays with `Failed to deserialize the JSON + // body into the target type: messages[N]: invalid type: sequence, expected + // a string`. + // Root cause: OpenAI's chat API permits `content` as either `string` or an + // array of content parts; some compatible servers only implement the + // string variant. + // Source/context: https://github.com/moeru-ai/airi/issues/1500 + // Removal condition: when every supported provider accepts content-part + // arrays uniformly (no longer realistic for the OpenAI-compatible + // ecosystem, so this is effectively load-bearing). if (message && Array.isArray(message.content)) { const contentParts = message.content as { type?: string, text?: string }[] - if (!contentParts.some(part => part?.type === 'image_url')) { + const hasNonTextPart = contentParts.some(part => part?.type && part.type !== 'text') + // When the provider supports arrays, only flatten pure-text arrays so we + // never silently drop image / audio / file parts on a vision-capable + // model. When it doesn't, flatten unconditionally; non-text parts are + // dropped because the provider can't carry them anyway. + if (!supportsContentArray || !hasNonTextPart) { return { ...message, content: contentParts.map(part => part?.text ?? '').join('') } as Message } } @@ -34,12 +75,26 @@ export function modelKey(model: string, chatProvider: ChatProvider): string { } export function streamOptionsToolsCompatibilityOk(model: string, chatProvider: ChatProvider, options?: StreamOptions): boolean { - if (options?.supportsTools) - return true + if (options?.supportsTools !== undefined) + return options.supportsTools const key = modelKey(model, chatProvider) return options?.toolsCompatibility?.get(key) !== false } +/** + * Resolve whether the active model+provider currently supports content-part + * arrays. Defaults to `true` so first-time calls keep multimodal payloads; + * flips to `false` once {@link isContentArrayRelatedError} has fired on this + * model key and the caller has cached the degrade in + * {@link StreamOptions.contentArrayCompatibility}. + */ +export function streamOptionsContentArrayCompatibilityOk(model: string, chatProvider: ChatProvider, options?: StreamOptions): boolean { + if (options?.supportsContentArray !== undefined) + return options.supportsContentArray + const key = modelKey(model, chatProvider) + return options?.contentArrayCompatibility?.get(key) !== false +} + async function resolveTools(options?: StreamOptions) { const tools = typeof options?.tools === 'function' ? await options.tools() @@ -114,7 +169,8 @@ export async function streamFrom({ builtinToolsResolver, }: StreamFromOptions) { const chatConfig = chatProvider.chat(model) - const sanitized = sanitizeMessages(messages as unknown[]) + const supportsContentArray = streamOptionsContentArrayCompatibilityOk(model, chatProvider, options) + const sanitized = sanitizeMessages(messages as unknown[], supportsContentArray) const supportedTools = streamOptionsToolsCompatibilityOk(model, chatProvider, options) const builtinTools = supportedTools @@ -224,3 +280,41 @@ export function isToolRelatedError(error: unknown): boolean { const message = String(error) return TOOLS_RELATED_ERROR_PATTERNS.some(pattern => pattern.test(message)) } + +// Runtime auto-degrade: patterns that indicate the provider rejected +// content-part arrays and only accepts a plain string for `messages[].content`. +// +// The first pattern matches the Rust/serde wire-level error format used by +// many strict OpenAI-compatible gateways (e.g. DeepSeek-style servers): +// "Failed to deserialize the JSON body into the target type: +// messages[7]: invalid type: sequence, expected a string at line 1 column …" +// The second pattern covers Python/Pydantic-style errors like +// "messages.0.content: Input should be a valid string" +// and other variants that surface the same root cause. +// +// See: https://github.com/moeru-ai/airi/issues/1500 +const CONTENT_ARRAY_RELATED_ERROR_PATTERNS: RegExp[] = [ + /messages\[\d+\][^"]*invalid type:\s*sequence,\s*expected\s+a\s+string/i, + /messages\.\d+\.content[^"]*(?:expected|should be).*string/i, +] + +/** + * Whether the given error indicates the provider rejected content-part arrays + * and the caller should auto-degrade to string-only `content` for this model. + * + * Use when: + * - Catching errors thrown by {@link streamFrom} so the chat store can flip + * `contentArrayCompatibility` for the failing model key. + * + * Expects: + * - `error` may be an Error instance, a thrown SDK response object, a string, + * or anything else; we coerce via `String(error)` and pattern-match. + * + * Returns: + * - `true` when the message matches a known "content array unsupported" wire + * format from an OpenAI-compatible gateway, otherwise `false`. + */ +export function isContentArrayRelatedError(error: unknown): boolean { + const message = String(error) + return CONTENT_ARRAY_RELATED_ERROR_PATTERNS.some(pattern => pattern.test(message)) +} diff --git a/packages/core-agent/src/types/llm.ts b/packages/core-agent/src/types/llm.ts index e61dccb04..061be57bf 100644 --- a/packages/core-agent/src/types/llm.ts +++ b/packages/core-agent/src/types/llm.ts @@ -18,6 +18,23 @@ export interface StreamOptions { waitForTools?: boolean captureToolErrors?: boolean tools?: Tool[] | (() => Promise) + /** + * Per-model runtime cache of whether the provider accepts content-part arrays + * (e.g. `[{type:'text',...},{type:'image_url',...}]`) for `messages[].content`. + * + * Some OpenAI-compatible providers (notably Rust/serde-strict gateways) only + * deserialize `content` as a plain string and reject arrays with HTTP 400 + * `Failed to deserialize the JSON body into the target type: messages[N]: + * invalid type: sequence, expected a string`. When a stream surfaces such an + * error we set the entry to `false` for the model key and force-flatten on + * the next attempt. + * + * Mirrors {@link toolsCompatibility} for the tool-calling capability. + * + * See: https://github.com/moeru-ai/airi/issues/1500 + */ + contentArrayCompatibility?: Map + supportsContentArray?: boolean } export type BuiltinToolsResolver = (model: string, chatProvider: ChatProvider) => Promise diff --git a/packages/stage-ui/src/stores/llm.ts b/packages/stage-ui/src/stores/llm.ts index e79e31597..a61ce91c8 100644 --- a/packages/stage-ui/src/stores/llm.ts +++ b/packages/stage-ui/src/stores/llm.ts @@ -3,7 +3,7 @@ import type { WebSocketEvents } from '@proj-airi/server-sdk' import type { ChatProvider } from '@xsai-ext/providers/utils' import type { Message, Tool } from '@xsai/shared-chat' -import { streamFrom as coreStreamFrom, isToolRelatedError, modelKey } from '@proj-airi/core-agent' +import { streamFrom as coreStreamFrom, isContentArrayRelatedError, isToolRelatedError, modelKey } from '@proj-airi/core-agent' import { listModels } from '@xsai/model' import { uniqBy } from 'es-toolkit' import { defineStore } from 'pinia' @@ -14,7 +14,7 @@ import { useLlmToolsStore } from './llm-tools' import { useModsServerChannelStore } from './mods/api/channel-server' export type { StreamEvent, StreamOptions } from '@proj-airi/core-agent' -export { isToolRelatedError } from '@proj-airi/core-agent' +export { isContentArrayRelatedError, isToolRelatedError } from '@proj-airi/core-agent' function toolNameFrom(tool: Tool) { const candidate = tool as Tool & { @@ -29,54 +29,75 @@ function toolNameFrom(tool: Tool) { export const useLLM = defineStore('llm', () => { const toolsCompatibility = ref>(new Map()) + const contentArrayCompatibility = ref>(new Map()) const modsServerChannelStore = useModsServerChannelStore() const llmToolsStore = useLlmToolsStore() async function stream(model: string, chatProvider: ChatProvider, messages: Message[], options?: StreamOptions) { const key = modelKey(model, chatProvider) - try { - // TODO(@nekomeowww,@shinohara-rin): we should not register the command callback on every stream anyway... - const sendSparkCommand = (command: WebSocketEvents['spark:command']) => { - // TODO(@nekomeowww): instruct the LLM to understand what destination is. - // Currently without skill like prompt injection, many issues occur. - // destination mostly are wrong or hallucinated, we need to find a way to make it more reliable. - // - // For now, since destinations as array will always broadcast to all connected modules/agents, we can set it to - // empty array to avoid wrong routing. - command.destinations = [] + // TODO(@nekomeowww,@shinohara-rin): we should not register the command callback on every stream anyway... + const sendSparkCommand = (command: WebSocketEvents['spark:command']) => { + // TODO(@nekomeowww): instruct the LLM to understand what destination is. + // Currently without skill like prompt injection, many issues occur. + // destination mostly are wrong or hallucinated, we need to find a way to make it more reliable. + // + // For now, since destinations as array will always broadcast to all connected modules/agents, we can set it to + // empty array to avoid wrong routing. + command.destinations = [] - modsServerChannelStore.send({ - type: 'spark:command', - data: command, - }) - } - - await coreStreamFrom({ - model, - chatProvider, - messages, - options: { ...options, toolsCompatibility: toolsCompatibility.value }, - builtinToolsResolver: async () => { - await llmToolsStore.awaitPendingRegistrations() - - // Reverse twice so later runtime registrations win while original tool order stays stable. - return uniqBy( - [ - ...await mcp(), - ...await debug(), - ...await createSparkCommandTool({ sendSparkCommand }), - ...await llmToolsStore.activeTools, - ].toReversed(), - tool => toolNameFrom(tool) ?? tool, - ).toReversed() - }, + modsServerChannelStore.send({ + type: 'spark:command', + data: command, }) } + + const builtinToolsResolver = async () => { + await llmToolsStore.awaitPendingRegistrations() + + // Reverse twice so later runtime registrations win while original tool order stays stable. + return uniqBy( + [ + ...await mcp(), + ...await debug(), + ...await createSparkCommandTool({ sendSparkCommand }), + ...await llmToolsStore.activeTools, + ].toReversed(), + tool => toolNameFrom(tool) ?? tool, + ).toReversed() + } + + const runStream = () => coreStreamFrom({ + model, + chatProvider, + messages, + options: { + ...options, + toolsCompatibility: toolsCompatibility.value, + contentArrayCompatibility: contentArrayCompatibility.value, + }, + builtinToolsResolver, + }) + + try { + await runStream() + } catch (err) { if (isToolRelatedError(err)) { console.warn(`[llm] Auto-disabling tools for "${key}" due to tool-related error`) toolsCompatibility.value.set(key, false) } + // NOTICE: + // Auto-degrade content-part arrays to plain strings on the next attempt + // when the provider returned the Rust/serde-style "expected a string" + // 400. We retry once inline so the user's failing turn recovers without + // requiring them to resend; subsequent calls reuse the cached degrade. + // See: https://github.com/moeru-ai/airi/issues/1500 + if (isContentArrayRelatedError(err) && contentArrayCompatibility.value.get(key) !== false) { + console.warn(`[llm] Auto-disabling content-part arrays for "${key}" and retrying once`) + contentArrayCompatibility.value.set(key, false) + await runStream() + return + } throw err } }