fix(stage-ui): auto-degrade content arrays for strict providers (#1740)

packages/core-agent/src/index.ts

@@ -8,10 +8,12 @@ export { createChatHooks } from './runtime/agent-hooks'
 export type { ContextHistoryEntry, ContextRegistry } from './runtime/context-registry'
 export { createContextRegistry } from './runtime/context-registry'
 export {
+  isContentArrayRelatedError,
   isToolRelatedError,
   modelKey,
   sanitizeMessages,
   streamFrom,
+  streamOptionsContentArrayCompatibilityOk,
   streamOptionsToolsCompatibilityOk,
 } from './runtime/llm-service'
 export { mergeLoadedSessionMessages } from './session/merge-loaded-session-messages'

packages/core-agent/src/runtime/llm-service.test.ts

@@ -3,7 +3,7 @@ import type { Message, Tool } from '@xsai/shared-chat'
 
 import { describe, expect, it, vi } from 'vitest'
 
-import { streamFrom } from './llm-service'
+import { isContentArrayRelatedError, sanitizeMessages, streamFrom } from './llm-service'
 
 const { streamTextMock } = vi.hoisted(() => ({
   streamTextMock: vi.fn(),
@@ -111,3 +111,147 @@ describe('streamFrom tool error capture', () => {
     }))
   })
 })
+
+describe('sanitizeMessages', () => {
+  it('rewrites internal `error`-role messages as user-role narrations', () => {
+    /**
+     * @example
+     * sanitizeMessages([{ role: 'error', content: 'Remote sent 400' }])
+     * // -> [{ role: 'user', content: 'User encountered error: Remote sent 400' }]
+     */
+    const out = sanitizeMessages([{ role: 'error', content: 'Remote sent 400' }])
+    expect(out).toEqual([
+      { role: 'user', content: 'User encountered error: Remote sent 400' },
+    ])
+  })
+
+  it('flattens text-only content arrays to a string by default', () => {
+    /**
+     * @example
+     * sanitizeMessages([{
+     *   role: 'user',
+     *   content: [{ type: 'text', text: 'hi' }, { type: 'text', text: ' there' }],
+     * }])
+     * // -> [{ role: 'user', content: 'hi there' }]
+     */
+    const out = sanitizeMessages([{
+      role: 'user',
+      content: [
+        { type: 'text', text: 'hi' },
+        { type: 'text', text: ' there' },
+      ],
+    }])
+    expect(out).toEqual([{ role: 'user', content: 'hi there' }])
+  })
+
+  it('preserves multimodal arrays when supportsContentArray is true (default)', () => {
+    /**
+     * @example
+     * sanitizeMessages([{ role: 'user', content: [{type:'text',text:'see'},{type:'image_url',...}] }])
+     * // -> unchanged: image_url part stays so vision-capable providers receive the image
+     */
+    const message = {
+      role: 'user',
+      content: [
+        { type: 'text', text: 'see this' },
+        { type: 'image_url', image_url: { url: 'data:image/png;base64,AAA' } },
+      ],
+    }
+    const out = sanitizeMessages([message])
+    expect(out[0]).toEqual(message)
+  })
+
+  // ROOT CAUSE:
+  //
+  // Some Rust/serde-based OpenAI-compatible gateways only deserialize
+  // `messages[].content` as a plain string and reject content-part arrays
+  // with HTTP 400 "Failed to deserialize the JSON body into the target type:
+  // messages[N]: invalid type: sequence, expected a string". Before the fix,
+  // historical messages that contained an `image_url` part (uploaded image,
+  // vision capture, restored session) bypassed the existing flatten branch
+  // and stayed as arrays, so every subsequent send re-tripped the 400.
+  //
+  // We fixed this by adding a `supportsContentArray` flag — when the runtime
+  // auto-degrade has flipped it to `false`, we force-flatten arrays to a
+  // text-only string and drop non-text parts so the request shape matches
+  // what a string-only provider can deserialize.
+  //
+  // See: https://github.com/moeru-ai/airi/issues/1500
+  it('issue #1500: drops image_url parts and flattens to string when supportsContentArray=false', () => {
+    /**
+     * @example
+     * sanitizeMessages([{ role:'user', content: [{type:'text',text:'hi'},{type:'image_url',...}] }], false)
+     * // -> [{ role: 'user', content: 'hi' }]
+     */
+    const out = sanitizeMessages([
+      {
+        role: 'user',
+        content: [
+          { type: 'text', text: 'hi' },
+          { type: 'image_url', image_url: { url: 'data:image/png;base64,AAA' } },
+        ],
+      },
+    ], false)
+    expect(out).toEqual([{ role: 'user', content: 'hi' }])
+  })
+
+  it('issue #1500: drops audio/file parts when supportsContentArray=false', () => {
+    /**
+     * @example
+     * sanitizeMessages([{ role:'user', content: [{type:'text',text:'q'},{type:'input_audio',...},{type:'file',...}] }], false)
+     * // -> [{ role: 'user', content: 'q' }]
+     */
+    const out = sanitizeMessages([
+      {
+        role: 'user',
+        content: [
+          { type: 'text', text: 'q' },
+          { type: 'input_audio', input_audio: { data: 'AAA', format: 'wav' } },
+          { type: 'file', file: { file_id: 'f_1' } },
+        ],
+      },
+    ], false)
+    expect(out).toEqual([{ role: 'user', content: 'q' }])
+  })
+
+  it('passes string content through untouched regardless of the flag', () => {
+    expect(sanitizeMessages([{ role: 'user', content: 'plain' }], true))
+      .toEqual([{ role: 'user', content: 'plain' }])
+    expect(sanitizeMessages([{ role: 'user', content: 'plain' }], false))
+      .toEqual([{ role: 'user', content: 'plain' }])
+  })
+})
+
+describe('isContentArrayRelatedError', () => {
+  it('issue #1500: detects the Rust/serde "expected a string" wire error', () => {
+    /**
+     * @example
+     * isContentArrayRelatedError(
+     *   `Remote sent 400 response: {"error":{"message":"Failed to deserialize the JSON body into the target type: messages[7]: invalid type: sequence, expected a string at line 1 column 5603","code":"invalid_request_error"}}`
+     * )
+     * // -> true
+     */
+    const wire = 'Remote sent 400 response: {"error":{"message":"Failed to deserialize the JSON body into the target type: messages[7]: invalid type: sequence, expected a string at line 1 column 5603","type":"invalid_request_error","param":null,"code":"invalid_request_error"}}'
+    expect(isContentArrayRelatedError(wire)).toBe(true)
+    expect(isContentArrayRelatedError(new Error(wire))).toBe(true)
+  })
+
+  it('detects the Pydantic/Python "Input should be a valid string" variant', () => {
+    /**
+     * @example
+     * isContentArrayRelatedError('messages.0.content: Input should be a valid string')
+     * // -> true
+     */
+    expect(isContentArrayRelatedError('messages.0.content: Input should be a valid string'))
+      .toBe(true)
+    expect(isContentArrayRelatedError('messages.3.content expected string, got list'))
+      .toBe(true)
+  })
+
+  it('does not false-positive on unrelated 400s', () => {
+    expect(isContentArrayRelatedError('Remote sent 400 response: model not found')).toBe(false)
+    expect(isContentArrayRelatedError('Remote sent 401 response: invalid api key')).toBe(false)
+    expect(isContentArrayRelatedError('Tool call failed: invalid schema for function')).toBe(false)
+    expect(isContentArrayRelatedError(undefined)).toBe(false)
+  })
+})

packages/core-agent/src/runtime/llm-service.ts

@@ -7,7 +7,32 @@ import { errorMessageFrom } from '@moeru/std'
 import { stepCountAtLeast } from '@xsai/shared-chat'
 import { streamText } from '@xsai/stream-text'
 
-export function sanitizeMessages(messages: unknown[]): Message[] {
+/**
+ * Normalize chat messages so they match the wire format the active provider
+ * actually accepts, flattening content-part arrays back to plain strings when
+ * the provider can't deserialize arrays.
+ *
+ * Use when:
+ * - Composing the final message list right before handing it to the OpenAI-
+ *   compatible chat SDK.
+ *
+ * Expects:
+ * - `role: 'error'` entries (AIRI-internal markers from the chat UI). They are
+ *   rewritten as user-role narrations so the provider doesn't reject them.
+ * - `content` may be a string, a content-part array, or undefined.
+ *
+ * Returns:
+ * - A new array of `Message` values; original objects are not mutated.
+ *
+ * @param messages - Raw messages from the chat session, may include AIRI's
+ *   `error` role.
+ * @param supportsContentArray - When `false`, force-flatten every array
+ *   content (including text + `image_url` mixes) to a text-only string and
+ *   drop non-text parts. Drives the runtime auto-degrade for strict providers.
+ *   Defaults to `true` to preserve vision/multimodal payloads on capable
+ *   providers.
+ */
+export function sanitizeMessages(messages: unknown[], supportsContentArray: boolean = true): Message[] {
   return messages.map((message: any) => {
     if (message && message.role === 'error') {
       return {
@@ -16,11 +41,27 @@ export function sanitizeMessages(messages: unknown[]): Message[] {
       } as Message
     }
 
-    // NOTICE: Flatten array content for providers (e.g. DeepSeek) that expect string,
-    // not content-part arrays. Skipped when image_url parts are present.
+    // NOTICE:
+    // Flatten array content for providers (e.g. DeepSeek and other Rust/serde-
+    // strict OpenAI-compatible gateways) that only accept `messages[].content`
+    // as a plain string and reject arrays with `Failed to deserialize the JSON
+    // body into the target type: messages[N]: invalid type: sequence, expected
+    // a string`.
+    // Root cause: OpenAI's chat API permits `content` as either `string` or an
+    // array of content parts; some compatible servers only implement the
+    // string variant.
+    // Source/context: https://github.com/moeru-ai/airi/issues/1500
+    // Removal condition: when every supported provider accepts content-part
+    // arrays uniformly (no longer realistic for the OpenAI-compatible
+    // ecosystem, so this is effectively load-bearing).
     if (message && Array.isArray(message.content)) {
       const contentParts = message.content as { type?: string, text?: string }[]
-      if (!contentParts.some(part => part?.type === 'image_url')) {
+      const hasNonTextPart = contentParts.some(part => part?.type && part.type !== 'text')
+      // When the provider supports arrays, only flatten pure-text arrays so we
+      // never silently drop image / audio / file parts on a vision-capable
+      // model. When it doesn't, flatten unconditionally; non-text parts are
+      // dropped because the provider can't carry them anyway.
+      if (!supportsContentArray || !hasNonTextPart) {
         return { ...message, content: contentParts.map(part => part?.text ?? '').join('') } as Message
       }
     }
@@ -34,12 +75,26 @@ export function modelKey(model: string, chatProvider: ChatProvider): string {
 }
 
 export function streamOptionsToolsCompatibilityOk(model: string, chatProvider: ChatProvider, options?: StreamOptions): boolean {
-  if (options?.supportsTools)
-    return true
+  if (options?.supportsTools !== undefined)
+    return options.supportsTools
   const key = modelKey(model, chatProvider)
   return options?.toolsCompatibility?.get(key) !== false
 }
 
+/**
+ * Resolve whether the active model+provider currently supports content-part
+ * arrays. Defaults to `true` so first-time calls keep multimodal payloads;
+ * flips to `false` once {@link isContentArrayRelatedError} has fired on this
+ * model key and the caller has cached the degrade in
+ * {@link StreamOptions.contentArrayCompatibility}.
+ */
+export function streamOptionsContentArrayCompatibilityOk(model: string, chatProvider: ChatProvider, options?: StreamOptions): boolean {
+  if (options?.supportsContentArray !== undefined)
+    return options.supportsContentArray
+  const key = modelKey(model, chatProvider)
+  return options?.contentArrayCompatibility?.get(key) !== false
+}
+
 async function resolveTools(options?: StreamOptions) {
   const tools = typeof options?.tools === 'function'
     ? await options.tools()
@@ -114,7 +169,8 @@ export async function streamFrom({
   builtinToolsResolver,
 }: StreamFromOptions) {
   const chatConfig = chatProvider.chat(model)
-  const sanitized = sanitizeMessages(messages as unknown[])
+  const supportsContentArray = streamOptionsContentArrayCompatibilityOk(model, chatProvider, options)
+  const sanitized = sanitizeMessages(messages as unknown[], supportsContentArray)
 
   const supportedTools = streamOptionsToolsCompatibilityOk(model, chatProvider, options)
   const builtinTools = supportedTools
@@ -224,3 +280,41 @@ export function isToolRelatedError(error: unknown): boolean {
   const message = String(error)
   return TOOLS_RELATED_ERROR_PATTERNS.some(pattern => pattern.test(message))
 }
+
+// Runtime auto-degrade: patterns that indicate the provider rejected
+// content-part arrays and only accepts a plain string for `messages[].content`.
+//
+// The first pattern matches the Rust/serde wire-level error format used by
+// many strict OpenAI-compatible gateways (e.g. DeepSeek-style servers):
+//   "Failed to deserialize the JSON body into the target type:
+//    messages[7]: invalid type: sequence, expected a string at line 1 column …"
+// The second pattern covers Python/Pydantic-style errors like
+//   "messages.0.content: Input should be a valid string"
+// and other variants that surface the same root cause.
+//
+// See: https://github.com/moeru-ai/airi/issues/1500
+const CONTENT_ARRAY_RELATED_ERROR_PATTERNS: RegExp[] = [
+  /messages\[\d+\][^"]*invalid type:\s*sequence,\s*expected\s+a\s+string/i,
+  /messages\.\d+\.content[^"]*(?:expected|should be).*string/i,
+]
+
+/**
+ * Whether the given error indicates the provider rejected content-part arrays
+ * and the caller should auto-degrade to string-only `content` for this model.
+ *
+ * Use when:
+ * - Catching errors thrown by {@link streamFrom} so the chat store can flip
+ *   `contentArrayCompatibility` for the failing model key.
+ *
+ * Expects:
+ * - `error` may be an Error instance, a thrown SDK response object, a string,
+ *   or anything else; we coerce via `String(error)` and pattern-match.
+ *
+ * Returns:
+ * - `true` when the message matches a known "content array unsupported" wire
+ *   format from an OpenAI-compatible gateway, otherwise `false`.
+ */
+export function isContentArrayRelatedError(error: unknown): boolean {
+  const message = String(error)
+  return CONTENT_ARRAY_RELATED_ERROR_PATTERNS.some(pattern => pattern.test(message))
+}

packages/core-agent/src/types/llm.ts

@@ -18,6 +18,23 @@ export interface StreamOptions {
   waitForTools?: boolean
   captureToolErrors?: boolean
   tools?: Tool[] | (() => Promise<Tool[] | undefined>)
+  /**
+   * Per-model runtime cache of whether the provider accepts content-part arrays
+   * (e.g. `[{type:'text',...},{type:'image_url',...}]`) for `messages[].content`.
+   *
+   * Some OpenAI-compatible providers (notably Rust/serde-strict gateways) only
+   * deserialize `content` as a plain string and reject arrays with HTTP 400
+   * `Failed to deserialize the JSON body into the target type: messages[N]:
+   * invalid type: sequence, expected a string`. When a stream surfaces such an
+   * error we set the entry to `false` for the model key and force-flatten on
+   * the next attempt.
+   *
+   * Mirrors {@link toolsCompatibility} for the tool-calling capability.
+   *
+   * See: https://github.com/moeru-ai/airi/issues/1500
+   */
+  contentArrayCompatibility?: Map<string, boolean>
+  supportsContentArray?: boolean
 }
 
 export type BuiltinToolsResolver = (model: string, chatProvider: ChatProvider) => Promise<Tool[]>

packages/stage-ui/src/stores/llm.ts

@@ -3,7 +3,7 @@ import type { WebSocketEvents } from '@proj-airi/server-sdk'
 import type { ChatProvider } from '@xsai-ext/providers/utils'
 import type { Message, Tool } from '@xsai/shared-chat'
 
-import { streamFrom as coreStreamFrom, isToolRelatedError, modelKey } from '@proj-airi/core-agent'
+import { streamFrom as coreStreamFrom, isContentArrayRelatedError, isToolRelatedError, modelKey } from '@proj-airi/core-agent'
 import { listModels } from '@xsai/model'
 import { uniqBy } from 'es-toolkit'
 import { defineStore } from 'pinia'
@@ -14,7 +14,7 @@ import { useLlmToolsStore } from './llm-tools'
 import { useModsServerChannelStore } from './mods/api/channel-server'
 
 export type { StreamEvent, StreamOptions } from '@proj-airi/core-agent'
-export { isToolRelatedError } from '@proj-airi/core-agent'
+export { isContentArrayRelatedError, isToolRelatedError } from '@proj-airi/core-agent'
 
 function toolNameFrom(tool: Tool) {
   const candidate = tool as Tool & {
@@ -29,54 +29,75 @@ function toolNameFrom(tool: Tool) {
 
 export const useLLM = defineStore('llm', () => {
   const toolsCompatibility = ref<Map<string, boolean>>(new Map())
+  const contentArrayCompatibility = ref<Map<string, boolean>>(new Map())
   const modsServerChannelStore = useModsServerChannelStore()
   const llmToolsStore = useLlmToolsStore()
 
   async function stream(model: string, chatProvider: ChatProvider, messages: Message[], options?: StreamOptions) {
     const key = modelKey(model, chatProvider)
-    try {
-      // TODO(@nekomeowww,@shinohara-rin): we should not register the command callback on every stream anyway...
-      const sendSparkCommand = (command: WebSocketEvents['spark:command']) => {
-        // TODO(@nekomeowww): instruct the LLM to understand what destination is.
-        // Currently without skill like prompt injection, many issues occur.
-        // destination mostly are wrong or hallucinated, we need to find a way to make it more reliable.
-        //
-        // For now, since destinations as array will always broadcast to all connected modules/agents, we can set it to
-        // empty array to avoid wrong routing.
-        command.destinations = []
+    // TODO(@nekomeowww,@shinohara-rin): we should not register the command callback on every stream anyway...
+    const sendSparkCommand = (command: WebSocketEvents['spark:command']) => {
+      // TODO(@nekomeowww): instruct the LLM to understand what destination is.
+      // Currently without skill like prompt injection, many issues occur.
+      // destination mostly are wrong or hallucinated, we need to find a way to make it more reliable.
+      //
+      // For now, since destinations as array will always broadcast to all connected modules/agents, we can set it to
+      // empty array to avoid wrong routing.
+      command.destinations = []
 
-        modsServerChannelStore.send({
-          type: 'spark:command',
-          data: command,
-        })
-      }
-
-      await coreStreamFrom({
-        model,
-        chatProvider,
-        messages,
-        options: { ...options, toolsCompatibility: toolsCompatibility.value },
-        builtinToolsResolver: async () => {
-          await llmToolsStore.awaitPendingRegistrations()
-
-          // Reverse twice so later runtime registrations win while original tool order stays stable.
-          return uniqBy(
-            [
-              ...await mcp(),
-              ...await debug(),
-              ...await createSparkCommandTool({ sendSparkCommand }),
-              ...await llmToolsStore.activeTools,
-            ].toReversed(),
-            tool => toolNameFrom(tool) ?? tool,
-          ).toReversed()
-        },
+      modsServerChannelStore.send({
+        type: 'spark:command',
+        data: command,
       })
     }
+
+    const builtinToolsResolver = async () => {
+      await llmToolsStore.awaitPendingRegistrations()
+
+      // Reverse twice so later runtime registrations win while original tool order stays stable.
+      return uniqBy(
+        [
+          ...await mcp(),
+          ...await debug(),
+          ...await createSparkCommandTool({ sendSparkCommand }),
+          ...await llmToolsStore.activeTools,
+        ].toReversed(),
+        tool => toolNameFrom(tool) ?? tool,
+      ).toReversed()
+    }
+
+    const runStream = () => coreStreamFrom({
+      model,
+      chatProvider,
+      messages,
+      options: {
+        ...options,
+        toolsCompatibility: toolsCompatibility.value,
+        contentArrayCompatibility: contentArrayCompatibility.value,
+      },
+      builtinToolsResolver,
+    })
+
+    try {
+      await runStream()
+    }
     catch (err) {
       if (isToolRelatedError(err)) {
         console.warn(`[llm] Auto-disabling tools for "${key}" due to tool-related error`)
         toolsCompatibility.value.set(key, false)
       }
+      // NOTICE:
+      // Auto-degrade content-part arrays to plain strings on the next attempt
+      // when the provider returned the Rust/serde-style "expected a string"
+      // 400. We retry once inline so the user's failing turn recovers without
+      // requiring them to resend; subsequent calls reuse the cached degrade.
+      // See: https://github.com/moeru-ai/airi/issues/1500
+      if (isContentArrayRelatedError(err) && contentArrayCompatibility.value.get(key) !== false) {
+        console.warn(`[llm] Auto-disabling content-part arrays for "${key}" and retrying once`)
+        contentArrayCompatibility.value.set(key, false)
+        await runStream()
+        return
+      }
       throw err
     }
   }