feat(minecraft): add llm trace endpoint and compact prompt payloads

2026-02-18 11:14:39 +08:00
parent 10251a9a80
commit b32c2cac89
6 changed files with 292 additions and 3 deletions
@@ -0,0 +1,69 @@
+---
+name: minecraft-debug-mcp
+description: Operate and debug the live Minecraft bot through its built-in MCP REPL server. Use when work requires starting the bot with `pnpm dev`, connecting to the local MCP endpoint, inspecting cognitive state/logs/history, injecting synthetic chat/events, or running targeted REPL code against the running brain during investigation and development.
+---
+
+# Minecraft Debug MCP
+
+## Overview
+
+Use this skill to run the local bot and interact with its MCP debug interface safely and quickly.
+
+## Quick Start Workflow
+
+1. Run `pnpm dev` from `/Users/rinshinohara/Repo/airi/services/minecraft` and keep it running.
+2. Wait for `MCP REPL server running at http://localhost:3001` in logs.
+3. Connect MCP client to `http://localhost:3001/sse`.
+4. Verify readiness with a read-only call:
+   - Read resource `brain://state`, or
+   - Call tool `get_state`.
+5. Continue with the smallest tool/action that answers the task.
+
+## Execution Rules
+
+- Start read-only, then escalate to mutation tools only when needed.
+- Prefer `get_state`, `get_last_prompt`, and `get_logs` for diagnostics before `execute_repl`.
+- Prefer `get_llm_trace` for structured per-attempt reasoning/content inspection.
+- Keep `execute_repl` snippets minimal and reversible.
+- Use `inject_chat` for conversational simulation and `inject_event` only when specific event-shape testing is required.
+- Treat `inject_chat` as side-effectful: it can trigger actual in-game bot replies/actions.
+- If MCP connection fails, check that `pnpm dev` is still running and port `3001` is free.
+
+## Tooling Strategy
+
+- Use `get_state` to inspect queue/processing state.
+- Use `get_logs` with a small `limit` first.
+- Use `get_last_prompt` to inspect latest LLM input.
+- Use `execute_repl` for deep object inspection or one-off targeted calls on the running brain.
+- Use `inject_chat` to simulate player chat and verify behavior loop.
+- Use `get_llm_trace` to assert planner behavior in automation (for example, detect repeated `await skip()` on specific events).
+
+Read `references/mcp-surface.md` for exact tool/resource names and argument schemas.
+
+## Live-Tested Notes
+
+- `get_state` returns a large variable snapshot; prefer it over REPL for first-pass health checks.
+- `get_last_prompt` can return very large payloads; call only when prompt-level debugging is needed.
+- `execute_repl` returns a structured result where `returnValue` is stringified; parse mentally as display output, not typed JSON.
+- `get_logs(limit=10)` is enough to verify whether an injected event reached planner/executor.
+- `get_llm_trace(limit, turnId?)` gives structured attempt-level trace data (messages, content, reasoning, usage, duration).
+- `get_last_prompt` and `get_llm_trace` are compacted for MCP: system prompt/system-role messages are omitted to reduce token cost.
+- If environment summary shows `"SOMETHING WENT WRONG, YOU SHOULD NOTIFY THE USER OF THIS"`, treat it as degraded runtime context and avoid high-confidence world actions.
+
+## Live Testing Workflow
+
+1. Confirm MCP health:
+   - Call `get_state`.
+2. Capture baseline inventory:
+   - `execute_repl` with `query.inventory().list().map(i => ({ name: i.name, count: i.count }))`.
+3. Trigger a task through normal cognition path:
+   - Call `inject_chat` with a clear instruction (example: "please gather 3 dirt blocks").
+4. Verify execution trace:
+   - Call `get_logs(limit=10)` and check for:
+     - bot acknowledgement chat
+     - action tool feedback (for example `collectBlocks`)
+     - planner result summary
+   - Call `get_llm_trace(limit=5)` when you need exact model output/reasoning for assertions.
+5. Re-check inventory using the same REPL snippet and compare against baseline.
+
+Use this workflow when validating behavior changes, tool wiring, or regressions in planning/execution loops.
@@ -0,0 +1,4 @@
+interface:
+  display_name: 'Minecraft Debug MCP'
+  short_description: 'Operate the live Minecraft debug MCP bot safely'
+  default_prompt: 'Start the bot with pnpm dev, connect to the minecraft-debug MCP, inspect brain state/logs, and execute focused debug actions.'
@@ -0,0 +1,98 @@
+# Minecraft Debug MCP Surface
+
+Implementation source: `/Users/rinshinohara/Repo/airi/services/minecraft/src/debug/mcp-repl-server.ts`.
+
+## Endpoint
+
+- Base server: `http://localhost:3001`
+- MCP endpoint: `http://localhost:3001/sse`
+- SSE fallback endpoint: `GET /sse` + `POST /messages`
+
+The bot starts this server during normal runtime from:
+- `/Users/rinshinohara/Repo/airi/services/minecraft/src/cognitive/index.ts`
+
+## Resources
+
+- `brain://state`
+  - Summary state: processing, queue length, turn, give-up timer.
+- `brain://context`
+  - Current context view text.
+- `brain://history`
+  - Conversation history JSON.
+- `brain://logs`
+  - Latest LLM log entries JSON (last 50 in resource output).
+
+## Tools
+
+- `get_state()`
+  - Returns current REPL/brain state JSON.
+
+- `get_last_prompt()`
+  - Returns latest LLM input JSON.
+  - Returns error when no prompt exists yet.
+  - Compacted payload: omits `systemPrompt` and drops `messages` items with `role: "system"`.
+
+- `get_logs(limit?: number)`
+  - Returns recent LLM logs; start with small limits.
+
+- `get_llm_trace(limit?: number, turnId?: number)`
+  - Returns structured LLM trace entries captured per attempt.
+  - Includes: turn/source metadata, messages, generated content, reasoning (if available), token usage, and duration.
+  - Use `turnId` to isolate trace for one injected test event.
+  - Compacted payload: drops `messages` items with `role: "system"` to save tokens.
+
+- `execute_repl(code: string)`
+  - Executes debug REPL code in running brain context.
+  - Use for focused inspection/action only.
+
+- `inject_chat(username: string, message: string)`
+  - Injects a synthetic chat perception event.
+
+- `inject_event(type, payload, source)`
+  - `type`: `perception | feedback | world_update | system_alert`
+  - `source.type`: `minecraft | airi | system`
+  - `source.id`: string
+  - Use only with deliberate, test-specific payloads.
+
+## Troubleshooting
+
+- Connection refused:
+  - Ensure `pnpm dev` is running in the service directory.
+  - Confirm logs include `MCP REPL server running at http://localhost:3001`.
+- 404/invalid endpoint:
+  - Use `/sse` as MCP entrypoint.
+- Empty prompt/logs:
+  - Trigger activity first (for example via `inject_chat`) and retry `get_last_prompt` or `get_logs`.
+
+## Live-Tested Behavior Notes
+
+- `inject_chat` is not a passive write: it enters the normal cognition pipeline and can cause the bot to send chat/actions.
+- `get_last_prompt` may be very large (full system prompt + history); avoid repeated calls unless needed.
+- `get_last_prompt` is now MCP-compacted (no raw system prompt text), which makes it cheaper for automation checks.
+- `execute_repl` response includes metadata (`source`, `durationMs`, `actions`, `logs`) and a stringified `returnValue`.
+- Log verification pattern that worked reliably:
+  1. `inject_chat(...)`
+  2. `get_logs(limit: 10)`
+  3. Confirm sequence: `turn_input` -> `llm_attempt` -> `feedback` -> `planner_result`
+
+## Repeatable Smoke Test Recipe
+
+Use this exact sequence for fast live validation:
+
+1. Baseline
+   - `get_state()`
+   - `execute_repl("query.inventory().list().map(i => ({ name: i.name, count: i.count }))")`
+2. Task trigger
+   - `inject_chat({ username: \"codex-live-test\", message: \"please gather 3 dirt blocks\" })`
+3. Execution proof
+   - `get_logs({ limit: 10 })`
+   - Expect acknowledgement chat + `collectBlocks` success feedback + planner summary.
+   - `get_llm_trace({ limit: 5 })`
+   - Assert expected LLM behavior (for example response code, or repeated `await skip()`).
+   - Assert trace payload does not include `role: "system"` entries.
+4. Outcome proof
+   - Run the same inventory `execute_repl` call again and compare item counts.
+
+## Runtime Caveat Seen Live
+
+- If a turn includes `Environment: SOMETHING WENT WRONG, YOU SHOULD NOTIFY THE USER OF THIS`, treat the world snapshot as degraded and avoid issuing risky autonomous actions until context stabilizes.
@@ -75,6 +75,26 @@ interface LlmInputSnapshot {
  attempt: number
 }

+interface LlmTraceEntry {
+  id: number
+  turnId: number
+  timestamp: number
+  eventType: string
+  sourceType: string
+  sourceId: string
+  attempt: number
+  model: string
+  messages: Message[]
+  content: string
+  reasoning?: string
+  usage?: {
+    prompt_tokens?: number
+    completion_tokens?: number
+    total_tokens?: number
+  }
+  durationMs: number
+}
+
 interface RuntimeInputEnvelope {
  id: number
  turnId: number
@@ -137,6 +157,8 @@ export class Brain {
  private runtimeMineflayer: MineflayerWithAgents | null = null
  private readonly llmLogEntries: LlmLogEntry[] = []
  private llmLogIdCounter = 0
+  private readonly llmTraceEntries: LlmTraceEntry[] = []
+  private llmTraceIdCounter = 0
  private turnCounter = 0
  private currentInputEnvelope: RuntimeInputEnvelope | null = null
  private readonly llmLogRuntime = createLlmLogRuntime(() => this.llmLogEntries)
@@ -277,6 +299,20 @@ export class Brain {
    return entries.slice(-Math.floor(limit))
  }

+  public getLlmTrace(limit?: number, turnId?: number): LlmTraceEntry[] {
+    let entries = [...this.llmTraceEntries]
+    if (typeof turnId === 'number' && Number.isFinite(turnId)) {
+      const normalizedTurnId = Math.floor(turnId)
+      entries = entries.filter(entry => entry.turnId === normalizedTurnId)
+    }
+
+    if (typeof limit === 'number' && Number.isFinite(limit) && limit > 0) {
+      entries = entries.slice(-Math.floor(limit))
+    }
+
+    return JSON.parse(JSON.stringify(entries)) as LlmTraceEntry[]
+  }
+
  public async injectDebugEvent(event: BotEvent): Promise<void> {
    if (!this.runtimeMineflayer) {
      throw new Error('Brain runtime is not initialized yet')
@@ -636,6 +672,24 @@ export class Brain {
          model: config.openai.model,
          duration: Date.now() - traceStart,
        })
+        this.llmTraceEntries.push({
+          id: ++this.llmTraceIdCounter,
+          turnId,
+          timestamp: Date.now(),
+          eventType: event.type,
+          sourceType: event.source.type,
+          sourceId: event.source.id,
+          attempt,
+          model: config.openai.model,
+          messages: this.cloneMessages(messages),
+          content,
+          reasoning,
+          usage: llmResult.usage,
+          durationMs: Date.now() - traceStart,
+        })
+        if (this.llmTraceEntries.length > 500) {
+          this.llmTraceEntries.shift()
+        }
        this.currentInputEnvelope.llm = {
          attempt,
          model: config.openai.model,
@@ -53,8 +53,27 @@ describe('mcpReplServer', () => {
      executeDebugRepl: vi.fn().mockResolvedValue({ result: 'success' }),
      injectDebugEvent: vi.fn().mockResolvedValue(undefined),
      getReplState: vi.fn().mockReturnValue({ variables: [], updatedAt: 0 }),
-      getLastLlmInput: vi.fn().mockReturnValue({ systemPrompt: 'sys', userMessage: 'user' }),
+      getLastLlmInput: vi.fn().mockReturnValue({
+        systemPrompt: 'sys',
+        userMessage: 'user',
+        messages: [
+          { role: 'system', content: 'sys' },
+          { role: 'user', content: 'user' },
+        ],
+        conversationHistory: [],
+        updatedAt: 0,
+        attempt: 1,
+      }),
      getLlmLogs: vi.fn().mockReturnValue([{ id: 1, text: 'log' }]),
+      getLlmTrace: vi.fn().mockReturnValue([{
+        id: 1,
+        turnId: 1,
+        content: 'await skip()',
+        messages: [
+          { role: 'system', content: 'sys' },
+          { role: 'user', content: 'u' },
+        ],
+      }]),
    } as unknown as Brain

    server = new McpReplServer(brain)
@@ -75,6 +94,7 @@ describe('mcpReplServer', () => {
    expect(mocks.tool).toHaveBeenCalledWith('get_state', expect.anything(), expect.any(Function))
    expect(mocks.tool).toHaveBeenCalledWith('get_last_prompt', expect.anything(), expect.any(Function))
    expect(mocks.tool).toHaveBeenCalledWith('get_logs', expect.anything(), expect.any(Function))
+    expect(mocks.tool).toHaveBeenCalledWith('get_llm_trace', expect.anything(), expect.any(Function))
  })

  it('executes repl via tool handler', async () => {
@@ -117,9 +137,12 @@ describe('mcpReplServer', () => {
    const handler = toolCall[2]

    const result = await handler({})
+    const text = result.content[0].text as string

    expect(brain.getLastLlmInput).toHaveBeenCalled()
-    expect(result.content[0].text).toContain('sys')
+    expect(text).toContain('user')
+    expect(text).not.toContain('systemPrompt')
+    expect(text).not.toContain('"role":"system"')
  })

  it('gets logs via tool handler', async () => {
@@ -131,4 +154,17 @@ describe('mcpReplServer', () => {
    expect(brain.getLlmLogs).toHaveBeenCalledWith(10)
    expect(result.content[0].text).toContain('log')
  })
+
+  it('gets llm trace via tool handler', async () => {
+    const toolCall = mocks.tool.mock.calls.find(call => call[0] === 'get_llm_trace')
+    const handler = toolCall[2]
+
+    const result = await handler({ limit: 5, turnId: 3 })
+    const text = result.content[0].text as string
+
+    expect(brain.getLlmTrace).toHaveBeenCalledWith(5, 3)
+    expect(text).toContain('await skip()')
+    expect(text).toContain('"role":"user"')
+    expect(text).not.toContain('"role":"system"')
+  })
 })
@@ -198,8 +198,17 @@ export class McpReplServer {
            isError: true,
          }
        }
+        const {
+          systemPrompt: _systemPrompt,
+          messages,
+          ...rest
+        } = result
+        const compactMessages = messages.filter(message => message.role !== 'system')
        return {
-          content: [{ type: 'text', text: JSON.stringify(result) }],
+          content: [{ type: 'text', text: JSON.stringify({
+            ...rest,
+            messages: compactMessages,
+          }) }],
        }
      },
    )
@@ -216,6 +225,25 @@ export class McpReplServer {
        }
      },
    )
+
+    this.mcpServer.tool(
+      'get_llm_trace',
+      {
+        limit: z.number().optional(),
+        turnId: z.number().optional(),
+      },
+      async ({ limit, turnId }) => {
+        const result = this.brain
+          .getLlmTrace(limit, turnId)
+          .map(entry => ({
+            ...entry,
+            messages: entry.messages.filter(message => message.role !== 'system'),
+          }))
+        return {
+          content: [{ type: 'text', text: JSON.stringify(result) }],
+        }
+      },
+    )
  }

  start(): void {