feat(minecraft): use js REPL for llm interface, would could possibly go wrong

services/minecraft/src/cognitive/action/llm-actions.ts

@@ -7,6 +7,7 @@ import { collectBlock } from '../../skills/actions/collect-block'
 import { discard, equip, putInChest, takeFromChest } from '../../skills/actions/inventory'
 import { activateNearestBlock, breakBlockAt, placeBlock } from '../../skills/actions/world-interactions'
 import { ActionError } from '../../utils/errors'
+import { javascriptRepl } from '../../utils/javascript-repl'
 import { useLogger } from '../../utils/logger'
 import { describeRecipePlan, planRecipe } from '../../utils/recipe-planner'
 
@@ -37,6 +38,15 @@ export const actionsList: Action[] = [
       return `Sent message: "${message}"`
     },
   },
+  {
+    name: 'eval',
+    description: 'Evaluate JavaScript code in a persistent REPL context. Use this for quick calculations or temporary memory.',
+    execution: 'sync',
+    schema: z.object({
+      code: z.string().min(1).describe('JavaScript source code to evaluate.'),
+    }),
+    perform: () => (code: string): string => javascriptRepl.evaluate(code),
+  },
   // {\n  //   name: 'setReflexMode',
   //   description: 'Set (or clear) your reflex mode override. Use work/wander to disable idle-only reflex behaviors. Set override to null to return to automatic mode selection.',
   //   execution: 'sequential',

services/minecraft/src/cognitive/action/types/index.ts

@@ -9,7 +9,7 @@ export interface ActionInstruction {
 
 /**
  * LLM response format for the stateful agent.
- * Single action per turn, model uses native reasoning (no thought field).
+ * Legacy JSON response format retained for backward compatibility.
  */
 export interface LLMResponse {
   action: ActionInstruction

services/minecraft/src/cognitive/conscious/brain.ts

@@ -11,6 +11,7 @@ import type { ActionInstruction } from '../action/types'
 import { config } from '../../composables/config'
 import { DebugService } from '../../debug'
 import { buildConsciousContextView } from './context-view'
+import { JavaScriptPlanner } from './js-planner'
 import { LLMAgent } from './llm-agent'
 import {
   extractJsonCandidate,
@@ -42,6 +43,7 @@ interface QueuedEvent {
 
 export class Brain {
   private debugService: DebugService
+  private readonly planner = new JavaScriptPlanner()
 
   // State
   private queue: QueuedEvent[] = []
@@ -177,7 +179,6 @@ export class Brain {
 
         const llmResult = await this.deps.llmAgent.callLLM({
           messages,
-          responseFormat: { type: 'json_object' },
         })
 
         const content = llmResult.text
@@ -230,8 +231,7 @@ export class Brain {
     }
 
     try {
-      const parsed = this.parseResponse(result)
-      const action = parsed.action
+      const actions = this.parseResponse(result)
 
       // Only append to conversation history after successful parsing (avoid dirty data on retry)
       this.conversationHistory.push({ role: 'user', content: userMessage })
@@ -243,34 +243,16 @@ export class Brain {
         ...(capturedReasoning && { reasoning: capturedReasoning }),
       } as Message)
 
-      if (action.tool === 'skip') {
+      if (actions.length === 1 && actions[0]?.tool === 'skip') {
         this.deps.logger.log('INFO', 'Brain: Skipping turn (observing)')
         return
       }
 
-      this.deps.logger.log('INFO', `Brain: Decided action: ${action.tool}`, { params: action.params })
+      this.deps.logger.log('INFO', `Brain: Planned ${actions.length} action(s)`, {
+        actions: actions.map(action => ({ tool: action.tool, params: action.params })),
+      })
 
-      // Check if action is read-only
-      const availableActions = this.deps.taskExecutor.getAvailableActions()
-      const actionDef = availableActions.find(a => a.name === action.tool)
-
-      let token: CancellationToken | undefined
-
-      if (actionDef?.readonly) {
-        // Read-only Actions: Do not cancel background/physical actions
-        // Can be executed in parallel with physical actions
-        token = undefined
-      } else {
-        // Physical Actions: Cancel previous background action
-        if (this.currentCancellationToken) {
-          this.currentCancellationToken.cancel()
-        }
-        this.currentCancellationToken = createCancellationToken()
-        token = this.currentCancellationToken
-      }
-
-      // Execute
-      void this.deps.taskExecutor.executeAction(action, token)
+      this.executePlannedActions(actions)
 
     } catch (err) {
       this.deps.logger.withError(err).error('Brain: Failed to execute decision')
@@ -316,12 +298,56 @@ export class Brain {
     return parts.join('\n\n')
   }
 
-  private parseResponse(content: string): BrainResponse {
-    const jsonStr = extractJsonCandidate(content)
-    try {
-      return JSON.parse(jsonStr)
-    } catch (e) {
-      throw new Error(`Invalid JSON response: ${content.substring(0, 100)}...`)
+  private parseResponse(content: string): ActionInstruction[] {
+    const availableActions = this.deps.taskExecutor.getAvailableActions()
+    const trimmed = content.trim()
+    if (trimmed.startsWith('{')) {
+      try {
+        const parsed = JSON.parse(extractJsonCandidate(content)) as BrainResponse
+        if (parsed?.action?.tool === 'skip') {
+          return [{ tool: 'skip', params: {} }]
+        }
+
+        if (parsed?.action?.tool) {
+          const actionDef = availableActions.find(action => action.name === parsed.action.tool)
+          if (!actionDef) {
+            throw new Error(`Unknown tool in legacy JSON response: ${parsed.action.tool}`)
+          }
+
+          return [{
+            tool: actionDef.name,
+            params: actionDef.schema.parse(parsed.action.params ?? {}),
+          }]
+        }
+      } catch {
+        // Fallback to JS planner parsing.
+      }
     }
+
+    return this.planner.evaluate(content, availableActions)
+  }
+
+  private executePlannedActions(actions: ActionInstruction[]): void {
+    const availableActions = this.deps.taskExecutor.getAvailableActions()
+    const actionDefs = new Map(availableActions.map(action => [action.name, action]))
+    const hasNonReadonlyAction = actions.some(action => !actionDefs.get(action.tool)?.readonly && action.tool !== 'skip')
+
+    let token: CancellationToken | undefined
+    if (hasNonReadonlyAction) {
+      if (this.currentCancellationToken) {
+        this.currentCancellationToken.cancel()
+      }
+      this.currentCancellationToken = createCancellationToken()
+      token = this.currentCancellationToken
+    }
+
+    void (async () => {
+      for (const action of actions) {
+        if (token?.isCancelled)
+          return
+
+        await this.deps.taskExecutor.executeAction(action, token)
+      }
+    })()
   }
 }

services/minecraft/src/cognitive/conscious/js-planner.test.ts

@@ -0,0 +1,74 @@
+import type { Action } from '../../libs/mineflayer/action'
+
+import { describe, expect, it } from 'vitest'
+import { z } from 'zod'
+
+import { JavaScriptPlanner } from './js-planner'
+
+function createAction(name: string, schema: Action['schema']): Action {
+  return {
+    name,
+    description: `${name} tool`,
+    execution: 'sync',
+    schema,
+    perform: () => () => '',
+  }
+}
+
+const actions: Action[] = [
+  createAction('chat', z.object({ message: z.string() })),
+  createAction('goToPlayer', z.object({
+    player_name: z.string(),
+    closeness: z.number().min(0),
+  })),
+]
+
+describe('JavaScriptPlanner', () => {
+  it('maps positional and object tool args into validated action instructions', () => {
+    const planner = new JavaScriptPlanner()
+    const planned = planner.evaluate(`
+      chat("hello")
+      goToPlayer({ player_name: "Alex", closeness: 2 })
+    `, actions)
+
+    expect(planned).toEqual([
+      { tool: 'chat', params: { message: 'hello' } },
+      { tool: 'goToPlayer', params: { player_name: 'Alex', closeness: 2 } },
+    ])
+  })
+
+  it('supports dynamic dispatch with use(toolName, params)', () => {
+    const planner = new JavaScriptPlanner()
+    const planned = planner.evaluate(`use("chat", { message: "via-use" })`, actions)
+
+    expect(planned).toEqual([{ tool: 'chat', params: { message: 'via-use' } }])
+  })
+
+  it('persists script variables across turns', () => {
+    const planner = new JavaScriptPlanner()
+
+    planner.evaluate('const count = 2', actions)
+    const planned = planner.evaluate('chat("count=" + count)', actions)
+
+    expect(planned).toEqual([{ tool: 'chat', params: { message: 'count=2' } }])
+  })
+
+  it('returns skip when no tool is called', () => {
+    const planner = new JavaScriptPlanner()
+    const planned = planner.evaluate('const x = 1 + 1', actions)
+
+    expect(planned).toEqual([{ tool: 'skip', params: {} }])
+  })
+
+  it('rejects mixed skip + tool calls', () => {
+    const planner = new JavaScriptPlanner()
+
+    expect(() => planner.evaluate('skip(); chat("oops")', actions)).toThrow(/skip\(\) cannot be mixed/i)
+  })
+
+  it('enforces timeout on long-running scripts', () => {
+    const planner = new JavaScriptPlanner({ timeoutMs: 20 })
+
+    expect(() => planner.evaluate('while (true) {}', actions)).toThrow(/Script execution timed out/i)
+  })
+})

services/minecraft/src/cognitive/conscious/js-planner.ts

@@ -0,0 +1,173 @@
+import type { Action } from '../../libs/mineflayer/action'
+import type { ActionInstruction } from '../action/types'
+
+import { inspect } from 'node:util'
+import vm from 'node:vm'
+
+interface JavaScriptPlannerOptions {
+  timeoutMs?: number
+  maxActionsPerTurn?: number
+}
+
+interface ActionIntent {
+  index: number
+  params: Record<string, unknown>
+  tool: string
+}
+
+interface ActivePlannerRun {
+  actions: ActionInstruction[]
+  actionsByName: Map<string, Action>
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === 'object' && value !== null && !Array.isArray(value)
+}
+
+export function extractJavaScriptCandidate(input: string): string {
+  const trimmed = input.trim()
+  const fenced = trimmed.match(/^```(?:js|javascript|ts|typescript)?\s*([\s\S]*?)\s*```$/i)
+  if (fenced?.[1])
+    return fenced[1].trim()
+
+  return trimmed
+}
+
+export class JavaScriptPlanner {
+  private readonly context: vm.Context
+  private activeRun: ActivePlannerRun | null = null
+  private readonly maxActionsPerTurn: number
+  private readonly sandbox: Record<string, unknown>
+  private readonly timeoutMs: number
+
+  constructor(options: JavaScriptPlannerOptions = {}) {
+    this.timeoutMs = options.timeoutMs ?? 750
+    this.maxActionsPerTurn = options.maxActionsPerTurn ?? 5
+    this.sandbox = {}
+    this.context = vm.createContext(this.sandbox)
+    this.installBuiltins()
+  }
+
+  public evaluate(content: string, availableActions: Action[]): ActionInstruction[] {
+    const script = extractJavaScriptCandidate(content)
+    const run: ActivePlannerRun = {
+      actions: [],
+      actionsByName: new Map(availableActions.map(action => [action.name, action])),
+    }
+
+    this.activeRun = run
+    this.installActionTools(availableActions)
+
+    try {
+      const result = new vm.Script(script).runInContext(this.context, { timeout: this.timeoutMs })
+      if (typeof result !== 'undefined' && run.actions.length === 0) {
+        // Keep this visible in traces for debugging, without affecting behavior.
+        this.sandbox.__lastEvalResult = inspect(result, { depth: 2, breakLength: 100 })
+      }
+    }
+    finally {
+      this.activeRun = null
+    }
+
+    if (run.actions.length === 0)
+      return [{ tool: 'skip', params: {} }]
+
+    const containsSkip = run.actions.some(action => action.tool === 'skip')
+    if (containsSkip && run.actions.length > 1) {
+      throw new Error('skip() cannot be mixed with other tool calls in the same script')
+    }
+
+    return run.actions
+  }
+
+  private installBuiltins(): void {
+    this.defineGlobalTool('skip', () => this.enqueueAction('skip', {}))
+    this.defineGlobalTool('use', (toolName: unknown, params?: unknown) => {
+      if (typeof toolName !== 'string' || toolName.length === 0) {
+        throw new Error('use(toolName, params) requires a non-empty string toolName')
+      }
+
+      const mappedParams = isRecord(params) ? params : {}
+      return this.enqueueAction(toolName, mappedParams)
+    })
+  }
+
+  private installActionTools(availableActions: Action[]): void {
+    for (const action of availableActions) {
+      this.defineGlobalTool(action.name, (...args: unknown[]) => {
+        const params = this.mapArgsToParams(action, args)
+        return this.enqueueAction(action.name, params)
+      })
+    }
+  }
+
+  private mapArgsToParams(action: Action, args: unknown[]): Record<string, unknown> {
+    const shape = action.schema.shape as Record<string, unknown>
+    const keys = Object.keys(shape)
+
+    if (keys.length === 0)
+      return {}
+
+    if (args.length === 1) {
+      const [firstArg] = args
+      if (isRecord(firstArg))
+        return firstArg
+
+      if (keys.length === 1)
+        return { [keys[0]]: firstArg }
+    }
+
+    const params: Record<string, unknown> = {}
+    for (const [index, key] of keys.entries()) {
+      if (index >= args.length)
+        break
+      params[key] = args[index]
+    }
+
+    return params
+  }
+
+  private enqueueAction(tool: string, params: Record<string, unknown>): ActionIntent {
+    if (!this.activeRun) {
+      throw new Error('Tool calls are only allowed during planner evaluation')
+    }
+
+    if (this.activeRun.actions.length >= this.maxActionsPerTurn) {
+      throw new Error(`Action limit exceeded: max ${this.maxActionsPerTurn} actions per turn`)
+    }
+
+    if (tool !== 'skip') {
+      const action = this.activeRun.actionsByName.get(tool)
+      if (!action)
+        throw new Error(`Unknown tool: ${tool}`)
+
+      const parsed = action.schema.parse(params)
+      this.activeRun.actions.push({
+        tool,
+        params: parsed,
+      })
+    }
+    else {
+      this.activeRun.actions.push({ tool: 'skip', params: {} })
+    }
+
+    const intent: ActionIntent = {
+      index: this.activeRun.actions.length - 1,
+      tool,
+      params,
+    }
+    return Object.freeze(intent)
+  }
+
+  private defineGlobalTool(name: string, fn: (...args: unknown[]) => unknown): void {
+    if (Object.prototype.hasOwnProperty.call(this.sandbox, name))
+      return
+
+    Object.defineProperty(this.sandbox, name, {
+      value: fn,
+      configurable: false,
+      enumerable: true,
+      writable: false,
+    })
+  }
+}

services/minecraft/src/cognitive/conscious/prompts/brain-prompt.ts

@@ -36,17 +36,28 @@ function getZodTypeName(def: any): string {
 
 export function generateBrainSystemPrompt(availableActions: Action[]): string {
   const toolsFormatted = availableActions.map((a) => {
+    const paramKeys = Object.keys(a.schema.shape)
+    const positionalSignature = paramKeys.length > 0 ? `${a.name}(${paramKeys.join(', ')})` : `${a.name}()`
+    const objectSignature = paramKeys.length > 0 ? `${a.name}({ ${paramKeys.join(', ')} })` : `${a.name}()`
+
     let params = ''
     if (a.schema && 'shape' in a.schema) {
       params = Object.entries(a.schema.shape).map(([key, val]: [string, any]) => {
         const def = val._def
         const type = getZodTypeName(def)
-        const desc = val.description ? ` -> ${val.description}` : ''
-        return `- ${key}: ${type}${desc}`
+        const desc = val.description ? ` - ${val.description}` : ''
+        return ` * @param {${type}} ${key}${desc}`
       }).join('\n')
     }
 
-    return `[${a.name}]\nDescription: ${a.description}\n${params}`
+    const body = params ? `\n${params}\n ` : '\n '
+    return `/**
+ * ${a.description}
+ * @function ${a.name}
+ * @signature ${positionalSignature}
+ * @signature ${objectSignature}${body}*/
+${positionalSignature}
+`
   }).join('\n\n')
 
   return `
@@ -55,7 +66,7 @@ You are an autonomous agent playing Minecraft.
 
 # Self-Knowledge & Capabilities
 1. **Stateful Existence**: You maintain a memory of the conversation, but it's crucial to be aware that old history messages are less relevant than recent.
-2. **One Action Per Turn**: You can perform exactly one action at a time. If you decide to act, you must wait for its feedback before acting again.
+2. **Action Script Per Turn**: You can output one JavaScript script each turn, and it can queue multiple tool calls.
 3. **Interruption**: The world is real-time. Events (chat, damage, etc.) may happen *while* you are performing an action.
    - If a new critical event occurs, you may need to change your plans.
    - Feedback for your actions will arrive as a message starting with \`[FEEDBACK]\`.
@@ -66,37 +77,39 @@ You are an autonomous agent playing Minecraft.
    - It's possible for a fresh event to reach you while you're in the middle of a action, in that case, remember the action is still running in the background.
    - If the new situation requires you to change plan, you can use the stop tool to stop background actions or initiate a new one, which will automatically replace the old one.
    - Feel free to send chats while background actions are running, it will not interrupt them.
+6. **JavaScript Scratchpad**: The \`eval\` tool is available, you may use it as a persistent scratchpad for calculations and short-term memory.
+   - Values defined in \`eval\` persist across turns.
+   - \`eval\` has a timeout, so avoid long-running code.
+   - Prefer direct world tools for world interaction; use \`eval\` for reasoning support.
+7. **Planner Runtime**: Your script runs in a persistent JavaScript context with a timeout.
+   - Tool functions (listed below) queue actions; they do not execute instantly.
+   - The runtime validates queued actions before execution.
+   - Maximum actions per turn: 5.
 
 # Available Tools
 You must use the following tools to interact with the world.
-You cannot make up tools. You must use the JSON format described below.
+You cannot make up tools.
 
 ${toolsFormatted}
 
 # Response Format
-You must respond with valid JSON only. Do not include markdown code blocks (like \`\`\`json).
-Your response determines your single action for this turn.
+You must respond with JavaScript only (no markdown code fences).
+Call tool functions directly to queue actions.
+If you want to do nothing, call \`skip()\`.
+You can also use \`use(toolName, paramsObject)\` for dynamic tool calls.
 
-Schema:
-{
-  "action": {
-    "tool": "toolName",
-    "params": { "key": "value" }
-  }
-}
-
-OR, if you want to do nothing (if you want to wait for something to happen, or to ignore):
-
-{
-  "action": {
-    "tool": "skip",
-    "params": {}
-  }
-}
+Examples:
+- \`chat("hello")\`
+- \`goToPlayer("Alex", 2)\`
+- \`goToPlayer({ player_name: "Alex", closeness: 2 })\`
+- \`const steps = 3; for (let i = 0; i < steps; i++) chat("step " + i)\`
+- \`skip()\`
 
 # Rules
 - **Native Reasoning**: You can think before outputting your action.
-- **Strict JSON**: Output ONLY the JSON object. No preamble, no postscript.
+- **Strict JavaScript Output**: Output ONLY executable JavaScript. Comments are possible but discouraged and will be ignored.
 - **Handling Feedback**: When you perform an action, you will see a \`[FEEDBACK]\` message in the history later with the result. Use this to verify success.
+- **Tool Choice**: If a dedicated tool exists for a task, use it. Use \`eval\` for computation, scratch memory, or inspecting previously stored variables.
+- **Skip Rule**: If you call \`skip()\`, do not call any other tool in the same turn.
 `
 }

services/minecraft/src/utils/javascript-repl.test.ts

@@ -0,0 +1,19 @@
+import { describe, expect, it } from 'vitest'
+
+import { JavaScriptRepl } from './javascript-repl'
+
+describe('javascript-repl', () => {
+  it('persists variables between eval calls', () => {
+    const repl = new JavaScriptRepl()
+
+    repl.evaluate('const var_foo = 6 * 7')
+
+    expect(repl.evaluate('var_foo')).toBe('42')
+  })
+
+  it('times out long-running scripts', () => {
+    const repl = new JavaScriptRepl({ timeoutMs: 20 })
+
+    expect(() => repl.evaluate('while (true) {}')).toThrow(/Script execution timed out/i)
+  })
+})

services/minecraft/src/utils/javascript-repl.ts

@@ -0,0 +1,30 @@
+import { inspect } from 'node:util'
+import vm from 'node:vm'
+
+interface JavaScriptReplOptions {
+  timeoutMs?: number
+}
+
+export class JavaScriptRepl {
+  private readonly context: vm.Context
+  private readonly timeoutMs: number
+
+  constructor(options: JavaScriptReplOptions = {}) {
+    this.timeoutMs = options.timeoutMs ?? 500
+    this.context = vm.createContext({})
+  }
+
+  public evaluate(code: string): string {
+    const script = new vm.Script(code)
+    const result = script.runInContext(this.context, { timeout: this.timeoutMs })
+
+    if (result === undefined)
+      return 'undefined'
+
+    return typeof result === 'string'
+      ? result
+      : inspect(result, { depth: 4, maxArrayLength: 100, breakLength: 120 })
+  }
+}
+
+export const javascriptRepl = new JavaScriptRepl()