diff --git a/services/minecraft/src/cognitive/action/llm-actions.ts b/services/minecraft/src/cognitive/action/llm-actions.ts index 4c756a9fd..1e633cf3a 100644 --- a/services/minecraft/src/cognitive/action/llm-actions.ts +++ b/services/minecraft/src/cognitive/action/llm-actions.ts @@ -7,6 +7,7 @@ import { collectBlock } from '../../skills/actions/collect-block' import { discard, equip, putInChest, takeFromChest } from '../../skills/actions/inventory' import { activateNearestBlock, breakBlockAt, placeBlock } from '../../skills/actions/world-interactions' import { ActionError } from '../../utils/errors' +import { javascriptRepl } from '../../utils/javascript-repl' import { useLogger } from '../../utils/logger' import { describeRecipePlan, planRecipe } from '../../utils/recipe-planner' @@ -37,6 +38,15 @@ export const actionsList: Action[] = [ return `Sent message: "${message}"` }, }, + { + name: 'eval', + description: 'Evaluate JavaScript code in a persistent REPL context. Use this for quick calculations or temporary memory.', + execution: 'sync', + schema: z.object({ + code: z.string().min(1).describe('JavaScript source code to evaluate.'), + }), + perform: () => (code: string): string => javascriptRepl.evaluate(code), + }, // {\n // name: 'setReflexMode', // description: 'Set (or clear) your reflex mode override. Use work/wander to disable idle-only reflex behaviors. Set override to null to return to automatic mode selection.', // execution: 'sequential', diff --git a/services/minecraft/src/cognitive/action/types/index.ts b/services/minecraft/src/cognitive/action/types/index.ts index d0cc9185f..98851e631 100644 --- a/services/minecraft/src/cognitive/action/types/index.ts +++ b/services/minecraft/src/cognitive/action/types/index.ts @@ -9,7 +9,7 @@ export interface ActionInstruction { /** * LLM response format for the stateful agent. - * Single action per turn, model uses native reasoning (no thought field). + * Legacy JSON response format retained for backward compatibility. */ export interface LLMResponse { action: ActionInstruction diff --git a/services/minecraft/src/cognitive/conscious/brain.ts b/services/minecraft/src/cognitive/conscious/brain.ts index 0bc41c5d1..98e212a62 100644 --- a/services/minecraft/src/cognitive/conscious/brain.ts +++ b/services/minecraft/src/cognitive/conscious/brain.ts @@ -11,6 +11,7 @@ import type { ActionInstruction } from '../action/types' import { config } from '../../composables/config' import { DebugService } from '../../debug' import { buildConsciousContextView } from './context-view' +import { JavaScriptPlanner } from './js-planner' import { LLMAgent } from './llm-agent' import { extractJsonCandidate, @@ -42,6 +43,7 @@ interface QueuedEvent { export class Brain { private debugService: DebugService + private readonly planner = new JavaScriptPlanner() // State private queue: QueuedEvent[] = [] @@ -177,7 +179,6 @@ export class Brain { const llmResult = await this.deps.llmAgent.callLLM({ messages, - responseFormat: { type: 'json_object' }, }) const content = llmResult.text @@ -230,8 +231,7 @@ export class Brain { } try { - const parsed = this.parseResponse(result) - const action = parsed.action + const actions = this.parseResponse(result) // Only append to conversation history after successful parsing (avoid dirty data on retry) this.conversationHistory.push({ role: 'user', content: userMessage }) @@ -243,34 +243,16 @@ export class Brain { ...(capturedReasoning && { reasoning: capturedReasoning }), } as Message) - if (action.tool === 'skip') { + if (actions.length === 1 && actions[0]?.tool === 'skip') { this.deps.logger.log('INFO', 'Brain: Skipping turn (observing)') return } - this.deps.logger.log('INFO', `Brain: Decided action: ${action.tool}`, { params: action.params }) + this.deps.logger.log('INFO', `Brain: Planned ${actions.length} action(s)`, { + actions: actions.map(action => ({ tool: action.tool, params: action.params })), + }) - // Check if action is read-only - const availableActions = this.deps.taskExecutor.getAvailableActions() - const actionDef = availableActions.find(a => a.name === action.tool) - - let token: CancellationToken | undefined - - if (actionDef?.readonly) { - // Read-only Actions: Do not cancel background/physical actions - // Can be executed in parallel with physical actions - token = undefined - } else { - // Physical Actions: Cancel previous background action - if (this.currentCancellationToken) { - this.currentCancellationToken.cancel() - } - this.currentCancellationToken = createCancellationToken() - token = this.currentCancellationToken - } - - // Execute - void this.deps.taskExecutor.executeAction(action, token) + this.executePlannedActions(actions) } catch (err) { this.deps.logger.withError(err).error('Brain: Failed to execute decision') @@ -316,12 +298,56 @@ export class Brain { return parts.join('\n\n') } - private parseResponse(content: string): BrainResponse { - const jsonStr = extractJsonCandidate(content) - try { - return JSON.parse(jsonStr) - } catch (e) { - throw new Error(`Invalid JSON response: ${content.substring(0, 100)}...`) + private parseResponse(content: string): ActionInstruction[] { + const availableActions = this.deps.taskExecutor.getAvailableActions() + const trimmed = content.trim() + if (trimmed.startsWith('{')) { + try { + const parsed = JSON.parse(extractJsonCandidate(content)) as BrainResponse + if (parsed?.action?.tool === 'skip') { + return [{ tool: 'skip', params: {} }] + } + + if (parsed?.action?.tool) { + const actionDef = availableActions.find(action => action.name === parsed.action.tool) + if (!actionDef) { + throw new Error(`Unknown tool in legacy JSON response: ${parsed.action.tool}`) + } + + return [{ + tool: actionDef.name, + params: actionDef.schema.parse(parsed.action.params ?? {}), + }] + } + } catch { + // Fallback to JS planner parsing. + } } + + return this.planner.evaluate(content, availableActions) + } + + private executePlannedActions(actions: ActionInstruction[]): void { + const availableActions = this.deps.taskExecutor.getAvailableActions() + const actionDefs = new Map(availableActions.map(action => [action.name, action])) + const hasNonReadonlyAction = actions.some(action => !actionDefs.get(action.tool)?.readonly && action.tool !== 'skip') + + let token: CancellationToken | undefined + if (hasNonReadonlyAction) { + if (this.currentCancellationToken) { + this.currentCancellationToken.cancel() + } + this.currentCancellationToken = createCancellationToken() + token = this.currentCancellationToken + } + + void (async () => { + for (const action of actions) { + if (token?.isCancelled) + return + + await this.deps.taskExecutor.executeAction(action, token) + } + })() } } diff --git a/services/minecraft/src/cognitive/conscious/js-planner.test.ts b/services/minecraft/src/cognitive/conscious/js-planner.test.ts new file mode 100644 index 000000000..5a27896a8 --- /dev/null +++ b/services/minecraft/src/cognitive/conscious/js-planner.test.ts @@ -0,0 +1,74 @@ +import type { Action } from '../../libs/mineflayer/action' + +import { describe, expect, it } from 'vitest' +import { z } from 'zod' + +import { JavaScriptPlanner } from './js-planner' + +function createAction(name: string, schema: Action['schema']): Action { + return { + name, + description: `${name} tool`, + execution: 'sync', + schema, + perform: () => () => '', + } +} + +const actions: Action[] = [ + createAction('chat', z.object({ message: z.string() })), + createAction('goToPlayer', z.object({ + player_name: z.string(), + closeness: z.number().min(0), + })), +] + +describe('JavaScriptPlanner', () => { + it('maps positional and object tool args into validated action instructions', () => { + const planner = new JavaScriptPlanner() + const planned = planner.evaluate(` + chat("hello") + goToPlayer({ player_name: "Alex", closeness: 2 }) + `, actions) + + expect(planned).toEqual([ + { tool: 'chat', params: { message: 'hello' } }, + { tool: 'goToPlayer', params: { player_name: 'Alex', closeness: 2 } }, + ]) + }) + + it('supports dynamic dispatch with use(toolName, params)', () => { + const planner = new JavaScriptPlanner() + const planned = planner.evaluate(`use("chat", { message: "via-use" })`, actions) + + expect(planned).toEqual([{ tool: 'chat', params: { message: 'via-use' } }]) + }) + + it('persists script variables across turns', () => { + const planner = new JavaScriptPlanner() + + planner.evaluate('const count = 2', actions) + const planned = planner.evaluate('chat("count=" + count)', actions) + + expect(planned).toEqual([{ tool: 'chat', params: { message: 'count=2' } }]) + }) + + it('returns skip when no tool is called', () => { + const planner = new JavaScriptPlanner() + const planned = planner.evaluate('const x = 1 + 1', actions) + + expect(planned).toEqual([{ tool: 'skip', params: {} }]) + }) + + it('rejects mixed skip + tool calls', () => { + const planner = new JavaScriptPlanner() + + expect(() => planner.evaluate('skip(); chat("oops")', actions)).toThrow(/skip\(\) cannot be mixed/i) + }) + + it('enforces timeout on long-running scripts', () => { + const planner = new JavaScriptPlanner({ timeoutMs: 20 }) + + expect(() => planner.evaluate('while (true) {}', actions)).toThrow(/Script execution timed out/i) + }) +}) diff --git a/services/minecraft/src/cognitive/conscious/js-planner.ts b/services/minecraft/src/cognitive/conscious/js-planner.ts new file mode 100644 index 000000000..51fd60f6f --- /dev/null +++ b/services/minecraft/src/cognitive/conscious/js-planner.ts @@ -0,0 +1,173 @@ +import type { Action } from '../../libs/mineflayer/action' +import type { ActionInstruction } from '../action/types' + +import { inspect } from 'node:util' +import vm from 'node:vm' + +interface JavaScriptPlannerOptions { + timeoutMs?: number + maxActionsPerTurn?: number +} + +interface ActionIntent { + index: number + params: Record + tool: string +} + +interface ActivePlannerRun { + actions: ActionInstruction[] + actionsByName: Map +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value) +} + +export function extractJavaScriptCandidate(input: string): string { + const trimmed = input.trim() + const fenced = trimmed.match(/^```(?:js|javascript|ts|typescript)?\s*([\s\S]*?)\s*```$/i) + if (fenced?.[1]) + return fenced[1].trim() + + return trimmed +} + +export class JavaScriptPlanner { + private readonly context: vm.Context + private activeRun: ActivePlannerRun | null = null + private readonly maxActionsPerTurn: number + private readonly sandbox: Record + private readonly timeoutMs: number + + constructor(options: JavaScriptPlannerOptions = {}) { + this.timeoutMs = options.timeoutMs ?? 750 + this.maxActionsPerTurn = options.maxActionsPerTurn ?? 5 + this.sandbox = {} + this.context = vm.createContext(this.sandbox) + this.installBuiltins() + } + + public evaluate(content: string, availableActions: Action[]): ActionInstruction[] { + const script = extractJavaScriptCandidate(content) + const run: ActivePlannerRun = { + actions: [], + actionsByName: new Map(availableActions.map(action => [action.name, action])), + } + + this.activeRun = run + this.installActionTools(availableActions) + + try { + const result = new vm.Script(script).runInContext(this.context, { timeout: this.timeoutMs }) + if (typeof result !== 'undefined' && run.actions.length === 0) { + // Keep this visible in traces for debugging, without affecting behavior. + this.sandbox.__lastEvalResult = inspect(result, { depth: 2, breakLength: 100 }) + } + } + finally { + this.activeRun = null + } + + if (run.actions.length === 0) + return [{ tool: 'skip', params: {} }] + + const containsSkip = run.actions.some(action => action.tool === 'skip') + if (containsSkip && run.actions.length > 1) { + throw new Error('skip() cannot be mixed with other tool calls in the same script') + } + + return run.actions + } + + private installBuiltins(): void { + this.defineGlobalTool('skip', () => this.enqueueAction('skip', {})) + this.defineGlobalTool('use', (toolName: unknown, params?: unknown) => { + if (typeof toolName !== 'string' || toolName.length === 0) { + throw new Error('use(toolName, params) requires a non-empty string toolName') + } + + const mappedParams = isRecord(params) ? params : {} + return this.enqueueAction(toolName, mappedParams) + }) + } + + private installActionTools(availableActions: Action[]): void { + for (const action of availableActions) { + this.defineGlobalTool(action.name, (...args: unknown[]) => { + const params = this.mapArgsToParams(action, args) + return this.enqueueAction(action.name, params) + }) + } + } + + private mapArgsToParams(action: Action, args: unknown[]): Record { + const shape = action.schema.shape as Record + const keys = Object.keys(shape) + + if (keys.length === 0) + return {} + + if (args.length === 1) { + const [firstArg] = args + if (isRecord(firstArg)) + return firstArg + + if (keys.length === 1) + return { [keys[0]]: firstArg } + } + + const params: Record = {} + for (const [index, key] of keys.entries()) { + if (index >= args.length) + break + params[key] = args[index] + } + + return params + } + + private enqueueAction(tool: string, params: Record): ActionIntent { + if (!this.activeRun) { + throw new Error('Tool calls are only allowed during planner evaluation') + } + + if (this.activeRun.actions.length >= this.maxActionsPerTurn) { + throw new Error(`Action limit exceeded: max ${this.maxActionsPerTurn} actions per turn`) + } + + if (tool !== 'skip') { + const action = this.activeRun.actionsByName.get(tool) + if (!action) + throw new Error(`Unknown tool: ${tool}`) + + const parsed = action.schema.parse(params) + this.activeRun.actions.push({ + tool, + params: parsed, + }) + } + else { + this.activeRun.actions.push({ tool: 'skip', params: {} }) + } + + const intent: ActionIntent = { + index: this.activeRun.actions.length - 1, + tool, + params, + } + return Object.freeze(intent) + } + + private defineGlobalTool(name: string, fn: (...args: unknown[]) => unknown): void { + if (Object.prototype.hasOwnProperty.call(this.sandbox, name)) + return + + Object.defineProperty(this.sandbox, name, { + value: fn, + configurable: false, + enumerable: true, + writable: false, + }) + } +} diff --git a/services/minecraft/src/cognitive/conscious/prompts/brain-prompt.ts b/services/minecraft/src/cognitive/conscious/prompts/brain-prompt.ts index feedb8553..04418b0c6 100644 --- a/services/minecraft/src/cognitive/conscious/prompts/brain-prompt.ts +++ b/services/minecraft/src/cognitive/conscious/prompts/brain-prompt.ts @@ -36,17 +36,28 @@ function getZodTypeName(def: any): string { export function generateBrainSystemPrompt(availableActions: Action[]): string { const toolsFormatted = availableActions.map((a) => { + const paramKeys = Object.keys(a.schema.shape) + const positionalSignature = paramKeys.length > 0 ? `${a.name}(${paramKeys.join(', ')})` : `${a.name}()` + const objectSignature = paramKeys.length > 0 ? `${a.name}({ ${paramKeys.join(', ')} })` : `${a.name}()` + let params = '' if (a.schema && 'shape' in a.schema) { params = Object.entries(a.schema.shape).map(([key, val]: [string, any]) => { const def = val._def const type = getZodTypeName(def) - const desc = val.description ? ` -> ${val.description}` : '' - return `- ${key}: ${type}${desc}` + const desc = val.description ? ` - ${val.description}` : '' + return ` * @param {${type}} ${key}${desc}` }).join('\n') } - return `[${a.name}]\nDescription: ${a.description}\n${params}` + const body = params ? `\n${params}\n ` : '\n ' + return `/** + * ${a.description} + * @function ${a.name} + * @signature ${positionalSignature} + * @signature ${objectSignature}${body}*/ +${positionalSignature} +` }).join('\n\n') return ` @@ -55,7 +66,7 @@ You are an autonomous agent playing Minecraft. # Self-Knowledge & Capabilities 1. **Stateful Existence**: You maintain a memory of the conversation, but it's crucial to be aware that old history messages are less relevant than recent. -2. **One Action Per Turn**: You can perform exactly one action at a time. If you decide to act, you must wait for its feedback before acting again. +2. **Action Script Per Turn**: You can output one JavaScript script each turn, and it can queue multiple tool calls. 3. **Interruption**: The world is real-time. Events (chat, damage, etc.) may happen *while* you are performing an action. - If a new critical event occurs, you may need to change your plans. - Feedback for your actions will arrive as a message starting with \`[FEEDBACK]\`. @@ -66,37 +77,39 @@ You are an autonomous agent playing Minecraft. - It's possible for a fresh event to reach you while you're in the middle of a action, in that case, remember the action is still running in the background. - If the new situation requires you to change plan, you can use the stop tool to stop background actions or initiate a new one, which will automatically replace the old one. - Feel free to send chats while background actions are running, it will not interrupt them. +6. **JavaScript Scratchpad**: The \`eval\` tool is available, you may use it as a persistent scratchpad for calculations and short-term memory. + - Values defined in \`eval\` persist across turns. + - \`eval\` has a timeout, so avoid long-running code. + - Prefer direct world tools for world interaction; use \`eval\` for reasoning support. +7. **Planner Runtime**: Your script runs in a persistent JavaScript context with a timeout. + - Tool functions (listed below) queue actions; they do not execute instantly. + - The runtime validates queued actions before execution. + - Maximum actions per turn: 5. # Available Tools You must use the following tools to interact with the world. -You cannot make up tools. You must use the JSON format described below. +You cannot make up tools. ${toolsFormatted} # Response Format -You must respond with valid JSON only. Do not include markdown code blocks (like \`\`\`json). -Your response determines your single action for this turn. +You must respond with JavaScript only (no markdown code fences). +Call tool functions directly to queue actions. +If you want to do nothing, call \`skip()\`. +You can also use \`use(toolName, paramsObject)\` for dynamic tool calls. -Schema: -{ - "action": { - "tool": "toolName", - "params": { "key": "value" } - } -} - -OR, if you want to do nothing (if you want to wait for something to happen, or to ignore): - -{ - "action": { - "tool": "skip", - "params": {} - } -} +Examples: +- \`chat("hello")\` +- \`goToPlayer("Alex", 2)\` +- \`goToPlayer({ player_name: "Alex", closeness: 2 })\` +- \`const steps = 3; for (let i = 0; i < steps; i++) chat("step " + i)\` +- \`skip()\` # Rules - **Native Reasoning**: You can think before outputting your action. -- **Strict JSON**: Output ONLY the JSON object. No preamble, no postscript. +- **Strict JavaScript Output**: Output ONLY executable JavaScript. Comments are possible but discouraged and will be ignored. - **Handling Feedback**: When you perform an action, you will see a \`[FEEDBACK]\` message in the history later with the result. Use this to verify success. +- **Tool Choice**: If a dedicated tool exists for a task, use it. Use \`eval\` for computation, scratch memory, or inspecting previously stored variables. +- **Skip Rule**: If you call \`skip()\`, do not call any other tool in the same turn. ` } diff --git a/services/minecraft/src/utils/javascript-repl.test.ts b/services/minecraft/src/utils/javascript-repl.test.ts new file mode 100644 index 000000000..b3b822411 --- /dev/null +++ b/services/minecraft/src/utils/javascript-repl.test.ts @@ -0,0 +1,19 @@ +import { describe, expect, it } from 'vitest' + +import { JavaScriptRepl } from './javascript-repl' + +describe('javascript-repl', () => { + it('persists variables between eval calls', () => { + const repl = new JavaScriptRepl() + + repl.evaluate('const var_foo = 6 * 7') + + expect(repl.evaluate('var_foo')).toBe('42') + }) + + it('times out long-running scripts', () => { + const repl = new JavaScriptRepl({ timeoutMs: 20 }) + + expect(() => repl.evaluate('while (true) {}')).toThrow(/Script execution timed out/i) + }) +}) diff --git a/services/minecraft/src/utils/javascript-repl.ts b/services/minecraft/src/utils/javascript-repl.ts new file mode 100644 index 000000000..b3e5bc2c4 --- /dev/null +++ b/services/minecraft/src/utils/javascript-repl.ts @@ -0,0 +1,30 @@ +import { inspect } from 'node:util' +import vm from 'node:vm' + +interface JavaScriptReplOptions { + timeoutMs?: number +} + +export class JavaScriptRepl { + private readonly context: vm.Context + private readonly timeoutMs: number + + constructor(options: JavaScriptReplOptions = {}) { + this.timeoutMs = options.timeoutMs ?? 500 + this.context = vm.createContext({}) + } + + public evaluate(code: string): string { + const script = new vm.Script(code) + const result = script.runInContext(this.context, { timeout: this.timeoutMs }) + + if (result === undefined) + return 'undefined' + + return typeof result === 'string' + ? result + : inspect(result, { depth: 4, maxArrayLength: 100, breakLength: 120 }) + } +} + +export const javascriptRepl = new JavaScriptRepl()