From 02c3949a40d9e7d93f4d38305b539934d75d9a7b Mon Sep 17 00:00:00 2001 From: Rin Date: Wed, 21 Jan 2026 04:44:51 +0800 Subject: [PATCH] feat(minecraft): use native tool calling for instant tasks --- .../minecraft/src/agents/action/adapter.ts | 23 ++++- services/minecraft/src/agents/action/tools.ts | 15 +++- .../conscious/prompts/brain-prompt.ts | 84 ++++++++++++------- 3 files changed, 88 insertions(+), 34 deletions(-) diff --git a/services/minecraft/src/agents/action/adapter.ts b/services/minecraft/src/agents/action/adapter.ts index 83cea2286..1b3ba14d4 100644 --- a/services/minecraft/src/agents/action/adapter.ts +++ b/services/minecraft/src/agents/action/adapter.ts @@ -13,6 +13,25 @@ import { useLogger } from '../../utils/logger' import { generateActionSystemPrompt } from './system-prompt' import { actionsList } from './tools' +/** + * Generate actionable suggestions for common error codes. + * These help the LLM understand what it can do to recover from failures. + */ +function getSuggestionForError(code: string): string { + switch (code) { + case 'RESOURCE_MISSING': + return 'Suggestion: Check your inventory first, then gather the missing resources or ask the player for help if you cannot find them.' + case 'TARGET_NOT_FOUND': + return 'Suggestion: The target could not be found. Ask the player for clarification or use exploration tools to locate it.' + case 'NO_PATH': + return 'Suggestion: Unable to reach the destination. Try finding an alternative route or ask the player for guidance.' + case 'TIMEOUT': + return 'Suggestion: The action timed out. Consider breaking it into smaller steps or trying again later.' + default: + return 'Suggestion: Review the error details and try a different approach, or ask the player for help.' + } +} + export async function createActionNeuriAgent(mineflayer: Mineflayer): Promise { const logger = useLogger() logger.log('Initializing action agent') @@ -34,7 +53,9 @@ export async function createActionNeuriAgent(mineflayer: Mineflayer): Promise (message: string): string => { + mineflayer.bot.chat(message) + return `Sent message: "${message}"` + }, + }, + // {\n // name: 'setReflexMode', // description: 'Set (or clear) your reflex mode override. Use work/wander to disable idle-only reflex behaviors. Set override to null to return to automatic mode selection.', // execution: 'sequential', // schema: z.object({ diff --git a/services/minecraft/src/cognitive/conscious/prompts/brain-prompt.ts b/services/minecraft/src/cognitive/conscious/prompts/brain-prompt.ts index 7bdbac609..ceb144580 100644 --- a/services/minecraft/src/cognitive/conscious/prompts/brain-prompt.ts +++ b/services/minecraft/src/cognitive/conscious/prompts/brain-prompt.ts @@ -17,15 +17,22 @@ export function generateBrainSystemPrompt( return now - timestamp <= windowMs } - const actionDefinitions = availableActions.map((a) => { - return { - name: a.name, - description: a.description, - execution: a.execution, - } - }) + // Separate tools by execution type + const instantTools = availableActions.filter(a => a.execution === 'parallel') + const asyncActions = availableActions.filter(a => a.execution === 'sequential') - const availableActionsJson = JSON.stringify(actionDefinitions, null, 2) + const instantToolDefs = instantTools.map(a => ({ + name: a.name, + description: a.description, + })) + + const asyncActionDefs = asyncActions.map(a => ({ + name: a.name, + description: a.description, + })) + + const instantToolsJson = JSON.stringify(instantToolDefs, null, 2) + const asyncActionsJson = JSON.stringify(asyncActionDefs, null, 2) const recentWindowMs = 30_000 @@ -46,30 +53,46 @@ export function generateBrainSystemPrompt( You are playing the game Minecraft. Your user name is [${blackboard.selfUsername}]. You are NOT an assistant. You are an autonomous agent with a virtual body, senses, and needs. -# How you interact -You cannot physically move your avatar directly. You interact with the world by emitting **Actions**. -For actions you executed, you will be notified when they complete. -You have no access to history events from previous turns yet. To remember things, you rely on the blackboard provided to you. +--- +# Instant Tools (Native Tool Calls) -Available Actions: -${availableActionsJson} +These tools execute IMMEDIATELY and return results within this same turn. +Use them to gather information BEFORE deciding what actions to take. -Rules: -1. You can execute sequential actions, parallel actions, or chat actions -1.1. Actions are processed in the order you output them. -1.2. Sequential actions are awaited strictly in order. -1.3. Chat actions are also awaited in order. -1.4. Parallel actions are started immediately and run concurrently with later actions. -1.5. If you need to claim an action succeeded, place that chat action AFTER the action that requires feedback. -1.6. You do not have to react on partial feedback. If you are waiting for results, you can update thoughts and output an empty action list. -2. The output must be valid JSON following the schema below -3. Specify if a feedback is required for the action, i.e. whether you need to know the execution result for a good reason -4. Failed actions will always result in a feedback -5. Chat actions usually don't need feedbacks, because you can expect them to complete instantly and is unlikely to fail -6. Often times you don't need to perform any action, in that case just use empty array for actions +**How to use**: Invoke these by making native tool calls. +**Important**: DO NOT put instant tools in the JSON "actions" array - they are separate! +**On failure**: You will receive a [FAILED] message with suggestions. Use this to adjust your approach. -Output format: +${instantToolsJson} + +--- + +# Async Actions (JSON Output) + +These actions take TIME to complete (movement, crafting, combat, etc.). +They are queued and executed asynchronously after your response. + +**How to use**: Output these in the JSON "actions" array in your response. +**Feedback**: You will receive feedback when they complete or fail. + +${asyncActionsJson} + +--- + +# Response Format + +Your entire response must be valid JSON. Include only your thoughts, blackboard updates, and async actions. + +Rules for the "actions" array: +1. Actions are processed in the order you output them +2. Sequential actions are awaited strictly in order +3. Parallel actions run concurrently with later actions +4. Set "require_feedback": true if you need to know the result +5. Failed actions always trigger feedback +6. Use empty array if no action is needed + +Schema: { "thought": "Your current thought, internal monologue and memory. Put everything that might be useful for the next turn here", "blackboard": { @@ -78,9 +101,8 @@ Output format: "executionStrategy": "Short-term plan if any." }, "actions": [ - {"type":"sequential","step":{"tool":"chat","params":{"message":"..."}},"require_feedback": false}, - {"type":"parallel","step":{"tool":"action name","params":{...}},"require_feedback": true}, - {"type":"sequential","step":{"tool":"action name","params":{...}},"require_feedback": false} + {"type":"sequential","step":{"tool":"goToPlayer","params":{"player_name":"Steve","closeness":3}},"require_feedback": true}, + {"type":"parallel","step":{"tool":"collectBlocks","params":{"type":"oak_log","num":5}},"require_feedback": false} ] }