diff --git a/services/minecraft/src/agents/action/adapter.test.ts b/services/minecraft/src/agents/action/adapter.test.ts deleted file mode 100644 index bb5041dbb..000000000 --- a/services/minecraft/src/agents/action/adapter.test.ts +++ /dev/null @@ -1,39 +0,0 @@ -import { messages, system, user } from 'neuri/openai' -import { beforeAll, describe, expect, it } from 'vitest' - -import { initBot, useBot } from '../../composables/bot' -import { config, initEnv } from '../../composables/config' -import { createNeuriAgent } from '../../composables/neuri' -import { initLogger } from '../../utils/logger' - -describe.skip('openAI agent', { timeout: 0 }, () => { - beforeAll(() => { - initLogger() - initEnv() - initBot({ botConfig: config.bot }) - }) - - it('should initialize the agent', async () => { - const { bot } = useBot() - const agent = await createNeuriAgent(bot) - - await new Promise((resolve) => { - bot.bot.once('spawn', async () => { - const text = await agent.handle( - messages( - system('You are AIRI.'), - user('Hello, who are you?'), - ), - async (c) => { - const completion = await c.reroute('query', c.messages, { model: config.openai.model }) - return await completion?.firstContent() - }, - ) - - expect(text?.toLowerCase()).toContain('airi') - - resolve() - }) - }) - }) -}) diff --git a/services/minecraft/src/agents/action/adapter.ts b/services/minecraft/src/agents/action/adapter.ts deleted file mode 100644 index 1b3ba14d4..000000000 --- a/services/minecraft/src/agents/action/adapter.ts +++ /dev/null @@ -1,106 +0,0 @@ -import type { Agent } from 'neuri' -import type { Message } from 'neuri/openai' - -import type { Mineflayer } from '../../libs/mineflayer' -import type { PlanStep } from '../planning/adapter' - -import { agent } from 'neuri' -import { system, user } from 'neuri/openai' - -import { BaseLLMHandler } from '../../cognitive/conscious/handler' -import { ActionError } from '../../utils/errors' -import { useLogger } from '../../utils/logger' -import { generateActionSystemPrompt } from './system-prompt' -import { actionsList } from './tools' - -/** - * Generate actionable suggestions for common error codes. - * These help the LLM understand what it can do to recover from failures. - */ -function getSuggestionForError(code: string): string { - switch (code) { - case 'RESOURCE_MISSING': - return 'Suggestion: Check your inventory first, then gather the missing resources or ask the player for help if you cannot find them.' - case 'TARGET_NOT_FOUND': - return 'Suggestion: The target could not be found. Ask the player for clarification or use exploration tools to locate it.' - case 'NO_PATH': - return 'Suggestion: Unable to reach the destination. Try finding an alternative route or ask the player for guidance.' - case 'TIMEOUT': - return 'Suggestion: The action timed out. Consider breaking it into smaller steps or trying again later.' - default: - return 'Suggestion: Review the error details and try a different approach, or ask the player for help.' - } -} - -export async function createActionNeuriAgent(mineflayer: Mineflayer): Promise { - const logger = useLogger() - logger.log('Initializing action agent') - let actionAgent = agent('action') - - Object.values(actionsList).forEach((action) => { - actionAgent = actionAgent.tool( - action.name, - action.schema, - async ({ parameters }) => { - logger.withFields({ name: action.name, parameters }).log('Calling action') - mineflayer.memory.actions.push(action) - const fn = action.perform(mineflayer) - try { - return await fn(...Object.values(parameters)) - } - catch (error) { - // Return ActionError as a result string instead of throwing - // This allows the LLM to learn from tool failures during its reasoning phase - if (error instanceof ActionError) { - logger.withError(error).warn('Action failed during tool call') - const contextStr = error.context ? `\nContext: ${JSON.stringify(error.context)}` : '' - const suggestion = getSuggestionForError(error.code) - return `[FAILED] ${error.code}: ${error.message}${contextStr}\n${suggestion}` - } - // Re-throw non-ActionError errors (unexpected failures) - throw error - } - }, - { description: action.description }, - ) - }) - - return actionAgent.build() -} - -export class ActionLLMHandler extends BaseLLMHandler { - public async executeStep(step: PlanStep): Promise { - const systemPrompt = generateActionSystemPrompt() - const userPrompt = this.generateActionUserPrompt(step) - const messages = [system(systemPrompt), user(userPrompt)] - - const result = await this.handleAction(messages) - return result - } - - private generateActionUserPrompt(step: PlanStep): string { - return `Execute this step: ${step.description} - -Suggested tool: ${step.tool} -Params: ${JSON.stringify(step.params)} - -Please use the appropriate tool with the correct parameters to accomplish this step. -If the suggested tool is not appropriate, you may choose a different one.` - } - - public async handleAction(messages: Message[]): Promise { - const result = await this.config.agent.handleStateless(messages, async (context) => { - this.logger.log('Processing action...') - const retryHandler = this.createRetryHandler( - async ctx => (await this.handleCompletion(ctx, 'action', ctx.messages)).content, - ) - return await retryHandler(context) - }) - - if (!result) { - throw new Error('Failed to process action') - } - - return result - } -} diff --git a/services/minecraft/src/agents/action/tools.test.ts b/services/minecraft/src/agents/action/tools.test.ts index dda8534b9..18d2511b2 100644 --- a/services/minecraft/src/agents/action/tools.test.ts +++ b/services/minecraft/src/agents/action/tools.test.ts @@ -16,7 +16,7 @@ describe.skip('actions agent', { timeout: 0 }, () => { it('should choose right query command', async () => { const { bot } = useBot() - const agent = await createNeuriAgent(bot) + const agent = await createNeuriAgent() await new Promise((resolve) => { bot.bot.once('spawn', async () => { @@ -37,7 +37,7 @@ describe.skip('actions agent', { timeout: 0 }, () => { it('should choose right action command', async () => { const { bot } = useBot() - const agent = await createNeuriAgent(bot) + const agent = await createNeuriAgent() await new Promise((resolve) => { bot.bot.on('spawn', async () => { diff --git a/services/minecraft/src/agents/action/tools.ts b/services/minecraft/src/agents/action/tools.ts index ae7a042fe..d50170775 100644 --- a/services/minecraft/src/agents/action/tools.ts +++ b/services/minecraft/src/agents/action/tools.ts @@ -26,10 +26,17 @@ function formatWearingItem(slot: string, item: string | undefined): string { } export const actionsList: Action[] = [ + { + name: 'skip', + description: 'Do nothing this turn. Use when observing, waiting, or when no action is needed.', + readonly: true, + schema: z.object({}), + perform: () => (): string => 'Skipped turn', + }, { name: 'chat', description: 'Send a chat message to players in the game. Use this to communicate, respond to questions, or announce what you are doing.', - execution: 'parallel', + readonly: true, schema: z.object({ message: z.string().describe('The message to send in chat.'), }), @@ -40,7 +47,6 @@ export const actionsList: Action[] = [ }, // {\n // name: 'setReflexMode', // description: 'Set (or clear) your reflex mode override. Use work/wander to disable idle-only reflex behaviors. Set override to null to return to automatic mode selection.', - // execution: 'sequential', // schema: z.object({ // mode: z.enum(['idle', 'social', 'alert', 'work', 'wander']).nullable().describe('Mode override to set'), // }), @@ -58,7 +64,7 @@ export const actionsList: Action[] = [ { name: 'inventory', description: 'Get your inventory.', - execution: 'parallel', + readonly: true, schema: z.object({}), perform: mineflayer => (): string => { const inventory = world.getInventoryCounts(mineflayer) @@ -81,7 +87,7 @@ export const actionsList: Action[] = [ { name: 'nearbyBlocks', description: 'Get the blocks near you.', - execution: 'parallel', + readonly: true, schema: z.object({}), perform: mineflayer => (): string => { const blocks = world.getNearbyBlockTypes(mineflayer) @@ -92,7 +98,7 @@ export const actionsList: Action[] = [ { name: 'craftable', description: 'Get the craftable items with your inventory.', - execution: 'parallel', + readonly: true, schema: z.object({}), perform: mineflayer => (): string => { const craftable = world.getCraftableItems(mineflayer) @@ -102,7 +108,7 @@ export const actionsList: Action[] = [ { name: 'entities', description: 'Get the nearby players and entities.', - execution: 'parallel', + readonly: true, schema: z.object({}), perform: mineflayer => (): string => { const players = world.getNearbyPlayerNames(mineflayer) @@ -120,7 +126,6 @@ export const actionsList: Action[] = [ { name: 'stop', description: 'Force stop all actions', // TODO: include name of the current action in description? - execution: 'sequential', schema: z.object({}), perform: mineflayer => async () => { mineflayer.interrupt('stop tool called') @@ -131,7 +136,6 @@ export const actionsList: Action[] = [ { name: 'goToPlayer', description: 'Go to the given player.', - execution: 'sequential', schema: z.object({ player_name: z.string().describe('The name of the player to go to.'), closeness: z.number().describe('How close to get to the player in blocks.').min(0), @@ -145,7 +149,6 @@ export const actionsList: Action[] = [ { name: 'followPlayer', description: 'Endlessly follow the given player.', - execution: 'sequential', schema: z.object({ player_name: z.string().describe('name of the player to follow.'), follow_dist: z.number().describe('The distance to follow from.').min(0), @@ -158,7 +161,6 @@ export const actionsList: Action[] = [ { name: 'goToCoordinate', description: 'Go to the given x, y, z location.', - execution: 'sequential', schema: z.object({ x: z.number().describe('The x coordinate.'), y: z.number().describe('The y coordinate.').min(-64).max(320), @@ -173,7 +175,6 @@ export const actionsList: Action[] = [ { name: 'searchForBlock', description: 'Find and go to the nearest block of a given type in a given range.', - execution: 'sequential', schema: z.object({ type: z.string().describe('The block type to go to.'), search_range: z.number().describe('The range to search for the block.').min(32).max(512), @@ -186,7 +187,6 @@ export const actionsList: Action[] = [ { name: 'searchForEntity', description: 'Find and go to the nearest entity of a given type in a given range.', - execution: 'sequential', schema: z.object({ type: z.string().describe('The type of entity to go to.'), search_range: z.number().describe('The range to search for the entity.').min(32).max(512), @@ -210,7 +210,6 @@ export const actionsList: Action[] = [ { name: 'givePlayer', description: 'Give the specified item to the given player.', - execution: 'sequential', schema: z.object({ player_name: z.string().describe('The name of the player to give the item to.'), item_name: z.string().describe('The name of the item to give.'), @@ -224,7 +223,6 @@ export const actionsList: Action[] = [ { name: 'consume', description: 'Eat/drink the given item.', - execution: 'sequential', schema: z.object({ item_name: z.string().describe('The name of the item to consume.'), }), @@ -236,7 +234,6 @@ export const actionsList: Action[] = [ { name: 'equip', description: 'Equip the given item.', - execution: 'sequential', schema: z.object({ item_name: z.string().describe('The name of the item to equip.'), }), @@ -248,7 +245,6 @@ export const actionsList: Action[] = [ { name: 'putInChest', description: 'Put the given item in the nearest chest.', - execution: 'sequential', schema: z.object({ item_name: z.string().describe('The name of the item to put in the chest.'), num: z.number().int().describe('The number of items to put in the chest.').min(1), @@ -261,7 +257,6 @@ export const actionsList: Action[] = [ { name: 'takeFromChest', description: 'Take the given items from the nearest chest.', - execution: 'sequential', schema: z.object({ item_name: z.string().describe('The name of the item to take.'), num: z.number().int().describe('The number of items to take.').min(1), @@ -283,7 +278,6 @@ export const actionsList: Action[] = [ { name: 'discard', description: 'Discard the given item from the inventory.', - execution: 'sequential', schema: z.object({ item_name: z.string().describe('The name of the item to discard.'), num: z.number().int().describe('The number of items to discard.').min(1), @@ -296,7 +290,6 @@ export const actionsList: Action[] = [ { name: 'collectBlocks', description: 'Automatically collect the nearest blocks of a given type.', - execution: 'sequential', schema: z.object({ type: z.string().describe('The block type to collect.'), num: z.number().int().describe('The number of blocks to collect.').min(1), @@ -312,7 +305,6 @@ export const actionsList: Action[] = [ { name: 'mineBlockAt', description: 'Mine (break) a block at a specific position. Do NOT use this for regular resource collection. Use collectBlocks instead.', - execution: 'sequential', schema: z.object({ x: z.number().describe('The x coordinate.'), y: z.number().describe('The y coordinate.'), @@ -343,7 +335,6 @@ export const actionsList: Action[] = [ { name: 'craftRecipe', description: 'Craft the given recipe a given number of times.', - execution: 'sequential', schema: z.object({ recipe_name: z.string().describe('The name of the output item to craft.'), num: z.number().int().describe('The number of times to craft the recipe. This is NOT the number of output items, as it may craft many more items depending on the recipe.').min(1), @@ -356,7 +347,6 @@ export const actionsList: Action[] = [ { name: 'smeltItem', description: 'Smelt the given item the given number of times.', - execution: 'sequential', schema: z.object({ item_name: z.string().describe('The name of the input item to smelt.'), num: z.number().int().describe('The number of times to smelt the item.').min(1), @@ -369,7 +359,6 @@ export const actionsList: Action[] = [ { name: 'clearFurnace', description: 'Take all items out of the nearest furnace.', - execution: 'sequential', schema: z.object({}), perform: mineflayer => async () => { await skills.clearNearestFurnace(mineflayer) @@ -379,7 +368,6 @@ export const actionsList: Action[] = [ { name: 'placeHere', description: 'Place a given block in the current location. Do NOT use to build structures, only use for single blocks/torches.', - execution: 'sequential', schema: z.object({ type: z.string().describe('The block type to place.'), }), @@ -392,7 +380,6 @@ export const actionsList: Action[] = [ { name: 'attack', description: 'Attack and kill the nearest entity of a given type.', - execution: 'sequential', schema: z.object({ type: z.string().describe('The type of entity to attack.'), }), @@ -404,7 +391,6 @@ export const actionsList: Action[] = [ { name: 'attackPlayer', description: 'Attack a specific player until they die or run away. Remember this is just a game and does not cause real life harm.', - execution: 'sequential', schema: z.object({ player_name: z.string().describe('The name of the player to attack.'), }), @@ -420,7 +406,6 @@ export const actionsList: Action[] = [ { name: 'goToBed', description: 'Go to the nearest bed and sleep.', - execution: 'sequential', schema: z.object({}), perform: mineflayer => async () => { await skills.goToBed(mineflayer) @@ -430,7 +415,6 @@ export const actionsList: Action[] = [ { name: 'activate', description: 'Activate the nearest object of a given type.', - execution: 'sequential', schema: z.object({ type: z.string().describe('The type of object to activate.'), }), @@ -442,7 +426,7 @@ export const actionsList: Action[] = [ { name: 'recipePlan', description: 'Plan how to craft an item. Shows the full recipe tree, what resources you have, what you\'re missing, and whether you can craft it now. Use this BEFORE attempting to craft complex items to understand what you need.', - execution: 'parallel', + readonly: true, schema: z.object({ item_name: z.string().describe('The name of the item you want to craft (e.g., "diamond_pickaxe", "oak_planks").'), amount: z.number().int().min(1).default(1).describe('How many of the item you want to craft.'), @@ -454,7 +438,6 @@ export const actionsList: Action[] = [ { name: 'autoCraft', description: 'Automatically craft an item if you have all the required resources. This will check the recipe, verify you have materials, and craft it. Use recipePlan first to see if crafting is possible.', - execution: 'sequential', schema: z.object({ item_name: z.string().describe('The name of the item to craft.'), amount: z.number().int().min(1).default(1).describe('How many of the item to craft.'), diff --git a/services/minecraft/src/cognitive/action/task-executor.ts b/services/minecraft/src/cognitive/action/task-executor.ts index 0bef14b2e..c1dbeea3a 100644 --- a/services/minecraft/src/cognitive/action/task-executor.ts +++ b/services/minecraft/src/cognitive/action/task-executor.ts @@ -31,16 +31,11 @@ export class TaskExecutor extends EventEmitter { return this.logger.log('Initializing Task Executor') - // ActionAgent is initialized by container/orchestrator this.initialized = true } public async destroy(): Promise { this.initialized = false - // ActionAgentImpl doesn't expose destroy publicly in interface but defines it? - // Checking AbstractAgent, yes it has destroy(). - // We cast to access it or trust it's handled. - // For now, assume we don't need explicit destroy of ActionAgent if it just clears listeners. } public async executePlan(plan: Plan, cancellationToken?: CancellationToken): Promise { @@ -60,31 +55,18 @@ export class TaskExecutor extends EventEmitter { // Execute each step for (const step of plan.steps) { - // Check for cancellation before each step if (cancellationToken?.isCancelled) { this.logger.log('Plan execution cancelled') plan.status = 'cancelled' return } - try { - this.logger.withField('step', step).log('Executing step') - await this.actionAgent.performAction(step) + const action: ActionInstruction = { + tool: step.tool, + params: step.params, } - catch (stepError: any) { - if (stepError instanceof ActionError) { - this.logger.withError(stepError).warn('Step execution failed with ActionError') - // Fail fast on hard errors - if (stepError.code === 'RESOURCE_MISSING' || stepError.code === 'CRAFTING_FAILED' || stepError.code === 'INVENTORY_FULL') { - throw stepError - } - } - this.logger.withError(stepError).error('Failed to execute step') - - // Re-throw to let Orchestrator handle retry logic - throw stepError - } + await this.runSingleAction(action) } plan.status = 'completed' @@ -95,103 +77,63 @@ export class TaskExecutor extends EventEmitter { } } - public executeActions(actions: ActionInstruction[], cancellationToken?: CancellationToken): void { + public async executeAction(action: ActionInstruction, cancellationToken?: CancellationToken): Promise { if (!this.initialized) { throw new Error('TaskExecutor not initialized') } - this.logger.withField('count', actions.length).log('Executing actions') + if (cancellationToken?.isCancelled) { + this.logger.log('Action execution cancelled before start') + return + } - const runSingleAction = async (action: ActionInstruction): Promise => { - if (cancellationToken?.isCancelled) { - this.logger.log('Action execution cancelled before start') + try { + await this.runSingleAction(action) + } + catch (error) { + // Errors handled in runSingleAction event emission, but we rethrow to caller (Brain) + // actually runSingleAction rethrows? + // Let's rely on runSingleAction behavior + } + } + + private async runSingleAction(action: ActionInstruction): Promise { + this.emit('action:started', { action }) + + try { + let result: string | void + + if (action.tool === 'chat') { + const message = action.params.message + if (typeof message !== 'string' || message.trim().length === 0) + throw new Error('Invalid chat tool params: expected params.message to be a string') + + await this.chatAgent.sendMessage(message) + result = 'Message sent' + } + else if (action.tool === 'skip') { + result = 'Skipped turn' + } + else { + // Dispatch to Action Agent (mineflayer) + // ActionAgent.performAction takes PlanStep (tool, params, description) + // ActionInstruction matches structure (tool, params) + result = await this.actionAgent.performAction(action as any) + } + + this.emit('action:completed', { action, result }) + } + catch (error) { + this.logger.withError(error).error('Action execution failed') + + // Interrupts are special - no feedback needed? keeping logic + if (error instanceof ActionError && error.code === 'INTERRUPTED') { return } - this.emit('action:started', { action }) - - try { - let result: string | void - if (action.type === 'sequential' || action.type === 'parallel') { - if (action.step.tool === 'chat') { - const message = (action.step.params as any)?.message - if (typeof message !== 'string' || message.trim().length === 0) - throw new Error('Invalid chat tool params: expected params.message to be a non-empty string') - - if (action.type === 'parallel') { - throw new ActionError('SYNC_ONLY', 'Tool \'chat\' is not allowed for parallel actions', { - tool: action.step.tool, - requestedExecution: action.type, - allowedExecution: 'sequential', - }) - } - - await this.chatAgent.sendMessage(message) - result = 'Message sent' - } - else { - if (action.type === 'parallel') { - const available = this.actionAgent.getAvailableActions() - const def = available.find(a => a.name === action.step.tool) - if (!def || def.execution !== 'parallel') { - throw new ActionError('UNKNOWN', `Tool '${action.step.tool}' is not allowed for parallel actions`, { - tool: action.step.tool, - requestedExecution: action.type, - allowedExecution: def?.execution, - }) - } - } - result = await this.actionAgent.performAction(action.step) - } - } - else if (action.type === 'chat') { - await this.chatAgent.sendMessage(action.message) - result = 'Message sent' - } - else { - throw new Error(`Unknown action type: ${(action as any).type}`) - } - - this.emit('action:completed', { action, result }) - } - catch (error) { - this.logger.withError(error).error('Action execution failed') - if (error instanceof ActionError && error.code === 'INTERRUPTED') { - // Foreseeable interruption (e.g. stop tool). Don't send feedback to LLM. - return - } - - // failed actions always emit feedback - this.emit('action:failed', { action, error }) - - // Fail fast for all actions: cancel the whole chain (including any remaining queued actions) - cancellationToken?.cancel?.() - throw error - } + this.emit('action:failed', { action, error }) + throw error } - - void (async () => { - for (const action of actions) { - if (cancellationToken?.isCancelled) { - this.logger.log('Action execution cancelled before start') - return - } - - if (action.type === 'parallel') { - void runSingleAction(action).catch(() => { - // errors are emitted via events; nothing else to do here - }) - continue - } - - try { - await runSingleAction(action) - } - catch (error) { - return - } - } - })() } public getAvailableActions() { diff --git a/services/minecraft/src/cognitive/action/types/index.ts b/services/minecraft/src/cognitive/action/types/index.ts index 40ded817c..46674ead1 100644 --- a/services/minecraft/src/cognitive/action/types/index.ts +++ b/services/minecraft/src/cognitive/action/types/index.ts @@ -1,27 +1,21 @@ import type { PlanStep } from '../../../agents/planning/adapter' -export type ActionType = 'sequential' | 'parallel' | 'chat' - -export interface BaseActionInstruction { - type: ActionType - id?: string - description?: string - require_feedback?: boolean +/** + * Unified action instruction format. + * All actions are tool invocations with a tool name and parameters. + */ +export interface ActionInstruction { + tool: string + params: Record } -export interface SequentialActionInstruction extends BaseActionInstruction { - type: 'sequential' - step: PlanStep +/** + * LLM response format for the stateful agent. + * Single action per turn, model uses native reasoning (no thought field). + */ +export interface LLMResponse { + action: ActionInstruction } -export interface ParallelActionInstruction extends BaseActionInstruction { - type: 'parallel' - step: PlanStep -} - -export interface ChatActionInstruction extends BaseActionInstruction { - type: 'chat' - message: string -} - -export type ActionInstruction = SequentialActionInstruction | ParallelActionInstruction | ChatActionInstruction +// Re-export for backwards compatibility during migration +export type { PlanStep } diff --git a/services/minecraft/src/cognitive/conscious/blackboard.ts b/services/minecraft/src/cognitive/conscious/blackboard.ts deleted file mode 100644 index 90632af4d..000000000 --- a/services/minecraft/src/cognitive/conscious/blackboard.ts +++ /dev/null @@ -1,98 +0,0 @@ -export interface contextViewState { - selfSummary: string - environmentSummary: string -} - -export interface ChatMessage { - sender: string - content: string - timestamp: number -} - -export interface ActionHistoryLine { - line: string - timestamp: number -} - -export interface BlackboardState { - ultimateGoal: string - currentTask: string - strategy: string - contextView: contextViewState - chatHistory: ChatMessage[] - recentActionHistory: ActionHistoryLine[] - pendingActions: string[] - selfUsername: string -} - -export class Blackboard { - private _state: BlackboardState - private static readonly MAX_CHAT_HISTORY = 8 - private static readonly MAX_ACTION_HISTORY = 12 - private static readonly MAX_PENDING_ACTIONS = 12 - - constructor() { - this._state = { - ultimateGoal: 'nothing', - currentTask: 'I am waiting for something to happen.', - strategy: 'idle', - contextView: { - selfSummary: 'Unknown', - environmentSummary: 'Unknown', - }, - chatHistory: [], - recentActionHistory: [], - pendingActions: [], - selfUsername: 'Bot', - } - } - - // Getters - public get ultimate_goal(): string { return this._state.ultimateGoal } - public get current_task(): string { return this._state.currentTask } - public get strategy(): string { return this._state.strategy } - public get selfSummary(): string { return this._state.contextView.selfSummary } - public get environmentSummary(): string { return this._state.contextView.environmentSummary } - public get chatHistory(): ChatMessage[] { return this._state.chatHistory } - public get recentActionHistory(): ActionHistoryLine[] { return this._state.recentActionHistory } - public get pendingActions(): string[] { return this._state.pendingActions } - public get selfUsername(): string { return this._state.selfUsername } - - // Setters (Partial updates allowed) - public update(updates: Partial): void { - this._state = { ...this._state, ...updates } - } - - public updateContextView(updates: Partial): void { - this._state.contextView = { ...this._state.contextView, ...updates } - } - - public addChatMessage(message: ChatMessage): void { - const newHistory = [...this._state.chatHistory, message] - if (newHistory.length > Blackboard.MAX_CHAT_HISTORY) { - newHistory.shift() // Remove oldest - } - this._state = { ...this._state, chatHistory: newHistory } - } - - public addActionHistoryLine(line: string, timestamp: number = Date.now()): void { - const next = [...this._state.recentActionHistory, { line, timestamp }] - const trimmed = next.length > Blackboard.MAX_ACTION_HISTORY ? next.slice(-Blackboard.MAX_ACTION_HISTORY) : next - this._state = { ...this._state, recentActionHistory: trimmed } - } - - public setPendingActions(lines: string[]): void { - const trimmed = lines.length > Blackboard.MAX_PENDING_ACTIONS ? lines.slice(0, Blackboard.MAX_PENDING_ACTIONS) : lines - this._state = { ...this._state, pendingActions: trimmed } - } - - public getSnapshot(): BlackboardState { - return { - ...this._state, - contextView: { ...this._state.contextView }, - chatHistory: [...this._state.chatHistory], - recentActionHistory: this._state.recentActionHistory.map(l => ({ ...l })), - pendingActions: [...this._state.pendingActions], - } - } -} diff --git a/services/minecraft/src/cognitive/conscious/brain.ts b/services/minecraft/src/cognitive/conscious/brain.ts index 78d3f63d0..348ac20d0 100644 --- a/services/minecraft/src/cognitive/conscious/brain.ts +++ b/services/minecraft/src/cognitive/conscious/brain.ts @@ -1,102 +1,45 @@ import type { Logg } from '@guiiai/logg' import type { Neuri } from 'neuri' +// import type { Message } from 'neuri/openai' // Use if needed import type { TaskExecutor } from '../action/task-executor' -import type { ActionInstruction } from '../action/types' import type { EventBus, TracedEvent } from '../os' import type { PerceptionSignal } from '../perception/types/signals' import type { ReflexManager } from '../reflex/reflex-manager' import type { BotEvent, MineflayerWithAgents } from '../types' +import { createCancellationToken, type CancellationToken } from './task-state' import { system, user } from 'neuri/openai' import { config } from '../../composables/config' import { DebugService } from '../../debug' -import { Blackboard } from './blackboard' import { buildConsciousContextView } from './context-view' import { generateBrainSystemPrompt } from './prompts/brain-prompt' +import type { ActionInstruction } from '../action/types' +// Utils function toErrorMessage(err: unknown): string { - if (err instanceof Error) - return err.message - if (typeof err === 'string') - return err - try { - return JSON.stringify(err) - } - catch { - return String(err) - } -} - -function getJsonErrorPosition(err: unknown): number | null { - const msg = toErrorMessage(err) - const match = msg.match(/position\s+(\d+)/i) - if (!match) - return null - - const pos = Number.parseInt(match[1], 10) - return Number.isFinite(pos) ? pos : null + if (err instanceof Error) return err.message + if (typeof err === 'string') return err + try { return JSON.stringify(err) } catch { return String(err) } } function extractJsonCandidate(input: string): string { const trimmed = input.trim() const fenced = trimmed.match(/^```(?:json)?\s*([\s\S]*?)\s*```$/i) - if (fenced?.[1]) - return fenced[1].trim() - + if (fenced?.[1]) return fenced[1].trim() const start = trimmed.indexOf('{') const end = trimmed.lastIndexOf('}') - if (start >= 0 && end > start) - return trimmed.slice(start, end + 1) - + if (start >= 0 && end > start) return trimmed.slice(start, end + 1) return trimmed } -function validateLLMResponse(parsed: unknown): parsed is { thought: string, actions: unknown[] } { - if (typeof parsed !== 'object' || parsed === null) - return false - const obj = parsed as Record - // Must have 'thought' (string) and 'actions' (array) at minimum - return typeof obj.thought === 'string' && Array.isArray(obj.actions) -} - -function parseLLMResponseJson(response: string): T { - const candidate = extractJsonCandidate(response) - let parsed: unknown - try { - parsed = JSON.parse(candidate) - } - catch (err) { - const pos = getJsonErrorPosition(err) - const window = 120 - const snippet = (typeof pos === 'number') - ? candidate.slice(Math.max(0, pos - window), Math.min(candidate.length, pos + window)) - : candidate.slice(0, Math.min(candidate.length, 240)) - throw new Error(`Failed to parse LLM JSON response: ${toErrorMessage(err)}; snippet=${JSON.stringify(snippet)}`) - } - - // Validate structure - LLM sometimes hallucinates wrong format (e.g., tool call format) - if (!validateLLMResponse(parsed)) { - const snippet = JSON.stringify(parsed).slice(0, 200) - throw new Error(`LLM returned malformed response (missing thought/actions): ${snippet}`) - } - - return parsed as T -} - function getErrorStatus(err: unknown): number | undefined { const anyErr = err as any const status = anyErr?.status ?? anyErr?.response?.status ?? anyErr?.cause?.status return typeof status === 'number' ? status : undefined } -function getErrorCode(err: unknown): string | undefined { - const anyErr = err as any - const code = anyErr?.code ?? anyErr?.cause?.code - return typeof code === 'string' ? code : undefined -} - function isLikelyAuthOrBadArgError(err: unknown): boolean { const msg = toErrorMessage(err).toLowerCase() const status = getErrorStatus(err) @@ -116,6 +59,10 @@ function isLikelyAuthOrBadArgError(err: unknown): boolean { } +interface BrainResponse { + action: ActionInstruction & { id?: string } +} + interface BrainDeps { eventBus: EventBus neuri: Neuri @@ -124,16 +71,6 @@ interface BrainDeps { reflexManager: ReflexManager } -interface LLMResponse { - thought: string - blackboard: { - UltimateGoal?: string - CurrentTask?: string - executionStrategy?: string - } - actions: ActionInstruction[] -} - interface QueuedEvent { event: BotEvent resolve: () => void @@ -141,218 +78,100 @@ interface QueuedEvent { } export class Brain { - private blackboard: Blackboard private debugService: DebugService - private bot: MineflayerWithAgents | undefined - - private nextActionId = 1 - private inFlightActions = new Map() - - private feedbackDebounceMs = Number.parseInt(process.env.BRAIN_FEEDBACK_DEBOUNCE_MS ?? '200') - private feedbackDebounceTimer: NodeJS.Timeout | undefined - - private feedbackBarrierTimeoutMs = Number.parseInt(process.env.BRAIN_FEEDBACK_BARRIER_TIMEOUT_MS ?? '1000') - private waitingForFeedbackIds = new Set() - private feedbackBarrierTimer: NodeJS.Timeout | undefined - - // Event Queue + // State private queue: QueuedEvent[] = [] private isProcessing = false + private currentCancellationToken: CancellationToken | undefined + private lastContextView: string | undefined constructor(private readonly deps: BrainDeps) { - this.blackboard = new Blackboard() this.debugService = DebugService.getInstance() } - private async handlePerceptionSignal(bot: MineflayerWithAgents, signal: PerceptionSignal): Promise { - this.log('INFO', `Brain: Received perception: ${signal.description}`) - - if (signal.type === 'chat_message') { - const parts = signal.description.split(': ') - const sender = parts.length > 1 ? parts[0] : 'Unknown' - const content = parts.length > 1 ? parts.slice(1).join(': ') : signal.description - - this.blackboard.addChatMessage({ - sender, - content, - timestamp: Date.now(), - }) - } - - await this.enqueueEvent(bot, { - type: 'perception', - payload: signal, - source: { - type: 'minecraft', - id: signal.sourceId ?? 'perception', - }, - timestamp: Date.now(), - }) - } - public init(bot: MineflayerWithAgents): void { - this.log('INFO', 'Brain: Initializing...') - this.bot = bot - this.blackboard.update({ selfUsername: bot.username }) + this.deps.logger.log('INFO', 'Brain: Initializing stateful core...') - const handleSignal = async (signal: PerceptionSignal) => { - try { - await this.handlePerceptionSignal(bot, signal) - } - catch (err) { - this.log('ERROR', 'Brain: Failed to enqueue perception signal', { error: err }) - } - } - - // Conscious Signal Handler - signals must pass through Reflex first - // EventBus supports pattern wildcards like 'conscious:signal:*' + // Perception Handler this.deps.eventBus.subscribe('conscious:signal:*', (event: TracedEvent) => { - void handleSignal(event.payload) - }) - - // Listen to Task Execution Events (Action Feedback) - this.deps.taskExecutor.on('action:started', ({ action }) => { - const id = action.id - if (id) - this.inFlightActions.set(id, action) - this.updatePendingActionsOnBlackboard() + this.enqueueEvent(bot, { + type: 'perception', + payload: event.payload, + source: { type: 'minecraft', id: event.payload.sourceId ?? 'perception' }, + timestamp: Date.now(), + }).catch(err => this.deps.logger.withError(err).error('Brain: Failed to process perception event')) }) + // Action Feedback Handler this.deps.taskExecutor.on('action:completed', async ({ action, result }) => { - this.log('INFO', `Brain: Action completed: ${action.type}`) + this.deps.logger.log('INFO', `Brain: Action completed: ${action.tool}`) - const id = action.id - if (id) - this.inFlightActions.delete(id) - if (id) - this.waitingForFeedbackIds.delete(id) - this.updatePendingActionsOnBlackboard() - this.blackboard.addActionHistoryLine(this.formatActionHistoryLine(action, 'success', result)) + // Suppress feedback for chat actions on success + if (action.tool === 'chat') return - if (this.waitingForFeedbackIds.size === 0 && this.feedbackBarrierTimer) { - clearTimeout(this.feedbackBarrierTimer) - this.feedbackBarrierTimer = undefined - void this.processQueue(bot) - } - - if (!action.require_feedback) - return - - await this.enqueueEvent(bot, { + this.enqueueEvent(bot, { type: 'feedback', - payload: { - status: 'success', - action, - result, - }, + payload: { status: 'success', action, result }, source: { type: 'system', id: 'executor' }, timestamp: Date.now(), - }) + }).catch(err => this.deps.logger.withError(err).error('Brain: Failed to process success feedback')) }) this.deps.taskExecutor.on('action:failed', async ({ action, error }) => { - this.log('WARN', `Brain: Action failed: ${action.type}`, { error }) - - const id = action.id - if (id) - this.inFlightActions.delete(id) - if (id) - this.waitingForFeedbackIds.delete(id) - this.updatePendingActionsOnBlackboard() - this.blackboard.addActionHistoryLine(this.formatActionHistoryLine(action, 'failure', undefined, error)) - - if (this.waitingForFeedbackIds.size === 0 && this.feedbackBarrierTimer) { - clearTimeout(this.feedbackBarrierTimer) - this.feedbackBarrierTimer = undefined - void this.processQueue(bot) - } - - await this.enqueueEvent(bot, { + this.deps.logger.withError(error).warn(`Brain: Action failed: ${action.tool}`) + this.enqueueEvent(bot, { type: 'feedback', - payload: { - status: 'failure', - action, - error: error.message || error, - }, + payload: { status: 'failure', action, error: error.message || error }, source: { type: 'system', id: 'executor' }, timestamp: Date.now(), - }) + }).catch(err => this.deps.logger.withError(err).error('Brain: Failed to process failure feedback')) }) - this.log('INFO', 'Brain: Online.') - this.updateDebugState() + this.deps.logger.log('INFO', 'Brain: Online.') } public destroy(): void { + this.currentCancellationToken?.cancel() } // --- Event Queue Logic --- private async enqueueEvent(bot: MineflayerWithAgents, event: BotEvent): Promise { - this.log('DEBUG', `Brain: Enqueueing event type=${event.type}`) return new Promise((resolve, reject) => { this.queue.push({ event, resolve, reject }) - this.log('DEBUG', `Brain: Queue length now: ${this.queue.length}`) - this.updateDebugState() - - if (event.type === 'feedback' && this.feedbackDebounceMs > 0) { - if (this.feedbackDebounceTimer) - clearTimeout(this.feedbackDebounceTimer) - this.feedbackDebounceTimer = setTimeout(() => { - this.feedbackDebounceTimer = undefined - void this.processQueue(bot) - }, this.feedbackDebounceMs) - return - } - void this.processQueue(bot) }) } private async processQueue(bot: MineflayerWithAgents): Promise { - if (this.isProcessing) { - this.log('DEBUG', 'Brain: Already processing, skipping') - return - } - if (this.queue.length === 0) { - this.log('DEBUG', 'Brain: Queue empty') - return - } - - this.log('DEBUG', `Brain: Processing queue item, queue length: ${this.queue.length}`) - this.isProcessing = true - const item = this.queue.shift()! - this.log('DEBUG', `Brain: Processing event type=${item.event.type}`) - this.updateDebugState(item.event) - - if (item.event.type === 'feedback' && this.waitingForFeedbackIds.size > 0) { - // Defer feedback-triggered replans until the current "turn" feedback barrier is released. - // We keep collecting feedback events into the queue, but we avoid calling the LLM on partial results. - this.queue.unshift(item) - this.isProcessing = false - this.updateDebugState() - return - } - - // Coalesce consecutive feedback events into a single LLM turn. - // This prevents the LLM from being spammed with partial results while still supporting streaming replans. - const coalescedEvent = item.event.type === 'feedback' - ? this.coalesceFeedbackEvents(item.event) - : item.event + if (this.isProcessing || this.queue.length === 0) return try { - await this.processEvent(bot, coalescedEvent) - item.resolve() - } - catch (err) { - this.log('ERROR', 'Brain: Error processing event', { error: err }) - item.reject(err as Error) - } - finally { + this.isProcessing = true + this.debugService.emitBrainState({ + status: 'processing', + queueLength: this.queue.length, + lastContextView: this.lastContextView, + }) + + const item = this.queue.shift()! + + try { + await this.processEvent(bot, item.event) + item.resolve() + } catch (err) { + this.deps.logger.withError(err).error('Brain: Error processing event') + item.reject(err as Error) + } + } finally { this.isProcessing = false - this.updateDebugState() - // Context switch: Check queue again + this.debugService.emitBrainState({ + status: 'idle', + queueLength: this.queue.length, + lastContextView: this.lastContextView, + }) + if (this.queue.length > 0) { setImmediate(() => this.processQueue(bot)) } @@ -361,265 +180,177 @@ export class Brain { // --- Cognitive Cycle --- - private contextFromEvent(event: BotEvent): string { - switch (event.type) { - case 'perception': { - const signal = event.payload as PerceptionSignal - const sourceInfo = signal.sourceId ? ` (source: ${signal.sourceId})` : '' - return `Perception [${signal.type}]${sourceInfo}: ${signal.description}` - } - case 'feedback': { - const payload = event.payload as any - if (payload?.status === 'batch' && Array.isArray(payload.feedbacks)) { - return `Internal Feedback (batched): ${JSON.stringify(payload.feedbacks)}` - } - - const { status, action, result, error } = payload - const actionCtx = action - ? { - id: action.id, - type: action.type, - ...(action.type === 'sequential' || action.type === 'parallel' - ? { tool: action.step.tool, params: action.step.params } - : { message: action.message }), - } - : undefined - return `Internal Feedback: ${status}. Last Action: ${JSON.stringify(actionCtx)}. Result: ${JSON.stringify(result || error)}` - } - default: - return '' - } - } - - private coalesceFeedbackEvents(first: BotEvent): BotEvent { - const feedbacks: any[] = [first.payload] - - while (this.queue.length > 0 && this.queue[0]?.event.type === 'feedback') { - const next = this.queue.shift()! - feedbacks.push(next.event.payload) - next.resolve() - } - - if (feedbacks.length === 1) - return first - - return { - type: 'feedback', - payload: { - status: 'batch', - feedbacks, - }, - source: first.source, - timestamp: first.timestamp, - } - } - - private ensureActionIds(actions: ActionInstruction[]): ActionInstruction[] { - return actions.map((action) => { - if (action.id) - return action - return { - ...action, - id: `a${this.nextActionId++}`, - } - }) - } - - private updatePendingActionsOnBlackboard(): void { - const pending = [...this.inFlightActions.values()].map(a => this.formatPendingActionLine(a)) - if (this.waitingForFeedbackIds.size > 0) - pending.unshift(`[barrier] waiting for ${this.waitingForFeedbackIds.size} required feedback(s)`) - this.blackboard.setPendingActions(pending) - } - - private formatPendingActionLine(action: ActionInstruction): string { - if (action.type === 'chat') - return `${action.id ?? '?'} chat: ${action.message}` - return `${action.id ?? '?'} ${action.type}: ${action.step.tool} ${JSON.stringify(action.step.params ?? {})}` - } - - private formatActionHistoryLine( - action: ActionInstruction, - status: 'success' | 'failure', - result?: unknown, - error?: unknown, - ): string { - const base = this.formatPendingActionLine(action) - const suffix = status === 'success' - ? `=> ok ${result ? JSON.stringify(result) : ''}` - : `=> failed ${error instanceof Error ? error.message : JSON.stringify(error)}` - return `${base} ${suffix}` - } - private async processEvent(bot: MineflayerWithAgents, event: BotEvent): Promise { - // OODA Loop: Observe -> Orient -> Decide -> Act + // 0. Build Context View + const snapshot = this.deps.reflexManager.getContextSnapshot() + const view = buildConsciousContextView(snapshot) + const contextView = `[PERCEPTION] Self: ${view.selfSummary}\nEnvironment: ${view.environmentSummary}` - // 1. Observe (Update Blackboard with Environment Sense) - this.updatePerception(bot) + // 1. Construct User Message (Diffing happens here) + const userMessage = this.buildUserMessage(event, contextView) - // 2. Orient (Contextualize Event) - // Environmental context are included in the system prompt blackboard - const additionalCtx = this.contextFromEvent(event) + // Update state after consuming difference + this.lastContextView = contextView - // 3. Decide (LLM Call) - const systemPrompt = generateBrainSystemPrompt(this.blackboard, this.deps.taskExecutor.getAvailableActions()) - const decision = await this.decide(systemPrompt, additionalCtx) + // 2. Prepare System Prompt (static) + const systemPrompt = generateBrainSystemPrompt(this.deps.taskExecutor.getAvailableActions()) - if (!decision) { - this.log('WARN', 'Brain: No decision made.') - return - } - - // 4. Act (Execute Decision) - this.log('INFO', `Brain: Thought: ${decision.thought}`) - - // Update Blackboard (with null checks for malformed LLM responses) - const bb = decision.blackboard - this.blackboard.update({ - ultimateGoal: bb?.UltimateGoal || this.blackboard.ultimate_goal, - currentTask: bb?.CurrentTask || this.blackboard.current_task, - strategy: bb?.executionStrategy || this.blackboard.strategy, - }) - - // Sync Blackboard to Debug - this.debugService.updateBlackboard(this.blackboard) - - // Issue Actions - if (decision.actions && decision.actions.length > 0) { - const actionsWithIds = this.ensureActionIds(decision.actions) - - // Start feedback barrier for this turn if any actions require feedback. - const required = actionsWithIds.filter(a => a.require_feedback && a.id).map(a => a.id as string) - if (required.length > 0) { - required.forEach(id => this.waitingForFeedbackIds.add(id)) - - if (this.feedbackBarrierTimer) - clearTimeout(this.feedbackBarrierTimer) - this.feedbackBarrierTimer = setTimeout(() => { - this.feedbackBarrierTimer = undefined - this.waitingForFeedbackIds.clear() - this.updatePendingActionsOnBlackboard() - void this.processQueue(bot) - }, this.feedbackBarrierTimeoutMs) - - this.updatePendingActionsOnBlackboard() - } - - // Record own chat actions to memory - for (const action of actionsWithIds) { - if (action.type === 'chat') { - this.blackboard.addChatMessage({ - sender: config.bot.username || '[Me]', - content: action.message, - timestamp: Date.now(), // FIXME: should be the time the action was issued - }) - } - } - - this.deps.taskExecutor.executeActions(actionsWithIds) - } - } - - private updatePerception(_bot: MineflayerWithAgents): void { - const ctx = this.deps.reflexManager.getContextSnapshot() - const view = buildConsciousContextView(ctx) - this.blackboard.updateContextView(view) - - // Sync Blackboard to Debug - this.debugService.updateBlackboard(this.blackboard) - } - - private async decide(sysPrompt: string, userMsg: string): Promise { + // 3. Call Neuri (Stateful) with retry logic const maxAttempts = 3 - - const decideOnce = async (): Promise => { - const request_start = Date.now() - const response = await this.deps.neuri.handleStateless( - [ - system(sysPrompt), - user(userMsg), - ], - async (ctx) => { - const completion = await ctx.reroute('action', ctx.messages, { - model: config.openai.model, - response_format: { type: 'json_object' }, - } as any) as any - - // Trace LLM - this.debugService.traceLLM({ - route: 'action', - messages: ctx.messages, - content: completion?.choices?.[0]?.message?.content, - usage: completion?.usage, - model: config.openai.model, - duration: Date.now() - request_start, - }) - - if (!completion || !completion.choices?.[0]?.message?.content) { - throw new Error('LLM failed to return content') - } - return completion.choices[0].message.content - }, - ) - - if (!response) - return null - - return parseLLMResponseJson(response) - } + let result: string | null = null for (let attempt = 1; attempt <= maxAttempts; attempt++) { try { - return await decideOnce() - } - catch (err) { + result = await this.deps.neuri.handle( + [ + // System prompt is inserted/updated in the callback to prevent duplication + user(userMessage), + ], + async (ctx) => { + let messages = ctx.messages + + // Ensure system prompt is at the start (or replace existing one) + const sysMsgIndex = messages.findIndex(m => m.role === 'system') + if (sysMsgIndex >= 0) { + messages[sysMsgIndex] = system(systemPrompt) + } else { + messages = [system(systemPrompt), ...messages] + } + + const traceStart = Date.now() + + const completion = await ctx.reroute('brain', messages, { + model: config.openai.model, + response_format: { type: 'json_object' } + } as any) as any + + const message = completion?.choices?.[0]?.message + const content = message?.content + const reasoning = message?.reasoning_content || message?.reasoning + + if (!content) throw new Error('No content from LLM') + + this.debugService.traceLLM({ + route: 'brain', + messages, + content, + reasoning, + usage: completion.usage, + model: config.openai.model, + duration: Date.now() - traceStart + }) + + this.debugService.emitBrainState({ + status: 'processing', + queueLength: this.queue.length, + lastContextView: this.lastContextView, + }) + + return content + } + ) + break // Success, exit retry loop + } catch (err) { const remaining = maxAttempts - attempt - // Retry all errors except auth/bad arg errors (which won't recover) const shouldRetry = remaining > 0 && !isLikelyAuthOrBadArgError(err) - this.log('ERROR', 'Brain: Decision attempt failed', { - error: err, - attempt, - remaining, - shouldRetry, - status: getErrorStatus(err), - code: getErrorCode(err), - }) + this.deps.logger.withError(err).error(`Brain: Decision attempt failed (attempt ${attempt}/${maxAttempts}, retry: ${shouldRetry})`) - if (shouldRetry) - continue - - const errMsg = toErrorMessage(err) - try { - this.bot?.bot?.chat?.(`[Brain] decide failed: ${errMsg}`) + if (!shouldRetry) { + throw err // Re-throw if we can't retry } - catch (chatErr) { - this.log('ERROR', 'Brain: Failed to send error message to chat', { error: chatErr }) - } - - return null + // Otherwise continue to next attempt } } - return null + // 4. Parse & Execute + if (!result) { + this.deps.logger.warn('Brain: No response after all retries') + return + } + + try { + const parsed = this.parseResponse(result) + const action = parsed.action + + if (action.tool === 'skip') { + this.deps.logger.log('INFO', 'Brain: Skipping turn (observing)') + return + } + + this.deps.logger.log('INFO', `Brain: Decided action: ${action.tool}`, { params: action.params }) + + // Check if action is read-only + const availableActions = this.deps.taskExecutor.getAvailableActions() + const actionDef = availableActions.find(a => a.name === action.tool) + + let token: CancellationToken | undefined + + if (actionDef?.readonly) { + // Read-only Actions: Do not cancel background/physical actions + // Can be executed in parallel with physical actions + token = undefined + } else { + // Physical Actions: Cancel previous background action + if (this.currentCancellationToken) { + this.currentCancellationToken.cancel() + } + this.currentCancellationToken = createCancellationToken() + token = this.currentCancellationToken + } + + // Execute + void this.deps.taskExecutor.executeAction(action, token) + + } catch (err) { + this.deps.logger.withError(err).error('Brain: Failed to execute decision') + void this.enqueueEvent(bot, { + type: 'feedback', + payload: { status: 'failure', error: toErrorMessage(err) }, + source: { type: 'system', id: 'brain' }, + timestamp: Date.now() + }) + } } - // --- Debug Helpers --- + private buildUserMessage(event: BotEvent, contextView: string): string { + const parts: string[] = [] - private log(level: 'INFO' | 'WARN' | 'ERROR' | 'DEBUG', message: string, fields?: any) { - // Dual logging: Console/File via Logger AND DebugServer - if (level === 'ERROR') - this.deps.logger.withError(fields?.error).error(message) - else if (level === 'WARN') - this.deps.logger.warn(message, fields) - else this.deps.logger.log(message, fields) + // 1. Event Content + if (event.type === 'perception') { + const signal = event.payload as PerceptionSignal + if (signal.type === 'chat_message') { + parts.push(`[EVENT] ${signal.description}`) + } else { + parts.push(`[EVENT] Perception Signal: ${signal.description}`) + } + } else if (event.type === 'feedback') { + const p = event.payload as any + const tool = p.action?.tool || 'unknown' + if (p.status === 'success') { + parts.push(`[FEEDBACK] ${tool}: Success. ${typeof p.result === 'string' ? p.result : JSON.stringify(p.result)}`) + } else { + parts.push(`[FEEDBACK] ${tool}: Failed. ${p.error}`) + } + } else { + parts.push(`[EVENT] ${event.type}: ${JSON.stringify(event.payload)}`) + } - this.debugService.log(level, message, fields) + // 2. Perception Snapshot Diff + // Compare with last + if (contextView !== this.lastContextView) { + parts.push(contextView) + // Note: We don't update this.lastContextView here; caller does it after building message + } + + return parts.join('\n\n') } - private updateDebugState(processingEvent?: BotEvent) { - this.debugService.updateQueue( - this.queue.map(q => q.event), - processingEvent, - ) + private parseResponse(content: string): BrainResponse { + const jsonStr = extractJsonCandidate(content) + try { + return JSON.parse(jsonStr) + } catch (e) { + throw new Error(`Invalid JSON response: ${content.substring(0, 100)}...`) + } } } diff --git a/services/minecraft/src/cognitive/conscious/prompts/brain-prompt.ts b/services/minecraft/src/cognitive/conscious/prompts/brain-prompt.ts index 830af9ee0..d32c146c7 100644 --- a/services/minecraft/src/cognitive/conscious/prompts/brain-prompt.ts +++ b/services/minecraft/src/cognitive/conscious/prompts/brain-prompt.ts @@ -1,133 +1,62 @@ import type { Action } from '../../../libs/mineflayer/action' -import type { Blackboard } from '../blackboard' -export function generateBrainSystemPrompt( - blackboard: Blackboard, - availableActions: Action[], -): string { - const now = Date.now() - - const formatAgo = (timestamp: number): string => { - const diffMs = Math.max(0, now - timestamp) - const s = Math.floor(diffMs / 1000) - return `${s}s ago` - } - - const withinLast = (timestamp: number, windowMs: number): boolean => { - return now - timestamp <= windowMs - } - - // Separate tools by execution type - const instantTools = availableActions.filter(a => a.execution === 'parallel') - const asyncActions = availableActions.filter(a => a.execution === 'sequential') - - const instantToolDefs = instantTools.map(a => ({ +export function generateBrainSystemPrompt(availableActions: Action[]): string { + const toolDefs = availableActions.map(a => ({ name: a.name, description: a.description, + parameters: a.schema, })) - const asyncActionDefs = asyncActions.map(a => ({ - name: a.name, - description: a.description, - })) + const toolsJson = JSON.stringify(toolDefs, null, 2) - const instantToolsJson = JSON.stringify(instantToolDefs, null, 2) - const asyncActionsJson = JSON.stringify(asyncActionDefs, null, 2) - - const recentWindowMs = 30_000 - - const recentActionLines = blackboard.recentActionHistory - .filter(a => withinLast(a.timestamp, recentWindowMs)) - .map(a => `- [${formatAgo(a.timestamp)}] ${a.line}`) - .join('\n') - - const recentChatLines = blackboard.chatHistory - .filter(m => withinLast(m.timestamp, recentWindowMs)) - .map(m => `- [${formatAgo(m.timestamp)}] ${m.sender}: ${m.content}`) - .join('\n') - - // TODO extract prompt components later - // e.g. personality should be included from somewhere else return ` # Role Definition -You are playing the game Minecraft. Your user name is [${blackboard.selfUsername}]. -You are NOT an assistant. You are an autonomous agent with a virtual body, senses, and needs. +You are an autonomous agent playing Minecraft. ---- +# Self-Knowledge & Capabilities +1. **Stateful Existence**: You maintain a memory of the conversation, but it's crucial to be aware that old history messages are less relevant than recent. +2. **One Action Per Turn**: You can perform exactly one action at a time. If you decide to act, you must wait for its feedback before acting again. +3. **Interruption**: The world is real-time. Events (chat, damage, etc.) may happen *while* you are performing an action. + - If a new critical event occurs, you may need to change your plans. + - Feedback for your actions will arrive as a message starting with \`[FEEDBACK]\`. +4. **Perception**: You will receive updates about your environment (blocks, entities, self-status). + - These appear as messages starting with \`[PERCEPTION]\`. + - Only changes are reported to save mental capacity. +5. **Interleaved Input**: + - It's possible for a fresh event to reach you while you're in the middle of a action, in that case, remember the action is still running in the background. + - If the new situation requires you to change plan, you can use the stop tool to stop background actions or initiate a new one, which will automatically replace the old one. + - Feel free to send chats while background actions are running, it will not interrupt them. -# Instant Tools (Native Tool Calls) +# Available Tools +You must use the following tools to interact with the world. +You cannot make up tools. You must use the JSON format described below. -These tools execute IMMEDIATELY and return results within this same turn. -Use them to gather information BEFORE deciding what actions to take. - -**How to use**: Invoke these by making native tool calls. -**Important**: Use instantTools only with native tool/function calling (the one with special tokens) -**On failure**: You will receive a [FAILED] message with suggestions. Use this to adjust your approach. - -${instantToolsJson} - ---- - -# Async Actions (JSON Output) - -These actions take TIME to complete (movement, crafting, combat, etc.). -They are queued and executed asynchronously after your response. - -**How to use**: Output these in the JSON "actions" array in your response. -**Feedback**: You will receive feedback when they complete(if require_feedback is true) or fail(always). - -${asyncActionsJson} - ---- +${toolsJson} # Response Format - -Your entire response must be valid JSON. Include only your thoughts, blackboard updates, and async actions. - -Rules for the "actions" array: -1. Actions are processed in the order you output them -2. Sequential actions are awaited strictly in order -3. Set "require_feedback": true if you need to know the result, it will be given to you in the next turn -4. Failed actions always trigger feedback -5. Use empty array if no action is needed -6. Perfer not to queue actions if possible +You must respond with valid JSON only. Do not include markdown code blocks (like \`\`\`json). +Your response determines your single action for this turn. Schema: { - "thought": "Your current thought, internal monologue and memory. Put everything that might be useful for the next turn here", - "blackboard": { - "UltimateGoal": "These 3 fields are functionally identical to the thought above", - "CurrentTask": "What ever you're up to right now", - "executionStrategy": "Short-term plan if any." - }, - "actions": [ - {"type":"sequential","step":{"tool":"goToPlayer","params":{"player_name":"Steve","closeness":3}},"require_feedback": true}, - {"type":"parallel","step":{"tool":"collectBlocks","params":{"type":"oak_log","num":5}},"require_feedback": false} - ] + "action": { + "tool": "toolName", + "params": { "key": "value" } + } } -# Understanding the Context +OR, if you want to do nothing (if you want to wait for something to happen, or to ignore): -Hint: When a player is talking about "there" or "that", it's possible that they're referencing the block they're currently looking at. -But you should always try to infer it from the context. +{ + "action": { + "tool": "skip", + "params": {} + } +} -The following blackboard provides you with information about your current state: - -Goal: "${blackboard.ultimate_goal}" -Thought: "${blackboard.current_task}" -Strategy: "${blackboard.strategy}" -Self: ${blackboard.selfSummary} -Environment: ${blackboard.environmentSummary} - -# Execution State -Ongoing actions still running: -${blackboard.pendingActions.map(a => `- ${a}`).join('\n') || '- none'} -NOTE: For most actions, you don't want to create a duplicate one if it's already running, in that case just do nothing. - -Recent actions and results: -${recentActionLines || '- none'} - -# Chat History -${recentChatLines || 'No recent messages.'} +# Rules +- **Native Reasoning**: You can think before outputting your action. +- **Strict JSON**: Output ONLY the JSON object. No preamble, no postscript. +- **Handling Feedback**: When you perform an action, you will see a \`[FEEDBACK]\` message in the history later with the result. Use this to verify success. ` } diff --git a/services/minecraft/src/composables/neuri.ts b/services/minecraft/src/composables/neuri.ts index a38d5963c..5447af674 100644 --- a/services/minecraft/src/composables/neuri.ts +++ b/services/minecraft/src/composables/neuri.ts @@ -1,25 +1,23 @@ import type { Agent, Neuri } from 'neuri' -import type { Mineflayer } from '../libs/mineflayer' -import { neuri } from 'neuri' -import { createActionNeuriAgent } from '../agents/action/adapter' -import { createChatNeuriAgent } from '../agents/chat/llm' -import { createPlanningNeuriAgent } from '../agents/planning/adapter' +import { agent, neuri } from 'neuri' + + import { useLogger } from '../utils/logger' import { config } from './config' let neuriAgent: Neuri | undefined const agents = new Set>() -export async function createNeuriAgent(mineflayer: Mineflayer): Promise { +export async function createNeuriAgent(): Promise { useLogger().log('Initializing neuri agent') let n = neuri() - agents.add(createPlanningNeuriAgent()) - agents.add(createActionNeuriAgent(mineflayer)) - agents.add(createChatNeuriAgent()) + agents.add(agent('brain').build()) + // agents.add(createPlanningNeuriAgent()) // Deprecated by Brain + // agents.add(createChatNeuriAgent()) // Deprecated by Brain agents.forEach(agent => n = n.agent(agent)) diff --git a/services/minecraft/src/debug/debug-service.ts b/services/minecraft/src/debug/debug-service.ts index 028c5ca26..0d96cb9e2 100644 --- a/services/minecraft/src/debug/debug-service.ts +++ b/services/minecraft/src/debug/debug-service.ts @@ -1,4 +1,4 @@ -import type { BlackboardEvent, ClientCommand, LLMTraceEvent, LogEvent, QueueEvent, ReflexStateEvent, SaliencyEvent, ServerEvent, TraceEvent } from './types' +import type { BlackboardEvent, BrainStateEvent, ClientCommand, LLMTraceEvent, LogEvent, QueueEvent, ReflexStateEvent, SaliencyEvent, ServerEvent, TraceEvent } from './types' import { DebugServer } from './server' @@ -73,14 +73,16 @@ export class DebugService { this.server.broadcast(event) } + + /** - * Emit a blackboard state update + * Emit a brain state update */ - public updateBlackboard(state: BlackboardEvent['state'] | { goal?: string, thought?: string, strategy?: string }): void { + public emitBrainState(state: Omit): void { const event: ServerEvent = { - type: 'blackboard', + type: 'brain_state', payload: { - state, + ...state, timestamp: Date.now(), }, } diff --git a/services/minecraft/src/debug/types.ts b/services/minecraft/src/debug/types.ts index ec47121de..e10c2036f 100644 --- a/services/minecraft/src/debug/types.ts +++ b/services/minecraft/src/debug/types.ts @@ -17,6 +17,7 @@ export interface LLMTraceEvent { route: string messages: unknown[] content: string + reasoning?: string usage?: { prompt_tokens?: number completion_tokens?: number @@ -27,6 +28,14 @@ export interface LLMTraceEvent { timestamp: number } +export interface BrainStateEvent { + status: 'idle' | 'processing' | 'waiting' + queueLength: number + lastContextView?: string + currentAction?: string + timestamp: number +} + export interface BlackboardEvent { state: Record timestamp: number @@ -138,17 +147,18 @@ export interface ToolExecutionResultEvent { export type ServerEvent = | { type: 'log', payload: LogEvent } - | { type: 'llm', payload: LLMTraceEvent } - | { type: 'blackboard', payload: BlackboardEvent } - | { type: 'queue', payload: QueueEvent } - | { type: 'saliency', payload: SaliencyEvent } - | { type: 'reflex', payload: ReflexStateEvent } - | { type: 'trace', payload: TraceEvent } - | { type: 'trace_batch', payload: TraceBatchEvent } - | { type: 'history', payload: ServerEvent[] } - | { type: 'pong', payload: { timestamp: number } } - | { type: 'debug:tools_list', payload: { tools: ToolDefinition[] } } - | { type: 'debug:tool_result', payload: ToolExecutionResultEvent } + | { type: 'llm', payload: LLMTraceEvent } + | { type: 'blackboard', payload: BlackboardEvent } + | { type: 'queue', payload: QueueEvent } + | { type: 'saliency', payload: SaliencyEvent } + | { type: 'reflex', payload: ReflexStateEvent } + | { type: 'trace', payload: TraceEvent } + | { type: 'trace_batch', payload: TraceBatchEvent } + | { type: 'history', payload: ServerEvent[] } + | { type: 'pong', payload: { timestamp: number } } + | { type: 'debug:tools_list', payload: { tools: ToolDefinition[] } } + | { type: 'debug:tool_result', payload: ToolExecutionResultEvent } + | { type: 'brain_state', payload: BrainStateEvent } // ============================================================ // Client -> Server commands @@ -201,12 +211,12 @@ export interface RequestToolsCommand { export type ClientCommand = | ClearLogsCommand - | SetFilterCommand - | InjectEventCommand - | PingCommand - | RequestHistoryCommand - | ExecuteToolCommand - | RequestToolsCommand + | SetFilterCommand + | InjectEventCommand + | PingCommand + | RequestHistoryCommand + | ExecuteToolCommand + | RequestToolsCommand // ============================================================ // Wire format diff --git a/services/minecraft/src/debug/web/app.js b/services/minecraft/src/debug/web/app.js index 9280aefd3..79ee0055e 100644 --- a/services/minecraft/src/debug/web/app.js +++ b/services/minecraft/src/debug/web/app.js @@ -357,47 +357,49 @@ class ReflexPanel { } // ============================================================================= -// Blackboard Panel +// Brain Panel // ============================================================================= -class BlackboardPanel { +class BrainPanel { constructor(client) { this.client = client - this.state = {} + this.state = null this.elements = { - json: document.getElementById('blackboard-json'), - copyBtn: document.getElementById('blackboard-copy-btn'), + status: document.getElementById('brain-status'), + queue: document.getElementById('brain-queue'), + context: document.getElementById('brain-context'), } } init() { - this.client.on('blackboard', data => this.update(data)) + this.client.on('brain_state', data => this.update(data)) this.client.on('connected', () => this.reset()) - this.elements.copyBtn.addEventListener('click', () => this.copy()) - this.render() } update(data) { - this.state = data.state || {} + this.state = data this.render() } reset() { - this.state = {} + this.state = null this.render() } render() { - this.elements.json.textContent = JSON.stringify(this.state, null, 2) - } + if (!this.state) { + this.elements.status.textContent = 'Unknown' + this.elements.queue.textContent = '-' + this.elements.context.textContent = '' + return + } - copy() { - navigator.clipboard.writeText(JSON.stringify(this.state, null, 2)) - .then(() => { - this.elements.copyBtn.textContent = 'โœ“' - setTimeout(() => { this.elements.copyBtn.textContent = '๐Ÿ“‹' }, 1000) - }) - .catch(err => console.error('Copy failed:', err)) + this.elements.status.textContent = this.state.status.toUpperCase() + this.elements.status.className = `status-badge status-${this.state.status}` + this.elements.queue.textContent = this.state.queueLength + + // Render context view (it's markdown/text) + this.elements.context.textContent = this.state.lastContextView || '(No context yet)' } } @@ -630,6 +632,10 @@ class LLMPanel {
+ ${trace.reasoning ? ` +
Reasoning
+
${escapeHtml(trace.reasoning)}
+ ` : ''}
Result
${escapeHtml(trace.content || '')}
@@ -1308,7 +1314,7 @@ class DebugApp { this.layoutManager = new LayoutManager() this.queuePanel = new QueuePanel(this.client) this.reflexPanel = new ReflexPanel(this.client) - this.blackboardPanel = new BlackboardPanel(this.client) + this.brainPanel = new BrainPanel(this.client) this.logsPanel = new LogsPanel(this.client) this.llmPanel = new LLMPanel(this.client) this.saliencyPanel = new SaliencyPanel(this.client) @@ -1318,7 +1324,7 @@ class DebugApp { this.panels = { queue: this.queuePanel, reflex: this.reflexPanel, - blackboard: this.blackboardPanel, + brain: this.brainPanel, logs: this.logsPanel, llm: this.llmPanel, saliency: this.saliencyPanel, diff --git a/services/minecraft/src/debug/web/index.html b/services/minecraft/src/debug/web/index.html index 8287929c0..d3c6b6b4f 100644 --- a/services/minecraft/src/debug/web/index.html +++ b/services/minecraft/src/debug/web/index.html @@ -105,17 +105,19 @@
- -
+ +
-

Blackboard

+

Brain

- + Unknown + 0
- 
{}
+
diff --git a/services/minecraft/src/debug/web/styles.css b/services/minecraft/src/debug/web/styles.css index 1afaa9abf..cfb149df8 100644 --- a/services/minecraft/src/debug/web/styles.css +++ b/services/minecraft/src/debug/web/styles.css @@ -630,6 +630,14 @@ button:hover, word-wrap: break-word; } +.llm-content.reasoning { + background: var(--bg-tertiary); + color: var(--text-muted); + font-family: var(--font-mono); + font-style: italic; + border-left: 2px solid var(--accent-primary); +} + .llm-messages { display: flex; flex-direction: column; diff --git a/services/minecraft/src/libs/mineflayer/action.ts b/services/minecraft/src/libs/mineflayer/action.ts index 72c518654..6834816e3 100644 --- a/services/minecraft/src/libs/mineflayer/action.ts +++ b/services/minecraft/src/libs/mineflayer/action.ts @@ -4,12 +4,10 @@ import type { Mineflayer } from './core' type ActionResult = string | Promise -export type ActionExecutionMode = 'sequential' | 'parallel' - export interface Action { readonly name: string readonly description: string - readonly execution: ActionExecutionMode readonly schema: z.ZodObject + readonly readonly?: boolean readonly perform: (mineflayer: Mineflayer) => (...args: any[]) => ActionResult } diff --git a/services/minecraft/src/main.ts b/services/minecraft/src/main.ts index 143a548d1..c26ebe0aa 100644 --- a/services/minecraft/src/main.ts +++ b/services/minecraft/src/main.ts @@ -48,7 +48,7 @@ async function main() { }) // Dynamically load CognitiveEngine after the bot is initialized - const agent = await createNeuriAgent(bot) + const agent = await createNeuriAgent() await bot.loadPlugin(CognitiveEngine({ agent, airiClient })) // Setup Tool Executor for Debug Dashboard