From 8facbdacfcced03da3e98c7bfef694fbe17a2ca1 Mon Sep 17 00:00:00 2001 From: Rin Date: Mon, 29 Dec 2025 01:55:36 +0800 Subject: [PATCH] feat(minecraft): architecture redesign, blackboard pattern and OODA --- services/minecraft/README.md | 123 +++----- services/minecraft/src/agents/chat/index.ts | 25 ++ .../minecraft/src/agents/planning/index.ts | 104 +------ .../src/cognitive/action/task-executor.ts | 148 +++++++++ .../minecraft/src/cognitive/action/types.ts | 20 ++ .../src/cognitive/conscious/blackboard.ts | 83 +++++ .../src/cognitive/conscious/brain.ts | 215 +++++++++++++ .../src/cognitive/conscious/orchestrator.ts | 292 +++++++++++------- .../src/cognitive/conscious/task-manager.ts | 172 +++++++---- services/minecraft/src/cognitive/container.ts | 10 +- services/minecraft/src/cognitive/index.ts | 26 +- .../src/libs/mineflayer/base-agent.ts | 6 +- services/minecraft/src/skills/crafting.ts | 9 +- services/minecraft/src/web/dashboard.html | 60 ++-- 14 files changed, 885 insertions(+), 408 deletions(-) create mode 100644 services/minecraft/src/cognitive/action/task-executor.ts create mode 100644 services/minecraft/src/cognitive/action/types.ts create mode 100644 services/minecraft/src/cognitive/conscious/blackboard.ts create mode 100644 services/minecraft/src/cognitive/conscious/brain.ts diff --git a/services/minecraft/README.md b/services/minecraft/README.md index 3ccb117d2..8eb604827 100644 --- a/services/minecraft/README.md +++ b/services/minecraft/README.md @@ -106,7 +106,7 @@ You can also give the bot natural language commands, and it will try to understa ## 🧠 Cognitive Architecture -AIRI's Minecraft agent is built on a **three-layered cognitive architecture** inspired by cognitive science, enabling both reactive and deliberate behaviors. This design allows the bot to respond instantly to urgent situations while maintaining the ability to plan and execute complex tasks. +AIRI's Minecraft agent is built on a **four-layered cognitive architecture** inspired by cognitive science, enabling reactive, conscious, and physically grounded behaviors. ### Architecture Overview @@ -124,136 +124,91 @@ graph TB RM --> FSM end - subgraph "Layer C: Conscious" + subgraph "Layer C: Conscious (Reasoning)" ORC[Orchestrator] - Planning[Planning Agent] - Action[Action Agent] - Chat[Chat Agent] - ORC --> Planning - ORC --> Action + Planner[Planning Agent (LLM)] + Chat[Chat Agent (LLM)] + ORC --> Planner ORC --> Chat end + subgraph "Layer D: Action (Execution)" + TE[Task Executor] + AA[Action Agent] + Planner -->|Plan| TE + TE -->|Action Steps| AA + end + EM -->|High Priority| RM EM -->|All Events| ORC RM -.->|Inhibition Signal| ORC + ORC -->|Execution Request| TE style EM fill:#e1f5ff style RM fill:#fff4e1 style ORC fill:#ffe1f5 + style TE fill:#dcedc8 ``` ### Layer A: Perception **Location**: `src/cognitive/perception/` -The perception layer acts as the sensory input hub, receiving and preprocessing all events from the Minecraft world and external sources. +The perception layer acts as the sensory input hub, receiving and preprocesses all events from the Minecraft world and external sources. **Components**: - **Event Manager** (`event-manager.ts`): Centralized event distribution system - Emits standardized `BotEvent` objects - - Supports event prioritization (TODO: salience detection) - - Manages temporal context (TODO: short-term event memory) - -**Event Types**: -- `user_intent`: Player chat messages, voice commands -- `world_update`: Block changes, entity movements, damage events -- `system_alert`: Internal system notifications - -**Event Flow**: -```typescript -// Example: Chat message → Event -{ - type: 'user_intent', - payload: { content: 'build a house' }, - source: { type: 'minecraft', id: 'player123' }, - timestamp: 1234567890, - priority: 0, // Default priority - handled: false // Not yet processed -} -``` + - Supports event prioritization and concurrency ### Layer B: Reflex **Location**: `src/cognitive/reflex/` -The reflex layer handles immediate, instinctive reactions without LLM overhead. It operates on a finite state machine (FSM) pattern for predictable, fast responses. +The reflex layer handles immediate, instinctive reactions. It operates on a finite state machine (FSM) pattern for predictable, fast responses. **Components**: -- **Reflex Manager** (`reflex-manager.ts`): Coordinates all reflex behaviors - - Subscribes to high-priority events - - Executes instant responses - - Sets inhibition signals to prevent unnecessary LLM calls - -**Current Reflexes**: -- ✅ **Greeting Reflex**: Instantly responds to "hi" or "hello" -- 🚧 **Dodge Reflex** (TODO): Avoid incoming projectiles -- 🚧 **Survival Reflex** (TODO): Auto-eat when hungry, flee from danger - -**Inhibition Mechanism**: -When a reflex handles an event, it sets `event.handled = true`, preventing the expensive Conscious layer from processing the same event. - -```typescript -// Example: Greeting reflex -if (content === 'hi') { - bot.chat('Hi there! (Reflex)') - event.handled = true // Inhibit Conscious processing -} -``` +- **Reflex Manager** (`reflex-manager.ts`): Coordinates reflex behaviors +- **Inhibition**: Reflexes can inhibit Conscious layer processing to prevent redundant responses. ### Layer C: Conscious **Location**: `src/cognitive/conscious/` -The conscious layer handles complex reasoning, planning, and decision-making using LLM-powered agents. +The conscious layer handles complex reasoning, planning, and high-level decision-making. No physical execution happens here anymore. **Components**: -- **Orchestrator** (`orchestrator.ts`): Main coordinator for deliberate actions - - Checks inhibition signals from Reflex layer - - Manages processing state (prevents concurrent operations) - - Coordinates Planning → Execution → Response flow +- **Orchestrator**: Coordinates "Thinking" vs "Chatting" tasks. +- **Task Manager**: Manages concurrent Primary (Physical) and Secondary (Mental) tasks. +- **Planning Agent**: pure LLM reasoning to generate plans. +- **Chat Agent**: Generates natural language responses. -- **Planning Agent**: Creates multi-step plans to achieve goals -- **Action Agent**: Executes atomic actions (move, mine, build) -- **Chat Agent**: Generates natural language responses +### Layer D: Action -**Processing Pipeline**: -``` -1. Check Inhibition → 2. Update Memory → 3. Create Plan → -4. Execute Actions → 5. Generate Response → 6. Reply -``` +**Location**: `src/cognitive/action/` -**State Management**: -- Uses `isProcessing` lock to prevent race conditions -- Future: Queue system for handling concurrent intents +The action layer is responsible for the actual execution of tasks in the world. It isolates "Doing" from "Thinking". + +**Components**: +- **Task Executor**: Receives a `Plan` and executes it step-by-step. Handles retry logic and errors. +- **Action Agent**: The interface to low-level Mineflayer skills (move, place, break). ### 🔄 Event Flow Example -**Scenario 1: Simple Greeting (Reflex)** -``` -Player: "hi" - ↓ -[Perception] EventManager emits user_intent - ↓ -[Reflex] ReflexManager detects greeting → Replies instantly - ↓ -[Conscious] Orchestrator sees handled=true → Skips processing -``` - -**Scenario 2: Complex Command (Conscious)** +**Scenario: "Build a house"** ``` Player: "build a house" ↓ -[Perception] EventManager emits user_intent +[Perception] Event detected ↓ -[Reflex] ReflexManager ignores (not a reflex trigger) +[Conscious] Architect plans the structure ↓ -[Conscious] Orchestrator processes: - - PlanningAgent creates building plan - - ActionAgent executes steps (gather, place blocks) - - ChatAgent generates response +[Action] Executor takes the plan and manages the construction loop: + - Step 1: Collect wood (calls ActionAgent) + - Step 2: Craft planks + - Step 3: Build walls ↓ -Bot: "I've built a small house for you!" +[Conscious] ChatAgent confirms completion: "House is ready!" ``` ### 📁 Project Structure diff --git a/services/minecraft/src/agents/chat/index.ts b/services/minecraft/src/agents/chat/index.ts index 5def0e02d..3c51fd5d1 100644 --- a/services/minecraft/src/agents/chat/index.ts +++ b/services/minecraft/src/agents/chat/index.ts @@ -1,6 +1,8 @@ +import type { Mineflayer } from '../../libs/mineflayer' import type { ChatAgent } from '../../libs/mineflayer/base-agent' import type { ChatAgentConfig, ChatContext } from './types' +import { useBot } from '../../composables/bot' import { AbstractAgent } from '../../libs/mineflayer/base-agent' import { generateChatResponse } from './llm' @@ -10,13 +12,17 @@ export class ChatAgentImpl extends AbstractAgent implements ChatAgent { private maxHistoryLength: number private idleTimeout: number private llmConfig: ChatAgentConfig['llm'] + private mineflayer: Mineflayer constructor(config: ChatAgentConfig) { super(config) this.activeChats = new Map() this.maxHistoryLength = config.maxHistoryLength ?? 50 this.idleTimeout = config.idleTimeout ?? 5 * 60 * 1000 // 5 minutes + this.maxHistoryLength = config.maxHistoryLength ?? 50 + this.idleTimeout = config.idleTimeout ?? 5 * 60 * 1000 // 5 minutes this.llmConfig = config.llm + this.mineflayer = useBot().bot } protected async initializeAgent(): Promise { @@ -67,6 +73,25 @@ export class ChatAgentImpl extends AbstractAgent implements ChatAgent { } } + public async sendMessage(message: string): Promise { + if (!this.initialized) { + throw new Error('Chat agent not initialized') + } + this.logger.withField('message', message).log('Sending message') + + // We also record our own messages in history (handled by on('chat') listener usually? + // No, on('chat') filters out bot's own messages in LLMAgent plugin usually to avoid loops. + // So we manually add to history? + // Wait, the orchestrator call bot.chat() before. Did it record it? + // In processMessage, we added generated response to history. + // Here we are just sending. We should record it. + // But we don't know the receiver context? Global chat. + // If it's global chat, we treat it as broadcast. + + // Just send for now. + this.mineflayer.bot.chat(message) + } + public startConversation(player: string): void { if (!this.initialized) { throw new Error('Chat agent not initialized') diff --git a/services/minecraft/src/agents/planning/index.ts b/services/minecraft/src/agents/planning/index.ts index 5cba9e878..79496af42 100644 --- a/services/minecraft/src/agents/planning/index.ts +++ b/services/minecraft/src/agents/planning/index.ts @@ -1,12 +1,10 @@ import type { Neuri } from 'neuri' import type { Action } from '../../libs/mineflayer/action' -import type { ActionAgent, AgentConfig, MemoryAgent, Plan, PlanningAgent } from '../../libs/mineflayer/base-agent' +import type { AgentConfig, MemoryAgent, Plan, PlanningAgent } from '../../libs/mineflayer/base-agent' import type { PlanStep } from './adapter' import { AbstractAgent } from '../../libs/mineflayer/base-agent' -import { ActionError } from '../../utils/errors' -import { ActionAgentImpl } from '../action' import { PlanningLLMHandler } from './adapter' interface PlanContext { @@ -17,6 +15,7 @@ interface PlanContext { retryCount: number isGenerating: boolean pendingSteps: PlanStep[] + availableActions?: Action[] } export interface PlanningAgentConfig extends AgentConfig { @@ -30,7 +29,6 @@ export class PlanningAgentImpl extends AbstractAgent implements PlanningAgent { public readonly type = 'planning' as const private currentPlan: Plan | null = null private context: PlanContext | null = null - private actionAgent: ActionAgent | null = null private memoryAgent: MemoryAgent | null = null private llmConfig: PlanningAgentConfig['llm'] private llmHandler: PlanningLLMHandler @@ -47,13 +45,6 @@ export class PlanningAgentImpl extends AbstractAgent implements PlanningAgent { protected async initializeAgent(): Promise { this.logger.log('Initializing planning agent') - // Create action agent directly - this.actionAgent = new ActionAgentImpl({ - id: 'action', - type: 'action', - }) - await this.actionAgent.init() - // Set event listener this.on('message', async ({ sender, message }) => { await this.handleAgentMessage(sender, message) @@ -67,12 +58,11 @@ export class PlanningAgentImpl extends AbstractAgent implements PlanningAgent { protected async destroyAgent(): Promise { this.currentPlan = null this.context = null - this.actionAgent = null this.memoryAgent = null this.removeAllListeners() } - public async createPlan(goal: string): Promise { + public async createPlan(goal: string, availableActions: Action[] = []): Promise { if (!this.initialized) { throw new Error('Planning agent not initialized') } @@ -87,8 +77,7 @@ export class PlanningAgentImpl extends AbstractAgent implements PlanningAgent { return cachedPlan } - // Get available actions from action agent - const availableActions = this.actionAgent?.getAvailableActions() ?? [] + // Actions passed from Orchestrator/Executor // Check if the goal requires actions const requirements = this.parseGoalRequirements(goal) @@ -128,6 +117,7 @@ export class PlanningAgentImpl extends AbstractAgent implements PlanningAgent { retryCount: 0, isGenerating: false, pendingSteps: [], + availableActions, } return plan @@ -138,83 +128,7 @@ export class PlanningAgentImpl extends AbstractAgent implements PlanningAgent { } } - public async executePlan(plan: Plan, cancellationToken?: any): Promise { - if (!this.initialized) { - throw new Error('Planning agent not initialized') - } - - if (!plan.requiresAction) { - this.logger.log('Plan does not require actions, skipping execution') - return - } - - if (!this.actionAgent) { - throw new Error('Action agent not available') - } - - this.logger.withField('plan', plan).log('Executing plan') - - try { - plan.status = 'in_progress' - this.currentPlan = plan - - // Execute each step - for (const step of plan.steps) { - // Check for cancellation before each step - if (cancellationToken?.isCancelled) { - this.logger.log('Plan execution cancelled') - plan.status = 'cancelled' - return - } - - try { - this.logger.withField('step', step).log('Executing step') - await this.actionAgent.performAction(step) - } - catch (stepError) { - if (stepError instanceof ActionError) { - this.logger.withError(stepError).warn('Step execution failed with ActionError') - // If it's a resource failure or crafting failure that we've already tried to fix (implied by fail-fast skills), - // then we should abort and report failure instead of looping. - // We can check error types or context. - if (stepError.code === 'RESOURCE_MISSING' || stepError.code === 'CRAFTING_FAILED' || stepError.code === 'INVENTORY_FULL') { - // For now, fail fast on these hard errors. - // In the future we might want a "replanning" phase here, but NOT a blind retry. - throw stepError - } - } - - this.logger.withError(stepError).error('Failed to execute step') - - // Attempt to adjust plan and retry - if (this.context && this.context.retryCount < 3) { - this.context.retryCount++ - // Adjust plan and restart - const adjustedPlan = await this.adjustPlan( - plan, - stepError instanceof Error ? stepError.message : 'Unknown error', - 'system', - ) - await this.executePlan(adjustedPlan, cancellationToken) - return - } - - throw stepError - } - } - - plan.status = 'completed' - } - catch (error) { - plan.status = 'failed' - throw error // This will be caught by handleChatMessage and reported to user - } - finally { - this.context = null - } - } - - public async adjustPlan(plan: Plan, feedback: string, sender: string): Promise { + public async adjustPlan(plan: Plan, feedback: string, sender: string, availableActions: Action[] = []): Promise { if (!this.initialized) { throw new Error('Planning agent not initialized') } @@ -225,13 +139,13 @@ export class PlanningAgentImpl extends AbstractAgent implements PlanningAgent { // If there's a current context, use it to adjust the plan if (this.context) { const currentStep = this.context.currentStep - const availableActions = this.actionAgent?.getAvailableActions() ?? [] + const actions = availableActions.length > 0 ? availableActions : (this.context.availableActions || []) // Generate recovery steps based on feedback const recoverySteps = this.generateRecoverySteps(feedback) // Generate new steps from the current point - const newSteps = await this.generatePlanSteps(plan.goal, availableActions, sender, feedback) + const newSteps = await this.generatePlanSteps(plan.goal, actions, sender, feedback) // Create adjusted plan const adjustedPlan: Plan = { @@ -249,7 +163,7 @@ export class PlanningAgentImpl extends AbstractAgent implements PlanningAgent { } // If no context, create a new plan - return this.createPlan(plan.goal) + return this.createPlan(plan.goal, availableActions) } catch (error) { this.logger.withError(error).error('Failed to adjust plan') diff --git a/services/minecraft/src/cognitive/action/task-executor.ts b/services/minecraft/src/cognitive/action/task-executor.ts new file mode 100644 index 000000000..9257c12f3 --- /dev/null +++ b/services/minecraft/src/cognitive/action/task-executor.ts @@ -0,0 +1,148 @@ +import type { ActionAgent, ChatAgent, Plan } from '../../libs/mineflayer/base-agent' +import type { Logger } from '../../utils/logger' +import type { ActionInstruction } from './types' + +import { EventEmitter } from 'node:events' + +import { ActionError } from '../../utils/errors' + +interface CancellationToken { + isCancelled: boolean +} + +interface TaskExecutorConfig { + logger: Logger + actionAgent: ActionAgent + chatAgent: ChatAgent +} + +export class TaskExecutor extends EventEmitter { + private actionAgent: ActionAgent + private chatAgent: ChatAgent + private logger: Logger + private initialized = false + + constructor(config: TaskExecutorConfig) { + super() + this.logger = config.logger + this.actionAgent = config.actionAgent + this.chatAgent = config.chatAgent + } + + public async initialize(): Promise { + if (this.initialized) + return + + this.logger.log('Initializing Task Executor') + // ActionAgent is initialized by container/orchestrator + this.initialized = true + } + + public async destroy(): Promise { + this.initialized = false + // ActionAgentImpl doesn't expose destroy publicly in interface but defines it? + // Checking AbstractAgent, yes it has destroy(). + // We cast to access it or trust it's handled. + // For now, assume we don't need explicit destroy of ActionAgent if it just clears listeners. + } + + public async executePlan(plan: Plan, cancellationToken?: CancellationToken): Promise { + if (!this.initialized) { + throw new Error('TaskExecutor not initialized') + } + + if (!plan.requiresAction) { + this.logger.log('Plan does not require actions, skipping execution') + return + } + + this.logger.withField('plan', plan).log('Executing plan') + + try { + plan.status = 'in_progress' + + // Execute each step + for (const step of plan.steps) { + // Check for cancellation before each step + if (cancellationToken?.isCancelled) { + this.logger.log('Plan execution cancelled') + plan.status = 'cancelled' + return + } + + try { + this.logger.withField('step', step).log('Executing step') + await this.actionAgent.performAction(step) + } + catch (stepError: any) { + if (stepError instanceof ActionError) { + this.logger.withError(stepError).warn('Step execution failed with ActionError') + // Fail fast on hard errors + if (stepError.code === 'RESOURCE_MISSING' || stepError.code === 'CRAFTING_FAILED' || stepError.code === 'INVENTORY_FULL') { + throw stepError + } + } + + this.logger.withError(stepError).error('Failed to execute step') + + // Re-throw to let Orchestrator handle retry logic + throw stepError + } + } + + plan.status = 'completed' + } + catch (error) { + plan.status = 'failed' + throw error + } + } + + public executeActions(actions: ActionInstruction[], cancellationToken?: CancellationToken): void { + if (!this.initialized) { + throw new Error('TaskExecutor not initialized') + } + + this.logger.withField('count', actions.length).log('Executing actions') + + // Execute each action independently and asynchronously + actions.forEach(async (action) => { + if (cancellationToken?.isCancelled) { + this.logger.log('Action execution cancelled before start') + return + } + + this.emit('action:started', { action }) + + try { + let result: string | void + if (action.type === 'physical') { + result = await this.actionAgent.performAction(action.step) + } + else if (action.type === 'chat') { + await this.chatAgent.sendMessage(action.message) + result = 'Message sent' + } + else { + throw new Error(`Unknown action type: ${(action as any).type}`) + } + + if (cancellationToken?.isCancelled) { + // If cancelled during execution (and agent didn't throw), we might still consider it cancelled? + // But usually agents throw if cancelled. + // Just emit completed for now if it finished. + } + + this.emit('action:completed', { action, result }) + } + catch (error) { + this.logger.withError(error).error('Action execution failed') + this.emit('action:failed', { action, error }) + } + }) + } + + public getAvailableActions() { + return this.actionAgent.getAvailableActions() + } +} diff --git a/services/minecraft/src/cognitive/action/types.ts b/services/minecraft/src/cognitive/action/types.ts new file mode 100644 index 000000000..905dbd812 --- /dev/null +++ b/services/minecraft/src/cognitive/action/types.ts @@ -0,0 +1,20 @@ +import type { PlanStep } from '../../agents/planning/adapter' + +export type ActionType = 'physical' | 'chat' + +export interface BaseActionInstruction { + type: ActionType + description?: string +} + +export interface PhysicalActionInstruction extends BaseActionInstruction { + type: 'physical' + step: PlanStep +} + +export interface ChatActionInstruction extends BaseActionInstruction { + type: 'chat' + message: string +} + +export type ActionInstruction = PhysicalActionInstruction | ChatActionInstruction diff --git a/services/minecraft/src/cognitive/conscious/blackboard.ts b/services/minecraft/src/cognitive/conscious/blackboard.ts new file mode 100644 index 000000000..d9b4f2c8d --- /dev/null +++ b/services/minecraft/src/cognitive/conscious/blackboard.ts @@ -0,0 +1,83 @@ +import { Vec3 } from 'vec3' + +export interface SelfState { + status: 'idle' | 'moving' | 'working' | 'chatting' | 'busy' + location: Vec3 | null + holding: string | null + health: number + food: number + oxygen: number +} + +export interface EnvironmentState { + time: string // 'day' | 'night' | 'sunset' | 'sunrise' + weather: 'clear' | 'rain' | 'thunder' + nearbyPlayers: string[] + nearbyEntities: string[] // significant entities (mobs, dropped items of interest) + lightLevel: number +} + +export interface BlackboardState { + currentGoal: string + currentThought: string + executionStrategy: string + self: SelfState + environment: EnvironmentState +} + +export class Blackboard { + private _state: BlackboardState + + constructor() { + this._state = { + currentGoal: 'Idle', + currentThought: 'I am waiting for something to happen.', + executionStrategy: 'Observe surroundings.', + self: { + status: 'idle', + location: null, + holding: null, + health: 20, + food: 20, + oxygen: 20, + }, + environment: { + time: 'day', + weather: 'clear', + nearbyPlayers: [], + nearbyEntities: [], + lightLevel: 15, + }, + } + } + + // Getters + public get goal(): string { return this._state.currentGoal } + public get thought(): string { return this._state.currentThought } + public get strategy(): string { return this._state.executionStrategy } + public get self(): SelfState { return this._state.self } + public get environment(): EnvironmentState { return this._state.environment } + + // Setters (Partial updates allowed) + public update(updates: Partial): void { + this._state = { ...this._state, ...updates } + } + + public updateSelf(updates: Partial): void { + this._state.self = { ...this._state.self, ...updates } + } + + public updateEnvironment(updates: Partial): void { + this._state.environment = { ...this._state.environment, ...updates } + } + + public getSnapshot(): BlackboardState { + // Return a deep copy or safe reference? + // For now, return a shallow copy of the state structure + return { + ...this._state, + self: { ...this._state.self }, // location (Vec3) is an object, but usually treated efficiently. + environment: { ...this._state.environment, nearbyPlayers: [...this._state.environment.nearbyPlayers], nearbyEntities: [...this._state.environment.nearbyEntities] } + } + } +} diff --git a/services/minecraft/src/cognitive/conscious/brain.ts b/services/minecraft/src/cognitive/conscious/brain.ts new file mode 100644 index 000000000..b3fb755e5 --- /dev/null +++ b/services/minecraft/src/cognitive/conscious/brain.ts @@ -0,0 +1,215 @@ +import type { Logg } from '@guiiai/logg' +import type { Neuri } from 'neuri' + +import type { TaskExecutor } from '../action/task-executor' +import type { ActionInstruction } from '../action/types' +import type { EventManager } from '../perception/event-manager' +import type { MineflayerWithAgents, UserIntentPayload } from '../types' + +import { system, user } from 'neuri/openai' +import { zodToJsonSchema } from 'zod-to-json-schema' + +import { Blackboard } from './blackboard' + +interface BrainDeps { + eventManager: EventManager + neuri: Neuri + logger: Logg + taskExecutor: TaskExecutor +} + +interface BrainResponse { + thought: string + blackboard: { + currentGoal?: string + currentThought?: string + executionStrategy?: string + } + actions: ActionInstruction[] +} + +export class Brain { + private blackboard: Blackboard + + constructor(private readonly deps: BrainDeps) { + this.blackboard = new Blackboard() + } + + public init(bot: MineflayerWithAgents): void { + this.deps.logger.log('Brain: Initializing...') + + // Listen to User Intents (Chat/Voice) + // We treat these as "Sensory Inputs" that trigger the Cognitive Cycle + this.deps.eventManager.on('user_intent', async (event) => { + this.deps.logger.log(`Brain: Received intent from ${event.source.id}: ${event.payload.content}`) + await this.processEvent(bot, event) + }) + + // Listen to Task Execution Events (Action Feedback) + this.deps.taskExecutor.on('action:completed', async ({ action, result }) => { + this.deps.logger.log(`Brain: Action completed: ${action.type}`) + await this.processEvent(bot, { + type: 'action:feedback', + payload: { + status: 'success', + action, + result, + }, + source: { type: 'system', id: 'executor' }, + timestamp: Date.now(), + }) + }) + + this.deps.taskExecutor.on('action:failed', async ({ action, error }) => { + this.deps.logger.withError(error).warn(`Brain: Action failed: ${action.type}`) + await this.processEvent(bot, { + type: 'action:feedback', + payload: { + status: 'failure', + action, + error: error.message || error, + }, + source: { type: 'system', id: 'executor' }, + timestamp: Date.now(), + }) + }) + + this.deps.logger.log('Brain: Online.') + } + + private async processEvent(bot: MineflayerWithAgents, event: any): Promise { + // OODA Loop: Observe -> Orient -> Decide -> Act + + // 1. Observe (Update Blackboard with Environment Sense) + this.updatePerception(bot) + + // 2. Orient (Contextualize Event) + let contextMsg = '' + if (event.type === 'user_intent') { + contextMsg = `User ${event.source.id} says: "${event.payload.content}"` + } + else if (event.type === 'action:feedback') { + const { status, result, error, action } = event.payload + const actionDesc = action.type === 'physical' ? action.step.tool : 'chat' + contextMsg = `Action Feedback: ${actionDesc} ${status}. Result: ${JSON.stringify(result || error)}` + } + + // 3. Decide (LLM Call) + const systemPrompt = this.generateSystemPrompt(this.blackboard) + const decision = await this.decide(systemPrompt, contextMsg) + + if (!decision) { + this.deps.logger.warn('Brain: No decision made.') + return + } + + // 4. Act (Execute Decision) + this.deps.logger.log(`Brain: Thought: ${decision.thought}`) + + // Update Blackboard + this.blackboard.update({ + currentGoal: decision.blackboard.currentGoal || this.blackboard.goal, + currentThought: decision.blackboard.currentThought || this.blackboard.thought, + executionStrategy: decision.blackboard.executionStrategy || this.blackboard.strategy, + }) + + // Issue Actions + if (decision.actions && decision.actions.length > 0) { + this.deps.taskExecutor.executeActions(decision.actions) + } + } + + private updatePerception(bot: MineflayerWithAgents): void { + const pos = bot.bot.entity.position + this.blackboard.updateSelf({ + location: pos, + health: bot.bot.health, + food: bot.bot.food, + }) + + this.blackboard.updateEnvironment({ + time: bot.bot.time.isDay ? 'day' : 'night', + weather: bot.bot.isRaining ? 'rain' : 'clear', + nearbyPlayers: Object.keys(bot.bot.players).filter(p => p !== bot.bot.username), + }) + } + + private async decide(sysPrompt: string, userMsg: string): Promise { + try { + const response = await this.deps.neuri.handleStateless( + [ + system(sysPrompt), + user(userMsg), + ], + async (ctx) => { + const completion = await ctx.reroute('action', ctx.messages, { + response_format: { type: 'json_object' }, + // eslint-disable-next-line @typescript-eslint/no-explicit-any + } as any) as any + + if (!completion || !completion.choices?.[0]?.message?.content) { + throw new Error('LLM failed to return content') + } + return completion.choices[0].message.content + }, + ) + + if (!response) + return null + + const parsed = JSON.parse(response) as BrainResponse + return parsed + } + catch (err) { + this.deps.logger.withError(err).error('Brain: Decision failed') + return null + } + } + + private generateSystemPrompt(blackboard: Blackboard): string { + const actions = this.deps.taskExecutor.getAvailableActions() + const actionDefinitions = actions.map((a) => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const schema = zodToJsonSchema(a.schema as any) + return { + name: a.name, + description: a.description, + parameters: schema, + } + }) + + const availableActionsJson = JSON.stringify(actionDefinitions, null, 2) + + return `你是Minecraft自主Agent的大脑。 + +当前状态（黑板）: +目标: "${blackboard.goal}" +思绪: "${blackboard.thought}" +策略: "${blackboard.strategy}" +自身: 位置${blackboard.self.location} 生命${blackboard.self.health} 饱食${blackboard.self.food} +环境: ${blackboard.environment.time} ${blackboard.environment.weather} 玩家[${blackboard.environment.nearbyPlayers.join(',')}] + +可用动作: +${availableActionsJson} + +规则: +1. 可执行上述物理动作(physical)或聊天动作(chat) +2. 可并行执行不冲突的多个动作(如聊天+行走) +3. 必须输出JSON + +输出格式: +{ + "thought": "推理过程", + "blackboard": { + "currentGoal": "更新的目标", + "currentThought": "内心独白", + "executionStrategy": "短期计划" + }, + "actions": [ + {"type":"chat","message":"..."}, + {"type":"physical","step":{"tool":"动作名","params":{...}}} + ] +} +` + } +} diff --git a/services/minecraft/src/cognitive/conscious/orchestrator.ts b/services/minecraft/src/cognitive/conscious/orchestrator.ts index 6e308c0f9..dc5a2d95f 100644 --- a/services/minecraft/src/cognitive/conscious/orchestrator.ts +++ b/services/minecraft/src/cognitive/conscious/orchestrator.ts @@ -1,6 +1,7 @@ import type { Logg } from '@guiiai/logg' import type { Neuri, NeuriContext } from 'neuri' +import type { TaskExecutor } from '../action/task-executor' import type { EventManager } from '../perception/event-manager' import type { BotEvent, MineflayerWithAgents, UserIntentPayload } from '../types' import type { CancellationToken } from './task-state' @@ -9,20 +10,24 @@ import { withRetry } from '@moeru/std' import { system, user } from 'neuri/openai' import { DebugServer } from '../../debug-server' +import { ActionError } from '../../utils/errors' import { handleLLMCompletion } from './completion' import { generateStatusPrompt } from './prompt' import { TaskManager } from './task-manager' export class Orchestrator { private taskManager: TaskManager - private eventQueue: Array> = [] - private isProcessingQueue = false + + // We no longer need an event queue for blocking purposes, + // but we might keep it if we want to handle explicit queuing later. + // For now, removing the blocking logic. constructor( private readonly deps: { eventManager: EventManager neuri: Neuri logger: Logg + taskExecutor: TaskExecutor }, ) { this.taskManager = new TaskManager(deps.logger) @@ -30,6 +35,16 @@ export class Orchestrator { public init(bot: MineflayerWithAgents): void { this.deps.eventManager.on('user_intent', async (event) => { + // Don't await here to allow event loop to continue? + // Actually, if we await, the next event won't process until this one finishes + // ONLY IF eventManager awaits listeners. + // Assuming we want true parallelism, we should probably not await the full task execution, + // but we should await the initial decision making. + // However, handleUserIntent is async void, so awaiting it in event emitter is standard. + // To ensure non-blocking, handleUserIntent should return quickly. + + // Let's await it, but ensure handleUserIntent doesn't block on long operations + // before deciding if it's a new task. await this.handleUserIntent(bot, event) }) } @@ -45,41 +60,24 @@ export class Orchestrator { return } - // Check if there's a current task - if (this.taskManager.hasCurrentTask()) { - // Determine if we should cancel current task or queue this event - if (this.shouldCancelCurrentTask(event)) { - this.deps.logger - .withFields({ currentTaskId: this.taskManager.getCurrentTask()?.id, priority: event.priority }) - .log('Orchestrator: Cancelling current task for high-priority event') - this.taskManager.cancelCurrentTask('High-priority event received') - this.broadcastTaskStatus() - } - else { - // Queue the event for later processing - this.eventQueue.push(event) - this.deps.logger - .withFields({ queueSize: this.eventQueue.length, username, event: content }) - .log('Orchestrator: Event queued') - this.broadcastTaskStatus() + // High priority interruption check + if (this.shouldCancelPrimaryTask(event)) { + this.deps.logger + .withFields({ + currentPrimaryId: this.taskManager.getPrimaryTask()?.id, + priority: event.priority, + }) + .log('Orchestrator: Cancelling primary task for high-priority event') - // Notify user that we're busy - const busyMessage = 'I\'m busy right now, I\'ll get to that in a moment!' - if (source.reply) { - source.reply(busyMessage) - } - else { - bot.bot.chat(busyMessage) - } - return - } + this.taskManager.cancelPrimaryTask('High-priority event received') + // Note: We continue to process this event as a new task } - // Create new task + // Create new task (Secondary by default if primary exists, Primary if none exists) const task = this.taskManager.createTask(content) this.deps.logger .withFields({ username, content, taskId: task.id }) - .log('Orchestrator: Starting new task') + .log('Orchestrator: Starting new task processing') this.broadcastTaskStatus() try { @@ -87,126 +85,184 @@ export class Orchestrator { bot.memory.chatHistory.push(user(`${username}: ${content}`)) // 2. Execute task with cancellation support - await this.executeTaskWithCancellation(bot, event, task.cancellationToken) + // This will now handle conflicts internally + this.executeTaskWithCancellation(bot, event, task).catch((err) => { + this.deps.logger.withError(err).error('Orchestrator: Async task execution failed') + }) + + // We return immediately to allow event loop to process next event + // (Effectively making it fire-and-forget from the EventManager's perspective) } catch (error) { - this.deps.logger.withError(error).warn('Orchestrator: Failed to process intent') - const errorMessage = `Sorry, I encountered an error: ${error instanceof Error ? error.message : 'Unknown error'}` - if (source.reply) { - source.reply(errorMessage) - } - else { - bot.bot.chat(errorMessage) - } - } - finally { - this.taskManager.completeCurrentTask() - this.broadcastTaskStatus() - // Process next queued event - this.processNextQueuedEvent(bot) + this.deps.logger.withError(error).warn('Orchestrator: Failed to initiate task') } } private async executeTaskWithCancellation( bot: MineflayerWithAgents, event: BotEvent, - cancellationToken: CancellationToken, + task: { id: string, cancellationToken: CancellationToken }, // Use TaskContext type if imported ): Promise { const { payload, source } = event const { content } = payload + const { cancellationToken, id: taskId } = task - // Planning phase - if (cancellationToken.isCancelled) - return - this.taskManager.updateTaskStatus('planning') - this.broadcastTaskStatus() - this.deps.logger.log('Orchestrator: Starting planning phase') + try { + // Planning phase + if (cancellationToken.isCancelled) + return + this.taskManager.updateTaskStatus(taskId, 'planning') + this.broadcastTaskStatus() + this.deps.logger.log('Orchestrator: Starting planning phase') - const plan = await bot.planning.createPlan(content) - this.taskManager.setTaskPlan(plan) - this.deps.logger.withFields({ steps: plan.steps.length }).log('Orchestrator: Plan created') + const availableActions = this.deps.taskExecutor.getAvailableActions() + const plan = await bot.planning.createPlan(content, availableActions) + this.taskManager.setTaskPlan(taskId, plan) + this.deps.logger.withFields({ steps: plan.steps.length }).log('Orchestrator: Plan created') - // Execution phase - if (cancellationToken.isCancelled) - return - this.taskManager.updateTaskStatus('executing') - this.broadcastTaskStatus() - this.deps.logger.log('Orchestrator: Executing plan') + // CONFLICT RESOLUTION + if (plan.requiresAction) { + // If this task requires action, check if it conflicts with a primary task + const primaryTask = this.taskManager.getPrimaryTask() - await bot.planning.executePlan(plan, cancellationToken) - this.deps.logger.log('Orchestrator: Plan executed successfully') + // If there is a primary task and IT IS NOT THIS TASK + if (primaryTask && primaryTask.id !== taskId) { + this.deps.logger.log('Orchestrator: Conflict detected - Secondary task requires action while Primary is busy') - // Response generation phase - if (cancellationToken.isCancelled) - return - this.taskManager.updateTaskStatus('responding') - this.broadcastTaskStatus() - this.deps.logger.log('Orchestrator: Generating response') + // Conflict Policy: Reject secondary action tasks + const busyMessage = `I'm currently busy with "${primaryTask.goal}". Please ask me to "${content}" later or tell me to stop.` + if (source.reply) + source.reply(busyMessage) + else bot.bot.chat(busyMessage) - const statusPrompt = await generateStatusPrompt(bot) - const taskContext = this.taskManager.getTaskContextForLLM() + // Abort this secondary task + this.taskManager.completeTask(taskId) + this.broadcastTaskStatus() + return + } + } - const response = await this.deps.neuri.handleStateless( - [ - ...bot.memory.chatHistory, - system(statusPrompt), - system(`Task Context: ${taskContext}`), - ], - async (c: NeuriContext) => { - this.deps.logger.log('Orchestrator: thinking...') - return withRetry( - ctx => handleLLMCompletion(ctx, bot, this.deps.logger), - { - retry: 3, - retryDelay: 1000, - }, - )(c) - }, - ) + // Execution phase + if (cancellationToken.isCancelled) + return - // Reply - if (cancellationToken.isCancelled) - return - if (response) { - this.deps.logger.withFields({ response }).log('Orchestrator: Responded') - if (source.reply) { - source.reply(response) + if (plan.requiresAction) { + this.taskManager.updateTaskStatus(taskId, 'executing') + this.broadcastTaskStatus() + this.deps.logger.log('Orchestrator: Executing plan') + + // Retry loop implementation + let currentPlan = plan + let retryCount = 0 + const MAX_RETRIES = 3 + + while (retryCount < MAX_RETRIES) { + if (cancellationToken.isCancelled) + return + + try { + await this.deps.taskExecutor.executePlan(currentPlan, cancellationToken) + this.deps.logger.log('Orchestrator: Plan executed successfully') + break // Success + } + catch (error: any) { + if (cancellationToken.isCancelled) + return + + // Check if it's an actionable error + const isActionError = error instanceof ActionError + if (!isActionError) + throw error // Re-throw system errors + + retryCount++ + if (retryCount >= MAX_RETRIES) + throw error // Give up + + this.deps.logger.withError(error).warn(`Orchestrator: Plan execution failed (Attempt ${retryCount}/${MAX_RETRIES}). Adjusting plan...`) + + // Adjust plan + const availableActions = this.deps.taskExecutor.getAvailableActions() + currentPlan = await bot.planning.adjustPlan( + currentPlan, + error.message, + 'system', + availableActions, + ) + this.taskManager.setTaskPlan(taskId, currentPlan) + this.broadcastTaskStatus() + } + } } else { - bot.bot.chat(response) + this.deps.logger.log('Orchestrator: No physical actions required, skipping execution phase') } + + // Response generation phase + if (cancellationToken.isCancelled) + return + this.taskManager.updateTaskStatus(taskId, 'responding') + this.broadcastTaskStatus() + this.deps.logger.log('Orchestrator: Generating response') + + const statusPrompt = await generateStatusPrompt(bot) + const taskContext = this.taskManager.getTaskContextForLLM() + + const response = await this.deps.neuri.handleStateless( + [ + ...bot.memory.chatHistory, + system(statusPrompt), + system(`Task Context:\n${taskContext}`), // Provide full context of all tasks + ], + async (c: NeuriContext) => { + this.deps.logger.log('Orchestrator: thinking...') + return withRetry( + ctx => handleLLMCompletion(ctx, bot, this.deps.logger), + { + retry: 3, + retryDelay: 1000, + }, + )(c) + }, + ) + + // Reply + if (cancellationToken.isCancelled) + return + if (response) { + this.deps.logger.withFields({ response }).log('Orchestrator: Responded') + if (source.reply) { + source.reply(response) + } + else { + bot.bot.chat(response) + } + } + } + catch (error) { + this.deps.logger.withError(error).warn(`Orchestrator: Task ${taskId} processing failed`) + // Optional: notify user of failure + } + finally { + this.taskManager.completeTask(taskId) + this.broadcastTaskStatus() } } - private shouldCancelCurrentTask(event: BotEvent): boolean { + private shouldCancelPrimaryTask(event: BotEvent): boolean { + const primaryTask = this.taskManager.getPrimaryTask() + if (!primaryTask) + return false + // High priority events should cancel current task const HIGH_PRIORITY_THRESHOLD = 8 return (event.priority ?? 5) >= HIGH_PRIORITY_THRESHOLD } - private async processNextQueuedEvent(bot: MineflayerWithAgents): Promise { - // Prevent concurrent queue processing - if (this.isProcessingQueue || this.eventQueue.length === 0) { - return - } - - this.isProcessingQueue = true - const nextEvent = this.eventQueue.shift() - - if (nextEvent) { - this.deps.logger.log('Orchestrator: Processing next queued event') - await this.handleUserIntent(bot, nextEvent) - } - - this.isProcessingQueue = false - } - private broadcastTaskStatus(): void { const debugServer = DebugServer.getInstance() debugServer.broadcast('task-status', { - currentTask: this.taskManager.getCurrentTask(), - queueSize: this.eventQueue.length, - queue: this.eventQueue, + currentTask: this.taskManager.getPrimaryTask(), // For backward compatibility with UI + activeTasks: this.taskManager.getAllActiveTasks(), history: this.taskManager.getTaskHistory(), }) } diff --git a/services/minecraft/src/cognitive/conscious/task-manager.ts b/services/minecraft/src/cognitive/conscious/task-manager.ts index a69af1919..f1c27837e 100644 --- a/services/minecraft/src/cognitive/conscious/task-manager.ts +++ b/services/minecraft/src/cognitive/conscious/task-manager.ts @@ -6,14 +6,17 @@ import type { TaskContext, TaskStatus } from './task-state' import { createCancellationToken } from './task-state' export class TaskManager { - private currentTask: TaskContext | null = null + private primaryTask: TaskContext | null = null + private secondaryTasks: Map = new Map() private taskHistory: TaskContext[] = [] private readonly maxHistorySize = 10 constructor(private readonly logger: Logg) {} /** - * Create a new task with cancellation support + * Create a new task. + * If strictlySecondary is true, it will always be created as specific type (e.g. for forced background tasks). + * Otherwise, if no primary task exists, new task becomes primary. */ public createTask(goal: string): TaskContext { const task: TaskContext = { @@ -24,132 +27,166 @@ export class TaskManager { cancellationToken: createCancellationToken(), } - this.currentTask = task - this.logger.withFields({ taskId: task.id, goal }).log('TaskManager: Created new task') + if (!this.primaryTask) { + this.primaryTask = task + this.logger.withFields({ taskId: task.id, goal, type: 'primary' }).log('TaskManager: Created new primary task') + } else { + this.secondaryTasks.set(task.id, task) + this.logger.withFields({ taskId: task.id, goal, type: 'secondary' }).log('TaskManager: Created new secondary task') + } return task } /** - * Update the status of the current task + * Update the status of a specific task */ - public updateTaskStatus(status: TaskStatus, currentStep?: string): void { - if (!this.currentTask) { - this.logger.warn('TaskManager: No current task to update') + public updateTaskStatus(taskId: string, status: TaskStatus, currentStep?: string): void { + const task = this.getTaskById(taskId) + if (!task) { + this.logger.warn(`TaskManager: Task ${taskId} not found for update`) return } - this.currentTask.status = status + task.status = status if (currentStep) { - this.currentTask.currentStep = currentStep + task.currentStep = currentStep } this.logger.withFields({ - taskId: this.currentTask.id, + taskId: task.id, status, currentStep, }).log('TaskManager: Updated task status') } /** - * Set the plan for the current task + * Set the plan for a specific task */ - public setTaskPlan(plan: Plan): void { - if (!this.currentTask) { - this.logger.warn('TaskManager: No current task to set plan for') + public setTaskPlan(taskId: string, plan: Plan): void { + const task = this.getTaskById(taskId) + if (!task) { + this.logger.warn(`TaskManager: Task ${taskId} not found to set plan`) return } - this.currentTask.plan = plan - this.logger.withFields({ taskId: this.currentTask.id }).log('TaskManager: Set task plan') + task.plan = plan + this.logger.withFields({ taskId: task.id }).log('TaskManager: Set task plan') } /** - * Cancel the current task + * Cancel a specific task. If no taskId provided, cancels primary task. */ - public cancelCurrentTask(reason?: string): void { - if (!this.currentTask) { - this.logger.warn('TaskManager: No current task to cancel') + public cancelTask(taskId: string, reason?: string): void { + const task = this.getTaskById(taskId) + if (!task) { + this.logger.warn(`TaskManager: Task ${taskId} not found to cancel`) return } this.logger.withFields({ - taskId: this.currentTask.id, + taskId: task.id, reason, - }).log('TaskManager: Cancelling current task') + }).log('TaskManager: Cancelling task') - this.currentTask.status = 'cancelling' - this.currentTask.cancellationToken.cancel() + task.status = 'cancelling' + task.cancellationToken.cancel() - // Move to history - this.addToHistory(this.currentTask) - this.currentTask = null + // We don't remove it yet, we wait for completeTask to be called + this.addToHistory(task) + + // Cleanup reference immediately to allow new primary tasks if this was primary? + // No, we should wait for the orchestrator to call completeTask/cleanup. } /** - * Complete the current task + * Cancel currently active primary task */ - public completeCurrentTask(): void { - if (!this.currentTask) { - return + public cancelPrimaryTask(reason?: string): void { + if (this.primaryTask) { + this.cancelTask(this.primaryTask.id, reason) } - - this.logger.withFields({ taskId: this.currentTask.id }).log('TaskManager: Task completed') - - // Move to history - this.addToHistory(this.currentTask) - this.currentTask = null } /** - * Get the current task + * Complete a task and remove it from active list */ - public getCurrentTask(): TaskContext | null { - return this.currentTask + public completeTask(taskId: string): void { + const task = this.getTaskById(taskId) + if (!task) return + + this.logger.withFields({ taskId: task.id }).log('TaskManager: Task completed') + this.addToHistory(task) + + if (this.primaryTask?.id === taskId) { + this.primaryTask = null + } else { + this.secondaryTasks.delete(taskId) + } } /** - * Check if there is a current task + * Get the current primary task */ - public hasCurrentTask(): boolean { - return this.currentTask !== null + public getPrimaryTask(): TaskContext | null { + return this.primaryTask } /** - * Check if can accept a new task + * Check if there is a primary task running */ - public canAcceptNewTask(): boolean { - return this.currentTask === null + public hasPrimaryTask(): boolean { + return this.primaryTask !== null } /** - * Get formatted task context for LLM + * Get a task by ID + */ + public getTaskById(taskId: string): TaskContext | null { + if (this.primaryTask?.id === taskId) return this.primaryTask + return this.secondaryTasks.get(taskId) || null + } + + /** + * Get formatted task context for LLM. + * Includes Primary Task and summary of Secondary Tasks. */ public getTaskContextForLLM(): string { - if (!this.currentTask) { - return 'No active task' + const lines: string[] = [] + + // Primary Task + if (this.primaryTask) { + const { goal, status, startTime, currentStep, plan } = this.primaryTask + const elapsedSeconds = Math.floor((Date.now() - startTime) / 1000) + lines.push(`[PRIMARY TASK] (${status.toUpperCase()}): "${goal}"`) + lines.push(`- Duration: ${elapsedSeconds}s`) + if (currentStep) lines.push(`- Step: ${currentStep}`) + if (plan) lines.push(`- Plan: ${plan.steps.length} steps (${plan.status})`) + } else { + lines.push('No primary task active.') } - const { goal, status, startTime, currentStep, plan } = this.currentTask - const elapsedSeconds = Math.floor((Date.now() - startTime) / 1000) - - const lines = [ - `Current Task: [${status.toUpperCase()}] ${goal}`, - `- Started: ${elapsedSeconds} seconds ago`, - ] - - if (currentStep) { - lines.push(`- Current Step: ${currentStep}`) - } - - if (plan) { - const totalSteps = plan.steps.length - lines.push(`- Plan: ${totalSteps} steps total`) + // Secondary Tasks + if (this.secondaryTasks.size > 0) { + lines.push('\n[SECONDARY TASKS]') + for (const task of this.secondaryTasks.values()) { + lines.push(`- [${task.status.toUpperCase()}] "${task.goal}"`) + } } return lines.join('\n') } + /** + * Get all active tasks for debugging + */ + public getAllActiveTasks(): TaskContext[] { + const tasks: TaskContext[] = [] + if (this.primaryTask) tasks.push(this.primaryTask) + tasks.push(...this.secondaryTasks.values()) + return tasks + } + /** * Get task history */ @@ -162,9 +199,10 @@ export class TaskManager { } private addToHistory(task: TaskContext): void { + // Only add if not already in history (simple check) + if (this.taskHistory.some(t => t.id === task.id)) return + this.taskHistory.push(task) - - // Limit history size if (this.taskHistory.length > this.maxHistorySize) { this.taskHistory.shift() } diff --git a/services/minecraft/src/cognitive/container.ts b/services/minecraft/src/cognitive/container.ts index cac067eed..ab5600a21 100644 --- a/services/minecraft/src/cognitive/container.ts +++ b/services/minecraft/src/cognitive/container.ts @@ -7,7 +7,8 @@ import { asClass, asFunction, createContainer, InjectionMode } from 'awilix' import { ActionAgentImpl } from '../agents/action' import { ChatAgentImpl } from '../agents/chat' import { PlanningAgentImpl } from '../agents/planning' -import { Orchestrator } from './conscious/orchestrator' +import { TaskExecutor } from './action/task-executor' +import { Brain } from './conscious/brain' import { EventManager } from './perception/event-manager' import { ReflexManager } from './reflex/reflex-manager' @@ -18,7 +19,8 @@ export interface ContainerServices { chatAgent: ChatAgentImpl neuri: Neuri eventManager: EventManager - orchestrator: Orchestrator + taskExecutor: TaskExecutor + brain: Brain reflexManager: ReflexManager } @@ -73,7 +75,9 @@ export function createAgentContainer(options: { eventManager: asClass(EventManager).singleton(), - orchestrator: asClass(Orchestrator).singleton(), + taskExecutor: asClass(TaskExecutor).singleton(), + + brain: asClass(Brain).singleton(), reflexManager: asClass(ReflexManager).singleton(), }) diff --git a/services/minecraft/src/cognitive/index.ts b/services/minecraft/src/cognitive/index.ts index 849454c12..ecb1c685a 100644 --- a/services/minecraft/src/cognitive/index.ts +++ b/services/minecraft/src/cognitive/index.ts @@ -1,46 +1,41 @@ import type { MineflayerPlugin } from '../libs/mineflayer' import type { LLMAgentOptions, MineflayerWithAgents } from './types' -import { system } from 'neuri/openai' - import { config } from '../composables/config' import { ChatMessageHandler } from '../libs/mineflayer' -import { generateActionAgentPrompt } from './conscious/prompt' import { createAgentContainer } from './container' export function LLMAgent(options: LLMAgentOptions): MineflayerPlugin { + let container: ReturnType + return { async created(bot) { // Create container and get required services - const container = createAgentContainer({ + container = createAgentContainer({ neuri: options.agent, model: config.openai.model, }) const actionAgent = container.resolve('actionAgent') - const planningAgent = container.resolve('planningAgent') const chatAgent = container.resolve('chatAgent') const eventManager = container.resolve('eventManager') - const orchestrator = container.resolve('orchestrator') + const brain = container.resolve('brain') const reflexManager = container.resolve('reflexManager') + const taskExecutor = container.resolve('taskExecutor') // Initialize agents await actionAgent.init() - await planningAgent.init() await chatAgent.init() + await taskExecutor.initialize() // Type conversion const botWithAgents = bot as unknown as MineflayerWithAgents botWithAgents.action = actionAgent - botWithAgents.planning = planningAgent botWithAgents.chat = chatAgent // Initialize layers reflexManager.init(botWithAgents) - orchestrator.init(botWithAgents) - - // Initialize system prompt - bot.memory.chatHistory.push(system(generateActionAgentPrompt(bot))) + brain.init(botWithAgents) // Set message handling via EventManager const chatHandler = new ChatMessageHandler(bot.username) @@ -89,8 +84,13 @@ export function LLMAgent(options: LLMAgentOptions): MineflayerPlugin { async beforeCleanup(bot) { const botWithAgents = bot as unknown as MineflayerWithAgents await botWithAgents.action?.destroy() - await botWithAgents.planning?.destroy() await botWithAgents.chat?.destroy() + + if (container) { + const taskExecutor = container.resolve('taskExecutor') + await taskExecutor.destroy() + } + bot.bot.removeAllListeners('chat') }, } diff --git a/services/minecraft/src/libs/mineflayer/base-agent.ts b/services/minecraft/src/libs/mineflayer/base-agent.ts index b3023d44e..32bdd55c7 100644 --- a/services/minecraft/src/libs/mineflayer/base-agent.ts +++ b/services/minecraft/src/libs/mineflayer/base-agent.ts @@ -44,14 +44,14 @@ export interface Plan { export interface PlanningAgent extends BaseAgent { type: 'planning' - createPlan: (goal: string) => Promise - executePlan: (plan: Plan, cancellationToken?: any) => Promise - adjustPlan: (plan: Plan, feedback: string, sender: string) => Promise + createPlan: (goal: string, availableActions?: Action[]) => Promise + adjustPlan: (plan: Plan, feedback: string, sender: string, availableActions?: Action[]) => Promise } export interface ChatAgent extends BaseAgent { type: 'chat' processMessage: (message: string, sender: string) => Promise + sendMessage: (message: string) => Promise startConversation: (player: string) => void endConversation: (player: string) => void } diff --git a/services/minecraft/src/skills/crafting.ts b/services/minecraft/src/skills/crafting.ts index 3fa1d5f86..ba4877495 100644 --- a/services/minecraft/src/skills/crafting.ts +++ b/services/minecraft/src/skills/crafting.ts @@ -305,15 +305,12 @@ export async function clearNearestFurnace(mineflayer: Mineflayer): Promise + if (tasks.length > 0) { + taskInfo.innerHTML = tasks.map(task => { + // Determine if primary based on data or assumption + // Note: TaskManager now logs 'type' but task object structure might not have it explicitly unless we added it to TaskContext + // But we can infer order or just display all. + // Let's assume the first one might be primary if we used getAllActiveTasks() order (primary first) + + const elapsed = Math.floor((Date.now() - task.startTime) / 1000); + const statusColor = { + 'idle': '#888', + 'planning': '#ffb74d', + 'executing': '#4a9eff', + 'responding': '#9c27b0', + 'cancelling': '#ef5350' + }[task.status] || '#888'; + + // Visual distinction for secondary vs primary could be nice, but simple list is fine for now + return ` +
${task.goal} ${task.status.toUpperCase()}
-
Task ID: ${task.id}
-
Elapsed: ${elapsed}s
- ${task.currentStep ? `
Current Step: ${task.currentStep}
` : ''} - ${task.plan ? `
Plan: ${task.plan.steps.length} steps (${task.plan.status})
` : ''} +
+ ID: ${task.id.substr(0,12)}... + Time: ${elapsed}s +
+ ${task.currentStep ? `
Step: ${task.currentStep}
` : ''} + ${task.plan ? `
Plan: ${task.plan.steps.length} steps (${task.plan.status})
` : ''}
- `; + `; + }).join(''); } else { - taskInfo.innerHTML = '
No active task
'; + taskInfo.innerHTML = '
No active tasks
'; } // Update queue + // ... (rest same, removing queue updates here as they are separate chunks?) + // Actually I need to include the rest of the function or it will be cut off. + // Let's just replace the task rendering part if I target specific lines. + const queueCount = document.getElementById('queue-count'); const queueList = document.getElementById('queue-list'); queueCount.textContent = status.queueSize || 0;