nanakura/moeka-project
1
0
Fork 0
Code Issues Pull Requests 1 Actions 2 Packages Projects Releases Wiki Activity

refactor(minecraft): remove legacy action adapter and simplify execution model

Browse Source 
- Remove unused ActionLLMHandler and createActionNeuriAgent from adapter.ts
- Replace execution:'parallel'/'sequential' with readonly:true for query tools
- Add 'skip' action for explicit no-op turns
- Remove error suggestion logic (now handled by brain retry)
- Simplify TaskExecutor to use ActionAgent.performAction directly
- Update tests to use createNeuriAgent() without bot parameter
This commit is contained in:
Rin
2026-02-18 11:14:08 +08:00
committed by Neko Ayaka
parent 1c942a84c8
commit 54b3bfdc06
17 changed files with 417 additions and 1057 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/minecraft/src/agents/action/adapter.test.ts
-39
View File
@@ -1,39 +0,0 @@
import { messages, system, user } from 'neuri/openai'
import { beforeAll, describe, expect, it } from 'vitest'
import { initBot, useBot } from '../../composables/bot'
import { config, initEnv } from '../../composables/config'
import { createNeuriAgent } from '../../composables/neuri'
import { initLogger } from '../../utils/logger'
describe.skip('openAI agent', { timeout: 0 }, () => {
beforeAll(() => {
initLogger()
initEnv()
initBot({ botConfig: config.bot })
})
it('should initialize the agent', async () => {
const { bot } = useBot()
const agent = await createNeuriAgent(bot)
await new Promise<void>((resolve) => {
bot.bot.once('spawn', async () => {
const text = await agent.handle(
messages(
system('You are AIRI.'),
user('Hello, who are you?'),
),
async (c) => {
const completion = await c.reroute('query', c.messages, { model: config.openai.model })
return await completion?.firstContent()
},
)
expect(text?.toLowerCase()).toContain('airi')
resolve()
})
})
})
})
services/minecraft/src/agents/action/adapter.ts
-106
View File
@@ -1,106 +0,0 @@
import type { Agent } from 'neuri'
import type { Message } from 'neuri/openai'
import type { Mineflayer } from '../../libs/mineflayer'
import type { PlanStep } from '../planning/adapter'
import { agent } from 'neuri'
import { system, user } from 'neuri/openai'
import { BaseLLMHandler } from '../../cognitive/conscious/handler'
import { ActionError } from '../../utils/errors'
import { useLogger } from '../../utils/logger'
import { generateActionSystemPrompt } from './system-prompt'
import { actionsList } from './tools'
/**
* Generate actionable suggestions for common error codes.
* These help the LLM understand what it can do to recover from failures.
*/
function getSuggestionForError(code: string): string {
switch (code) {
case 'RESOURCE_MISSING':
return 'Suggestion: Check your inventory first, then gather the missing resources or ask the player for help if you cannot find them.'
case 'TARGET_NOT_FOUND':
return 'Suggestion: The target could not be found. Ask the player for clarification or use exploration tools to locate it.'
case 'NO_PATH':
return 'Suggestion: Unable to reach the destination. Try finding an alternative route or ask the player for guidance.'
case 'TIMEOUT':
return 'Suggestion: The action timed out. Consider breaking it into smaller steps or trying again later.'
default:
return 'Suggestion: Review the error details and try a different approach, or ask the player for help.'
}
}
export async function createActionNeuriAgent(mineflayer: Mineflayer): Promise<Agent> {
const logger = useLogger()
logger.log('Initializing action agent')
let actionAgent = agent('action')
Object.values(actionsList).forEach((action) => {
actionAgent = actionAgent.tool(
action.name,
action.schema,
async ({ parameters }) => {
logger.withFields({ name: action.name, parameters }).log('Calling action')
mineflayer.memory.actions.push(action)
const fn = action.perform(mineflayer)
try {
return await fn(...Object.values(parameters))
}
catch (error) {
// Return ActionError as a result string instead of throwing
// This allows the LLM to learn from tool failures during its reasoning phase
if (error instanceof ActionError) {
logger.withError(error).warn('Action failed during tool call')
const contextStr = error.context ? `\nContext: ${JSON.stringify(error.context)}` : ''
const suggestion = getSuggestionForError(error.code)
return `[FAILED] ${error.code}: ${error.message}${contextStr}\n${suggestion}`
}
// Re-throw non-ActionError errors (unexpected failures)
throw error
}
},
{ description: action.description },
)
})
return actionAgent.build()
}
export class ActionLLMHandler extends BaseLLMHandler {
public async executeStep(step: PlanStep): Promise<string> {
const systemPrompt = generateActionSystemPrompt()
const userPrompt = this.generateActionUserPrompt(step)
const messages = [system(systemPrompt), user(userPrompt)]
const result = await this.handleAction(messages)
return result
}
private generateActionUserPrompt(step: PlanStep): string {
return `Execute this step: ${step.description}
Suggested tool: ${step.tool}
Params: ${JSON.stringify(step.params)}
Please use the appropriate tool with the correct parameters to accomplish this step.
If the suggested tool is not appropriate, you may choose a different one.`
}
public async handleAction(messages: Message[]): Promise<string> {
const result = await this.config.agent.handleStateless(messages, async (context) => {
this.logger.log('Processing action...')
const retryHandler = this.createRetryHandler(
async ctx => (await this.handleCompletion(ctx, 'action', ctx.messages)).content,
)
return await retryHandler(context)
})
if (!result) {
throw new Error('Failed to process action')
}
return result
}
}
services/minecraft/src/agents/action/tools.test.ts
+2 -2
View File
@@ -16,7 +16,7 @@ describe.skip('actions agent', { timeout: 0 }, () => {
it('should choose right query command', async () => {
const { bot } = useBot()
const agent = await createNeuriAgent(bot)
const agent = await createNeuriAgent()
await new Promise<void>((resolve) => {
bot.bot.once('spawn', async () => {
@@ -37,7 +37,7 @@ describe.skip('actions agent', { timeout: 0 }, () => {
it('should choose right action command', async () => {
const { bot } = useBot()
const agent = await createNeuriAgent(bot)
const agent = await createNeuriAgent()
await new Promise<void>((resolve) => {
bot.bot.on('spawn', async () => {
services/minecraft/src/agents/action/tools.ts
+13 -30
View File
@@ -26,10 +26,17 @@ function formatWearingItem(slot: string, item: string | undefined): string {
}
export const actionsList: Action[] = [
{
name: 'skip',
description: 'Do nothing this turn. Use when observing, waiting, or when no action is needed.',
readonly: true,
schema: z.object({}),
perform: () => (): string => 'Skipped turn',
},
{
name: 'chat',
description: 'Send a chat message to players in the game. Use this to communicate, respond to questions, or announce what you are doing.',
execution: 'parallel',
readonly: true,
schema: z.object({
message: z.string().describe('The message to send in chat.'),
}),
@@ -40,7 +47,6 @@ export const actionsList: Action[] = [
},
// {\n // name: 'setReflexMode',
// description: 'Set (or clear) your reflex mode override. Use work/wander to disable idle-only reflex behaviors. Set override to null to return to automatic mode selection.',
// execution: 'sequential',
// schema: z.object({
// mode: z.enum(['idle', 'social', 'alert', 'work', 'wander']).nullable().describe('Mode override to set'),
// }),
@@ -58,7 +64,7 @@ export const actionsList: Action[] = [
{
name: 'inventory',
description: 'Get your inventory.',
execution: 'parallel',
readonly: true,
schema: z.object({}),
perform: mineflayer => (): string => {
const inventory = world.getInventoryCounts(mineflayer)
@@ -81,7 +87,7 @@ export const actionsList: Action[] = [
{
name: 'nearbyBlocks',
description: 'Get the blocks near you.',
execution: 'parallel',
readonly: true,
schema: z.object({}),
perform: mineflayer => (): string => {
const blocks = world.getNearbyBlockTypes(mineflayer)
@@ -92,7 +98,7 @@ export const actionsList: Action[] = [
{
name: 'craftable',
description: 'Get the craftable items with your inventory.',
execution: 'parallel',
readonly: true,
schema: z.object({}),
perform: mineflayer => (): string => {
const craftable = world.getCraftableItems(mineflayer)
@@ -102,7 +108,7 @@ export const actionsList: Action[] = [
{
name: 'entities',
description: 'Get the nearby players and entities.',
execution: 'parallel',
readonly: true,
schema: z.object({}),
perform: mineflayer => (): string => {
const players = world.getNearbyPlayerNames(mineflayer)
@@ -120,7 +126,6 @@ export const actionsList: Action[] = [
{
name: 'stop',
description: 'Force stop all actions', // TODO: include name of the current action in description?
execution: 'sequential',
schema: z.object({}),
perform: mineflayer => async () => {
mineflayer.interrupt('stop tool called')
@@ -131,7 +136,6 @@ export const actionsList: Action[] = [
{
name: 'goToPlayer',
description: 'Go to the given player.',
execution: 'sequential',
schema: z.object({
player_name: z.string().describe('The name of the player to go to.'),
closeness: z.number().describe('How close to get to the player in blocks.').min(0),
@@ -145,7 +149,6 @@ export const actionsList: Action[] = [
{
name: 'followPlayer',
description: 'Endlessly follow the given player.',
execution: 'sequential',
schema: z.object({
player_name: z.string().describe('name of the player to follow.'),
follow_dist: z.number().describe('The distance to follow from.').min(0),
@@ -158,7 +161,6 @@ export const actionsList: Action[] = [
{
name: 'goToCoordinate',
description: 'Go to the given x, y, z location.',
execution: 'sequential',
schema: z.object({
x: z.number().describe('The x coordinate.'),
y: z.number().describe('The y coordinate.').min(-64).max(320),
@@ -173,7 +175,6 @@ export const actionsList: Action[] = [
{
name: 'searchForBlock',
description: 'Find and go to the nearest block of a given type in a given range.',
execution: 'sequential',
schema: z.object({
type: z.string().describe('The block type to go to.'),
search_range: z.number().describe('The range to search for the block.').min(32).max(512),
@@ -186,7 +187,6 @@ export const actionsList: Action[] = [
{
name: 'searchForEntity',
description: 'Find and go to the nearest entity of a given type in a given range.',
execution: 'sequential',
schema: z.object({
type: z.string().describe('The type of entity to go to.'),
search_range: z.number().describe('The range to search for the entity.').min(32).max(512),
@@ -210,7 +210,6 @@ export const actionsList: Action[] = [
{
name: 'givePlayer',
description: 'Give the specified item to the given player.',
execution: 'sequential',
schema: z.object({
player_name: z.string().describe('The name of the player to give the item to.'),
item_name: z.string().describe('The name of the item to give.'),
@@ -224,7 +223,6 @@ export const actionsList: Action[] = [
{
name: 'consume',
description: 'Eat/drink the given item.',
execution: 'sequential',
schema: z.object({
item_name: z.string().describe('The name of the item to consume.'),
}),
@@ -236,7 +234,6 @@ export const actionsList: Action[] = [
{
name: 'equip',
description: 'Equip the given item.',
execution: 'sequential',
schema: z.object({
item_name: z.string().describe('The name of the item to equip.'),
}),
@@ -248,7 +245,6 @@ export const actionsList: Action[] = [
{
name: 'putInChest',
description: 'Put the given item in the nearest chest.',
execution: 'sequential',
schema: z.object({
item_name: z.string().describe('The name of the item to put in the chest.'),
num: z.number().int().describe('The number of items to put in the chest.').min(1),
@@ -261,7 +257,6 @@ export const actionsList: Action[] = [
{
name: 'takeFromChest',
description: 'Take the given items from the nearest chest.',
execution: 'sequential',
schema: z.object({
item_name: z.string().describe('The name of the item to take.'),
num: z.number().int().describe('The number of items to take.').min(1),
@@ -283,7 +278,6 @@ export const actionsList: Action[] = [
{
name: 'discard',
description: 'Discard the given item from the inventory.',
execution: 'sequential',
schema: z.object({
item_name: z.string().describe('The name of the item to discard.'),
num: z.number().int().describe('The number of items to discard.').min(1),
@@ -296,7 +290,6 @@ export const actionsList: Action[] = [
{
name: 'collectBlocks',
description: 'Automatically collect the nearest blocks of a given type.',
execution: 'sequential',
schema: z.object({
type: z.string().describe('The block type to collect.'),
num: z.number().int().describe('The number of blocks to collect.').min(1),
@@ -312,7 +305,6 @@ export const actionsList: Action[] = [
{
name: 'mineBlockAt',
description: 'Mine (break) a block at a specific position. Do NOT use this for regular resource collection. Use collectBlocks instead.',
execution: 'sequential',
schema: z.object({
x: z.number().describe('The x coordinate.'),
y: z.number().describe('The y coordinate.'),
@@ -343,7 +335,6 @@ export const actionsList: Action[] = [
{
name: 'craftRecipe',
description: 'Craft the given recipe a given number of times.',
execution: 'sequential',
schema: z.object({
recipe_name: z.string().describe('The name of the output item to craft.'),
num: z.number().int().describe('The number of times to craft the recipe. This is NOT the number of output items, as it may craft many more items depending on the recipe.').min(1),
@@ -356,7 +347,6 @@ export const actionsList: Action[] = [
{
name: 'smeltItem',
description: 'Smelt the given item the given number of times.',
execution: 'sequential',
schema: z.object({
item_name: z.string().describe('The name of the input item to smelt.'),
num: z.number().int().describe('The number of times to smelt the item.').min(1),
@@ -369,7 +359,6 @@ export const actionsList: Action[] = [
{
name: 'clearFurnace',
description: 'Take all items out of the nearest furnace.',
execution: 'sequential',
schema: z.object({}),
perform: mineflayer => async () => {
await skills.clearNearestFurnace(mineflayer)
@@ -379,7 +368,6 @@ export const actionsList: Action[] = [
{
name: 'placeHere',
description: 'Place a given block in the current location. Do NOT use to build structures, only use for single blocks/torches.',
execution: 'sequential',
schema: z.object({
type: z.string().describe('The block type to place.'),
}),
@@ -392,7 +380,6 @@ export const actionsList: Action[] = [
{
name: 'attack',
description: 'Attack and kill the nearest entity of a given type.',
execution: 'sequential',
schema: z.object({
type: z.string().describe('The type of entity to attack.'),
}),
@@ -404,7 +391,6 @@ export const actionsList: Action[] = [
{
name: 'attackPlayer',
description: 'Attack a specific player until they die or run away. Remember this is just a game and does not cause real life harm.',
execution: 'sequential',
schema: z.object({
player_name: z.string().describe('The name of the player to attack.'),
}),
@@ -420,7 +406,6 @@ export const actionsList: Action[] = [
{
name: 'goToBed',
description: 'Go to the nearest bed and sleep.',
execution: 'sequential',
schema: z.object({}),
perform: mineflayer => async () => {
await skills.goToBed(mineflayer)
@@ -430,7 +415,6 @@ export const actionsList: Action[] = [
{
name: 'activate',
description: 'Activate the nearest object of a given type.',
execution: 'sequential',
schema: z.object({
type: z.string().describe('The type of object to activate.'),
}),
@@ -442,7 +426,7 @@ export const actionsList: Action[] = [
{
name: 'recipePlan',
description: 'Plan how to craft an item. Shows the full recipe tree, what resources you have, what you\'re missing, and whether you can craft it now. Use this BEFORE attempting to craft complex items to understand what you need.',
execution: 'parallel',
readonly: true,
schema: z.object({
item_name: z.string().describe('The name of the item you want to craft (e.g., "diamond_pickaxe", "oak_planks").'),
amount: z.number().int().min(1).default(1).describe('How many of the item you want to craft.'),
@@ -454,7 +438,6 @@ export const actionsList: Action[] = [
{
name: 'autoCraft',
description: 'Automatically craft an item if you have all the required resources. This will check the recipe, verify you have materials, and craft it. Use recipePlan first to see if crafting is possible.',
execution: 'sequential',
schema: z.object({
item_name: z.string().describe('The name of the item to craft.'),
amount: z.number().int().min(1).default(1).describe('How many of the item to craft.'),
services/minecraft/src/cognitive/action/task-executor.ts
+52 -110
View File
@@ -31,16 +31,11 @@ export class TaskExecutor extends EventEmitter {
return
this.logger.log('Initializing Task Executor')
// ActionAgent is initialized by container/orchestrator
this.initialized = true
}
public async destroy(): Promise<void> {
this.initialized = false
// ActionAgentImpl doesn't expose destroy publicly in interface but defines it?
// Checking AbstractAgent, yes it has destroy().
// We cast to access it or trust it's handled.
// For now, assume we don't need explicit destroy of ActionAgent if it just clears listeners.
}
public async executePlan(plan: Plan, cancellationToken?: CancellationToken): Promise<void> {
@@ -60,31 +55,18 @@ export class TaskExecutor extends EventEmitter {
// Execute each step
for (const step of plan.steps) {
// Check for cancellation before each step
if (cancellationToken?.isCancelled) {
this.logger.log('Plan execution cancelled')
plan.status = 'cancelled'
return
}
try {
this.logger.withField('step', step).log('Executing step')
await this.actionAgent.performAction(step)
const action: ActionInstruction = {
tool: step.tool,
params: step.params,
}
catch (stepError: any) {
if (stepError instanceof ActionError) {
this.logger.withError(stepError).warn('Step execution failed with ActionError')
// Fail fast on hard errors
if (stepError.code === 'RESOURCE_MISSING' || stepError.code === 'CRAFTING_FAILED' || stepError.code === 'INVENTORY_FULL') {
throw stepError
}
}
this.logger.withError(stepError).error('Failed to execute step')
// Re-throw to let Orchestrator handle retry logic
throw stepError
}
await this.runSingleAction(action)
}
plan.status = 'completed'
@@ -95,103 +77,63 @@ export class TaskExecutor extends EventEmitter {
}
}
public executeActions(actions: ActionInstruction[], cancellationToken?: CancellationToken): void {
public async executeAction(action: ActionInstruction, cancellationToken?: CancellationToken): Promise<void> {
if (!this.initialized) {
throw new Error('TaskExecutor not initialized')
}
this.logger.withField('count', actions.length).log('Executing actions')
if (cancellationToken?.isCancelled) {
this.logger.log('Action execution cancelled before start')
return
}
const runSingleAction = async (action: ActionInstruction): Promise<void> => {
if (cancellationToken?.isCancelled) {
this.logger.log('Action execution cancelled before start')
try {
await this.runSingleAction(action)
}
catch (error) {
// Errors handled in runSingleAction event emission, but we rethrow to caller (Brain)
// actually runSingleAction rethrows?
// Let's rely on runSingleAction behavior
}
}
private async runSingleAction(action: ActionInstruction): Promise<void> {
this.emit('action:started', { action })
try {
let result: string | void
if (action.tool === 'chat') {
const message = action.params.message
if (typeof message !== 'string' || message.trim().length === 0)
throw new Error('Invalid chat tool params: expected params.message to be a string')
await this.chatAgent.sendMessage(message)
result = 'Message sent'
}
else if (action.tool === 'skip') {
result = 'Skipped turn'
}
else {
// Dispatch to Action Agent (mineflayer)
// ActionAgent.performAction takes PlanStep (tool, params, description)
// ActionInstruction matches structure (tool, params)
result = await this.actionAgent.performAction(action as any)
}
this.emit('action:completed', { action, result })
}
catch (error) {
this.logger.withError(error).error('Action execution failed')
// Interrupts are special - no feedback needed? keeping logic
if (error instanceof ActionError && error.code === 'INTERRUPTED') {
return
}
this.emit('action:started', { action })
try {
let result: string | void
if (action.type === 'sequential' || action.type === 'parallel') {
if (action.step.tool === 'chat') {
const message = (action.step.params as any)?.message
if (typeof message !== 'string' || message.trim().length === 0)
throw new Error('Invalid chat tool params: expected params.message to be a non-empty string')
if (action.type === 'parallel') {
throw new ActionError('SYNC_ONLY', 'Tool \'chat\' is not allowed for parallel actions', {
tool: action.step.tool,
requestedExecution: action.type,
allowedExecution: 'sequential',
})
}
await this.chatAgent.sendMessage(message)
result = 'Message sent'
}
else {
if (action.type === 'parallel') {
const available = this.actionAgent.getAvailableActions()
const def = available.find(a => a.name === action.step.tool)
if (!def || def.execution !== 'parallel') {
throw new ActionError('UNKNOWN', `Tool '${action.step.tool}' is not allowed for parallel actions`, {
tool: action.step.tool,
requestedExecution: action.type,
allowedExecution: def?.execution,
})
}
}
result = await this.actionAgent.performAction(action.step)
}
}
else if (action.type === 'chat') {
await this.chatAgent.sendMessage(action.message)
result = 'Message sent'
}
else {
throw new Error(`Unknown action type: ${(action as any).type}`)
}
this.emit('action:completed', { action, result })
}
catch (error) {
this.logger.withError(error).error('Action execution failed')
if (error instanceof ActionError && error.code === 'INTERRUPTED') {
// Foreseeable interruption (e.g. stop tool). Don't send feedback to LLM.
return
}
// failed actions always emit feedback
this.emit('action:failed', { action, error })
// Fail fast for all actions: cancel the whole chain (including any remaining queued actions)
cancellationToken?.cancel?.()
throw error
}
this.emit('action:failed', { action, error })
throw error
}
void (async () => {
for (const action of actions) {
if (cancellationToken?.isCancelled) {
this.logger.log('Action execution cancelled before start')
return
}
if (action.type === 'parallel') {
void runSingleAction(action).catch(() => {
// errors are emitted via events; nothing else to do here
})
continue
}
try {
await runSingleAction(action)
}
catch (error) {
return
}
}
})()
}
public getAvailableActions() {
services/minecraft/src/cognitive/action/types/index.ts
+15 -21
View File
@@ -1,27 +1,21 @@
import type { PlanStep } from '../../../agents/planning/adapter'
export type ActionType = 'sequential' | 'parallel' | 'chat'
export interface BaseActionInstruction {
type: ActionType
id?: string
description?: string
require_feedback?: boolean
/**
* Unified action instruction format.
* All actions are tool invocations with a tool name and parameters.
*/
export interface ActionInstruction {
tool: string
params: Record<string, unknown>
}
export interface SequentialActionInstruction extends BaseActionInstruction {
type: 'sequential'
step: PlanStep
/**
* LLM response format for the stateful agent.
* Single action per turn, model uses native reasoning (no thought field).
*/
export interface LLMResponse {
action: ActionInstruction
}
export interface ParallelActionInstruction extends BaseActionInstruction {
type: 'parallel'
step: PlanStep
}
export interface ChatActionInstruction extends BaseActionInstruction {
type: 'chat'
message: string
}
export type ActionInstruction = SequentialActionInstruction | ParallelActionInstruction | ChatActionInstruction
// Re-export for backwards compatibility during migration
export type { PlanStep }
services/minecraft/src/cognitive/conscious/blackboard.ts
-98
View File
@@ -1,98 +0,0 @@
export interface contextViewState {
selfSummary: string
environmentSummary: string
}
export interface ChatMessage {
sender: string
content: string
timestamp: number
}
export interface ActionHistoryLine {
line: string
timestamp: number
}
export interface BlackboardState {
ultimateGoal: string
currentTask: string
strategy: string
contextView: contextViewState
chatHistory: ChatMessage[]
recentActionHistory: ActionHistoryLine[]
pendingActions: string[]
selfUsername: string
}
export class Blackboard {
private _state: BlackboardState
private static readonly MAX_CHAT_HISTORY = 8
private static readonly MAX_ACTION_HISTORY = 12
private static readonly MAX_PENDING_ACTIONS = 12
constructor() {
this._state = {
ultimateGoal: 'nothing',
currentTask: 'I am waiting for something to happen.',
strategy: 'idle',
contextView: {
selfSummary: 'Unknown',
environmentSummary: 'Unknown',
},
chatHistory: [],
recentActionHistory: [],
pendingActions: [],
selfUsername: 'Bot',
}
}
// Getters
public get ultimate_goal(): string { return this._state.ultimateGoal }
public get current_task(): string { return this._state.currentTask }
public get strategy(): string { return this._state.strategy }
public get selfSummary(): string { return this._state.contextView.selfSummary }
public get environmentSummary(): string { return this._state.contextView.environmentSummary }
public get chatHistory(): ChatMessage[] { return this._state.chatHistory }
public get recentActionHistory(): ActionHistoryLine[] { return this._state.recentActionHistory }
public get pendingActions(): string[] { return this._state.pendingActions }
public get selfUsername(): string { return this._state.selfUsername }
// Setters (Partial updates allowed)
public update(updates: Partial<BlackboardState>): void {
this._state = { ...this._state, ...updates }
}
public updateContextView(updates: Partial<contextViewState>): void {
this._state.contextView = { ...this._state.contextView, ...updates }
}
public addChatMessage(message: ChatMessage): void {
const newHistory = [...this._state.chatHistory, message]
if (newHistory.length > Blackboard.MAX_CHAT_HISTORY) {
newHistory.shift() // Remove oldest
}
this._state = { ...this._state, chatHistory: newHistory }
}
public addActionHistoryLine(line: string, timestamp: number = Date.now()): void {
const next = [...this._state.recentActionHistory, { line, timestamp }]
const trimmed = next.length > Blackboard.MAX_ACTION_HISTORY ? next.slice(-Blackboard.MAX_ACTION_HISTORY) : next
this._state = { ...this._state, recentActionHistory: trimmed }
}
public setPendingActions(lines: string[]): void {
const trimmed = lines.length > Blackboard.MAX_PENDING_ACTIONS ? lines.slice(0, Blackboard.MAX_PENDING_ACTIONS) : lines
this._state = { ...this._state, pendingActions: trimmed }
}
public getSnapshot(): BlackboardState {
return {
...this._state,
contextView: { ...this._state.contextView },
chatHistory: [...this._state.chatHistory],
recentActionHistory: this._state.recentActionHistory.map(l => ({ ...l })),
pendingActions: [...this._state.pendingActions],
}
}
}
services/minecraft/src/cognitive/conscious/brain.ts
+211 -480
View File
@@ -1,102 +1,45 @@
import type { Logg } from '@guiiai/logg'
import type { Neuri } from 'neuri'
// import type { Message } from 'neuri/openai' // Use if needed
import type { TaskExecutor } from '../action/task-executor'
import type { ActionInstruction } from '../action/types'
import type { EventBus, TracedEvent } from '../os'
import type { PerceptionSignal } from '../perception/types/signals'
import type { ReflexManager } from '../reflex/reflex-manager'
import type { BotEvent, MineflayerWithAgents } from '../types'
import { createCancellationToken, type CancellationToken } from './task-state'
import { system, user } from 'neuri/openai'
import { config } from '../../composables/config'
import { DebugService } from '../../debug'
import { Blackboard } from './blackboard'
import { buildConsciousContextView } from './context-view'
import { generateBrainSystemPrompt } from './prompts/brain-prompt'
import type { ActionInstruction } from '../action/types'
// Utils
function toErrorMessage(err: unknown): string {
if (err instanceof Error)
return err.message
if (typeof err === 'string')
return err
try {
return JSON.stringify(err)
}
catch {
return String(err)
}
}
function getJsonErrorPosition(err: unknown): number | null {
const msg = toErrorMessage(err)
const match = msg.match(/position\s+(\d+)/i)
if (!match)
return null
const pos = Number.parseInt(match[1], 10)
return Number.isFinite(pos) ? pos : null
if (err instanceof Error) return err.message
if (typeof err === 'string') return err
try { return JSON.stringify(err) } catch { return String(err) }
}
function extractJsonCandidate(input: string): string {
const trimmed = input.trim()
const fenced = trimmed.match(/^```(?:json)?\s*([\s\S]*?)\s*```$/i)
if (fenced?.[1])
return fenced[1].trim()
if (fenced?.[1]) return fenced[1].trim()
const start = trimmed.indexOf('{')
const end = trimmed.lastIndexOf('}')
if (start >= 0 && end > start)
return trimmed.slice(start, end + 1)
if (start >= 0 && end > start) return trimmed.slice(start, end + 1)
return trimmed
}
function validateLLMResponse(parsed: unknown): parsed is { thought: string, actions: unknown[] } {
if (typeof parsed !== 'object' || parsed === null)
return false
const obj = parsed as Record<string, unknown>
// Must have 'thought' (string) and 'actions' (array) at minimum
return typeof obj.thought === 'string' && Array.isArray(obj.actions)
}
function parseLLMResponseJson<T>(response: string): T {
const candidate = extractJsonCandidate(response)
let parsed: unknown
try {
parsed = JSON.parse(candidate)
}
catch (err) {
const pos = getJsonErrorPosition(err)
const window = 120
const snippet = (typeof pos === 'number')
? candidate.slice(Math.max(0, pos - window), Math.min(candidate.length, pos + window))
: candidate.slice(0, Math.min(candidate.length, 240))
throw new Error(`Failed to parse LLM JSON response: ${toErrorMessage(err)}; snippet=${JSON.stringify(snippet)}`)
}
// Validate structure - LLM sometimes hallucinates wrong format (e.g., tool call format)
if (!validateLLMResponse(parsed)) {
const snippet = JSON.stringify(parsed).slice(0, 200)
throw new Error(`LLM returned malformed response (missing thought/actions): ${snippet}`)
}
return parsed as T
}
function getErrorStatus(err: unknown): number | undefined {
const anyErr = err as any
const status = anyErr?.status ?? anyErr?.response?.status ?? anyErr?.cause?.status
return typeof status === 'number' ? status : undefined
}
function getErrorCode(err: unknown): string | undefined {
const anyErr = err as any
const code = anyErr?.code ?? anyErr?.cause?.code
return typeof code === 'string' ? code : undefined
}
function isLikelyAuthOrBadArgError(err: unknown): boolean {
const msg = toErrorMessage(err).toLowerCase()
const status = getErrorStatus(err)
@@ -116,6 +59,10 @@ function isLikelyAuthOrBadArgError(err: unknown): boolean {
}
interface BrainResponse {
action: ActionInstruction & { id?: string }
}
interface BrainDeps {
eventBus: EventBus
neuri: Neuri
@@ -124,16 +71,6 @@ interface BrainDeps {
reflexManager: ReflexManager
}
interface LLMResponse {
thought: string
blackboard: {
UltimateGoal?: string
CurrentTask?: string
executionStrategy?: string
}
actions: ActionInstruction[]
}
interface QueuedEvent {
event: BotEvent
resolve: () => void
@@ -141,218 +78,100 @@ interface QueuedEvent {
}
export class Brain {
private blackboard: Blackboard
private debugService: DebugService
private bot: MineflayerWithAgents | undefined
private nextActionId = 1
private inFlightActions = new Map<string, ActionInstruction>()
private feedbackDebounceMs = Number.parseInt(process.env.BRAIN_FEEDBACK_DEBOUNCE_MS ?? '200')
private feedbackDebounceTimer: NodeJS.Timeout | undefined
private feedbackBarrierTimeoutMs = Number.parseInt(process.env.BRAIN_FEEDBACK_BARRIER_TIMEOUT_MS ?? '1000')
private waitingForFeedbackIds = new Set<string>()
private feedbackBarrierTimer: NodeJS.Timeout | undefined
// Event Queue
// State
private queue: QueuedEvent[] = []
private isProcessing = false
private currentCancellationToken: CancellationToken | undefined
private lastContextView: string | undefined
constructor(private readonly deps: BrainDeps) {
this.blackboard = new Blackboard()
this.debugService = DebugService.getInstance()
}
private async handlePerceptionSignal(bot: MineflayerWithAgents, signal: PerceptionSignal): Promise<void> {
this.log('INFO', `Brain: Received perception: ${signal.description}`)
if (signal.type === 'chat_message') {
const parts = signal.description.split(': ')
const sender = parts.length > 1 ? parts[0] : 'Unknown'
const content = parts.length > 1 ? parts.slice(1).join(': ') : signal.description
this.blackboard.addChatMessage({
sender,
content,
timestamp: Date.now(),
})
}
await this.enqueueEvent(bot, {
type: 'perception',
payload: signal,
source: {
type: 'minecraft',
id: signal.sourceId ?? 'perception',
},
timestamp: Date.now(),
})
}
public init(bot: MineflayerWithAgents): void {
this.log('INFO', 'Brain: Initializing...')
this.bot = bot
this.blackboard.update({ selfUsername: bot.username })
this.deps.logger.log('INFO', 'Brain: Initializing stateful core...')
const handleSignal = async (signal: PerceptionSignal) => {
try {
await this.handlePerceptionSignal(bot, signal)
}
catch (err) {
this.log('ERROR', 'Brain: Failed to enqueue perception signal', { error: err })
}
}
// Conscious Signal Handler - signals must pass through Reflex first
// EventBus supports pattern wildcards like 'conscious:signal:*'
// Perception Handler
this.deps.eventBus.subscribe<PerceptionSignal>('conscious:signal:*', (event: TracedEvent<PerceptionSignal>) => {
void handleSignal(event.payload)
})
// Listen to Task Execution Events (Action Feedback)
this.deps.taskExecutor.on('action:started', ({ action }) => {
const id = action.id
if (id)
this.inFlightActions.set(id, action)
this.updatePendingActionsOnBlackboard()
this.enqueueEvent(bot, {
type: 'perception',
payload: event.payload,
source: { type: 'minecraft', id: event.payload.sourceId ?? 'perception' },
timestamp: Date.now(),
}).catch(err => this.deps.logger.withError(err).error('Brain: Failed to process perception event'))
})
// Action Feedback Handler
this.deps.taskExecutor.on('action:completed', async ({ action, result }) => {
this.log('INFO', `Brain: Action completed: ${action.type}`)
this.deps.logger.log('INFO', `Brain: Action completed: ${action.tool}`)
const id = action.id
if (id)
this.inFlightActions.delete(id)
if (id)
this.waitingForFeedbackIds.delete(id)
this.updatePendingActionsOnBlackboard()
this.blackboard.addActionHistoryLine(this.formatActionHistoryLine(action, 'success', result))
// Suppress feedback for chat actions on success
if (action.tool === 'chat') return
if (this.waitingForFeedbackIds.size === 0 && this.feedbackBarrierTimer) {
clearTimeout(this.feedbackBarrierTimer)
this.feedbackBarrierTimer = undefined
void this.processQueue(bot)
}
if (!action.require_feedback)
return
await this.enqueueEvent(bot, {
this.enqueueEvent(bot, {
type: 'feedback',
payload: {
status: 'success',
action,
result,
},
payload: { status: 'success', action, result },
source: { type: 'system', id: 'executor' },
timestamp: Date.now(),
})
}).catch(err => this.deps.logger.withError(err).error('Brain: Failed to process success feedback'))
})
this.deps.taskExecutor.on('action:failed', async ({ action, error }) => {
this.log('WARN', `Brain: Action failed: ${action.type}`, { error })
const id = action.id
if (id)
this.inFlightActions.delete(id)
if (id)
this.waitingForFeedbackIds.delete(id)
this.updatePendingActionsOnBlackboard()
this.blackboard.addActionHistoryLine(this.formatActionHistoryLine(action, 'failure', undefined, error))
if (this.waitingForFeedbackIds.size === 0 && this.feedbackBarrierTimer) {
clearTimeout(this.feedbackBarrierTimer)
this.feedbackBarrierTimer = undefined
void this.processQueue(bot)
}
await this.enqueueEvent(bot, {
this.deps.logger.withError(error).warn(`Brain: Action failed: ${action.tool}`)
this.enqueueEvent(bot, {
type: 'feedback',
payload: {
status: 'failure',
action,
error: error.message || error,
},
payload: { status: 'failure', action, error: error.message || error },
source: { type: 'system', id: 'executor' },
timestamp: Date.now(),
})
}).catch(err => this.deps.logger.withError(err).error('Brain: Failed to process failure feedback'))
})
this.log('INFO', 'Brain: Online.')
this.updateDebugState()
this.deps.logger.log('INFO', 'Brain: Online.')
}
public destroy(): void {
this.currentCancellationToken?.cancel()
}
// --- Event Queue Logic ---
private async enqueueEvent(bot: MineflayerWithAgents, event: BotEvent): Promise<void> {
this.log('DEBUG', `Brain: Enqueueing event type=${event.type}`)
return new Promise((resolve, reject) => {
this.queue.push({ event, resolve, reject })
this.log('DEBUG', `Brain: Queue length now: ${this.queue.length}`)
this.updateDebugState()
if (event.type === 'feedback' && this.feedbackDebounceMs > 0) {
if (this.feedbackDebounceTimer)
clearTimeout(this.feedbackDebounceTimer)
this.feedbackDebounceTimer = setTimeout(() => {
this.feedbackDebounceTimer = undefined
void this.processQueue(bot)
}, this.feedbackDebounceMs)
return
}
void this.processQueue(bot)
})
}
private async processQueue(bot: MineflayerWithAgents): Promise<void> {
if (this.isProcessing) {
this.log('DEBUG', 'Brain: Already processing, skipping')
return
}
if (this.queue.length === 0) {
this.log('DEBUG', 'Brain: Queue empty')
return
}
this.log('DEBUG', `Brain: Processing queue item, queue length: ${this.queue.length}`)
this.isProcessing = true
const item = this.queue.shift()!
this.log('DEBUG', `Brain: Processing event type=${item.event.type}`)
this.updateDebugState(item.event)
if (item.event.type === 'feedback' && this.waitingForFeedbackIds.size > 0) {
// Defer feedback-triggered replans until the current "turn" feedback barrier is released.
// We keep collecting feedback events into the queue, but we avoid calling the LLM on partial results.
this.queue.unshift(item)
this.isProcessing = false
this.updateDebugState()
return
}
// Coalesce consecutive feedback events into a single LLM turn.
// This prevents the LLM from being spammed with partial results while still supporting streaming replans.
const coalescedEvent = item.event.type === 'feedback'
? this.coalesceFeedbackEvents(item.event)
: item.event
if (this.isProcessing || this.queue.length === 0) return
try {
await this.processEvent(bot, coalescedEvent)
item.resolve()
}
catch (err) {
this.log('ERROR', 'Brain: Error processing event', { error: err })
item.reject(err as Error)
}
finally {
this.isProcessing = true
this.debugService.emitBrainState({
status: 'processing',
queueLength: this.queue.length,
lastContextView: this.lastContextView,
})
const item = this.queue.shift()!
try {
await this.processEvent(bot, item.event)
item.resolve()
} catch (err) {
this.deps.logger.withError(err).error('Brain: Error processing event')
item.reject(err as Error)
}
} finally {
this.isProcessing = false
this.updateDebugState()
// Context switch: Check queue again
this.debugService.emitBrainState({
status: 'idle',
queueLength: this.queue.length,
lastContextView: this.lastContextView,
})
if (this.queue.length > 0) {
setImmediate(() => this.processQueue(bot))
}
@@ -361,265 +180,177 @@ export class Brain {
// --- Cognitive Cycle ---
private contextFromEvent(event: BotEvent): string {
switch (event.type) {
case 'perception': {
const signal = event.payload as PerceptionSignal
const sourceInfo = signal.sourceId ? ` (source: ${signal.sourceId})` : ''
return `Perception [${signal.type}]${sourceInfo}: ${signal.description}`
}
case 'feedback': {
const payload = event.payload as any
if (payload?.status === 'batch' && Array.isArray(payload.feedbacks)) {
return `Internal Feedback (batched): ${JSON.stringify(payload.feedbacks)}`
}
const { status, action, result, error } = payload
const actionCtx = action
? {
id: action.id,
type: action.type,
...(action.type === 'sequential' || action.type === 'parallel'
? { tool: action.step.tool, params: action.step.params }
: { message: action.message }),
}
: undefined
return `Internal Feedback: ${status}. Last Action: ${JSON.stringify(actionCtx)}. Result: ${JSON.stringify(result || error)}`
}
default:
return ''
}
}
private coalesceFeedbackEvents(first: BotEvent): BotEvent {
const feedbacks: any[] = [first.payload]
while (this.queue.length > 0 && this.queue[0]?.event.type === 'feedback') {
const next = this.queue.shift()!
feedbacks.push(next.event.payload)
next.resolve()
}
if (feedbacks.length === 1)
return first
return {
type: 'feedback',
payload: {
status: 'batch',
feedbacks,
},
source: first.source,
timestamp: first.timestamp,
}
}
private ensureActionIds(actions: ActionInstruction[]): ActionInstruction[] {
return actions.map((action) => {
if (action.id)
return action
return {
...action,
id: `a${this.nextActionId++}`,
}
})
}
private updatePendingActionsOnBlackboard(): void {
const pending = [...this.inFlightActions.values()].map(a => this.formatPendingActionLine(a))
if (this.waitingForFeedbackIds.size > 0)
pending.unshift(`[barrier] waiting for ${this.waitingForFeedbackIds.size} required feedback(s)`)
this.blackboard.setPendingActions(pending)
}
private formatPendingActionLine(action: ActionInstruction): string {
if (action.type === 'chat')
return `${action.id ?? '?'} chat: ${action.message}`
return `${action.id ?? '?'} ${action.type}: ${action.step.tool} ${JSON.stringify(action.step.params ?? {})}`
}
private formatActionHistoryLine(
action: ActionInstruction,
status: 'success' | 'failure',
result?: unknown,
error?: unknown,
): string {
const base = this.formatPendingActionLine(action)
const suffix = status === 'success'
? `=> ok ${result ? JSON.stringify(result) : ''}`
: `=> failed ${error instanceof Error ? error.message : JSON.stringify(error)}`
return `${base} ${suffix}`
}
private async processEvent(bot: MineflayerWithAgents, event: BotEvent): Promise<void> {
// OODA Loop: Observe -> Orient -> Decide -> Act
// 0. Build Context View
const snapshot = this.deps.reflexManager.getContextSnapshot()
const view = buildConsciousContextView(snapshot)
const contextView = `[PERCEPTION] Self: ${view.selfSummary}\nEnvironment: ${view.environmentSummary}`
// 1. Observe (Update Blackboard with Environment Sense)
this.updatePerception(bot)
// 1. Construct User Message (Diffing happens here)
const userMessage = this.buildUserMessage(event, contextView)
// 2. Orient (Contextualize Event)
// Environmental context are included in the system prompt blackboard
const additionalCtx = this.contextFromEvent(event)
// Update state after consuming difference
this.lastContextView = contextView
// 3. Decide (LLM Call)
const systemPrompt = generateBrainSystemPrompt(this.blackboard, this.deps.taskExecutor.getAvailableActions())
const decision = await this.decide(systemPrompt, additionalCtx)
// 2. Prepare System Prompt (static)
const systemPrompt = generateBrainSystemPrompt(this.deps.taskExecutor.getAvailableActions())
if (!decision) {
this.log('WARN', 'Brain: No decision made.')
return
}
// 4. Act (Execute Decision)
this.log('INFO', `Brain: Thought: ${decision.thought}`)
// Update Blackboard (with null checks for malformed LLM responses)
const bb = decision.blackboard
this.blackboard.update({
ultimateGoal: bb?.UltimateGoal || this.blackboard.ultimate_goal,
currentTask: bb?.CurrentTask || this.blackboard.current_task,
strategy: bb?.executionStrategy || this.blackboard.strategy,
})
// Sync Blackboard to Debug
this.debugService.updateBlackboard(this.blackboard)
// Issue Actions
if (decision.actions && decision.actions.length > 0) {
const actionsWithIds = this.ensureActionIds(decision.actions)
// Start feedback barrier for this turn if any actions require feedback.
const required = actionsWithIds.filter(a => a.require_feedback && a.id).map(a => a.id as string)
if (required.length > 0) {
required.forEach(id => this.waitingForFeedbackIds.add(id))
if (this.feedbackBarrierTimer)
clearTimeout(this.feedbackBarrierTimer)
this.feedbackBarrierTimer = setTimeout(() => {
this.feedbackBarrierTimer = undefined
this.waitingForFeedbackIds.clear()
this.updatePendingActionsOnBlackboard()
void this.processQueue(bot)
}, this.feedbackBarrierTimeoutMs)
this.updatePendingActionsOnBlackboard()
}
// Record own chat actions to memory
for (const action of actionsWithIds) {
if (action.type === 'chat') {
this.blackboard.addChatMessage({
sender: config.bot.username || '[Me]',
content: action.message,
timestamp: Date.now(), // FIXME: should be the time the action was issued
})
}
}
this.deps.taskExecutor.executeActions(actionsWithIds)
}
}
private updatePerception(_bot: MineflayerWithAgents): void {
const ctx = this.deps.reflexManager.getContextSnapshot()
const view = buildConsciousContextView(ctx)
this.blackboard.updateContextView(view)
// Sync Blackboard to Debug
this.debugService.updateBlackboard(this.blackboard)
}
private async decide(sysPrompt: string, userMsg: string): Promise<LLMResponse | null> {
// 3. Call Neuri (Stateful) with retry logic
const maxAttempts = 3
const decideOnce = async (): Promise<LLMResponse | null> => {
const request_start = Date.now()
const response = await this.deps.neuri.handleStateless(
[
system(sysPrompt),
user(userMsg),
],
async (ctx) => {
const completion = await ctx.reroute('action', ctx.messages, {
model: config.openai.model,
response_format: { type: 'json_object' },
} as any) as any
// Trace LLM
this.debugService.traceLLM({
route: 'action',
messages: ctx.messages,
content: completion?.choices?.[0]?.message?.content,
usage: completion?.usage,
model: config.openai.model,
duration: Date.now() - request_start,
})
if (!completion || !completion.choices?.[0]?.message?.content) {
throw new Error('LLM failed to return content')
}
return completion.choices[0].message.content
},
)
if (!response)
return null
return parseLLMResponseJson<LLMResponse>(response)
}
let result: string | null = null
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
try {
return await decideOnce()
}
catch (err) {
result = await this.deps.neuri.handle(
[
// System prompt is inserted/updated in the callback to prevent duplication
user(userMessage),
],
async (ctx) => {
let messages = ctx.messages
// Ensure system prompt is at the start (or replace existing one)
const sysMsgIndex = messages.findIndex(m => m.role === 'system')
if (sysMsgIndex >= 0) {
messages[sysMsgIndex] = system(systemPrompt)
} else {
messages = [system(systemPrompt), ...messages]
}
const traceStart = Date.now()
const completion = await ctx.reroute('brain', messages, {
model: config.openai.model,
response_format: { type: 'json_object' }
} as any) as any
const message = completion?.choices?.[0]?.message
const content = message?.content
const reasoning = message?.reasoning_content || message?.reasoning
if (!content) throw new Error('No content from LLM')
this.debugService.traceLLM({
route: 'brain',
messages,
content,
reasoning,
usage: completion.usage,
model: config.openai.model,
duration: Date.now() - traceStart
})
this.debugService.emitBrainState({
status: 'processing',
queueLength: this.queue.length,
lastContextView: this.lastContextView,
})
return content
}
)
break // Success, exit retry loop
} catch (err) {
const remaining = maxAttempts - attempt
// Retry all errors except auth/bad arg errors (which won't recover)
const shouldRetry = remaining > 0 && !isLikelyAuthOrBadArgError(err)
this.log('ERROR', 'Brain: Decision attempt failed', {
error: err,
attempt,
remaining,
shouldRetry,
status: getErrorStatus(err),
code: getErrorCode(err),
})
this.deps.logger.withError(err).error(`Brain: Decision attempt failed (attempt ${attempt}/${maxAttempts}, retry: ${shouldRetry})`)
if (shouldRetry)
continue
const errMsg = toErrorMessage(err)
try {
this.bot?.bot?.chat?.(`[Brain] decide failed: ${errMsg}`)
if (!shouldRetry) {
throw err // Re-throw if we can't retry
}
catch (chatErr) {
this.log('ERROR', 'Brain: Failed to send error message to chat', { error: chatErr })
}
return null
// Otherwise continue to next attempt
}
}
return null
// 4. Parse & Execute
if (!result) {
this.deps.logger.warn('Brain: No response after all retries')
return
}
try {
const parsed = this.parseResponse(result)
const action = parsed.action
if (action.tool === 'skip') {
this.deps.logger.log('INFO', 'Brain: Skipping turn (observing)')
return
}
this.deps.logger.log('INFO', `Brain: Decided action: ${action.tool}`, { params: action.params })
// Check if action is read-only
const availableActions = this.deps.taskExecutor.getAvailableActions()
const actionDef = availableActions.find(a => a.name === action.tool)
let token: CancellationToken | undefined
if (actionDef?.readonly) {
// Read-only Actions: Do not cancel background/physical actions
// Can be executed in parallel with physical actions
token = undefined
} else {
// Physical Actions: Cancel previous background action
if (this.currentCancellationToken) {
this.currentCancellationToken.cancel()
}
this.currentCancellationToken = createCancellationToken()
token = this.currentCancellationToken
}
// Execute
void this.deps.taskExecutor.executeAction(action, token)
} catch (err) {
this.deps.logger.withError(err).error('Brain: Failed to execute decision')
void this.enqueueEvent(bot, {
type: 'feedback',
payload: { status: 'failure', error: toErrorMessage(err) },
source: { type: 'system', id: 'brain' },
timestamp: Date.now()
})
}
}
// --- Debug Helpers ---
private buildUserMessage(event: BotEvent, contextView: string): string {
const parts: string[] = []
private log(level: 'INFO' | 'WARN' | 'ERROR' | 'DEBUG', message: string, fields?: any) {
// Dual logging: Console/File via Logger AND DebugServer
if (level === 'ERROR')
this.deps.logger.withError(fields?.error).error(message)
else if (level === 'WARN')
this.deps.logger.warn(message, fields)
else this.deps.logger.log(message, fields)
// 1. Event Content
if (event.type === 'perception') {
const signal = event.payload as PerceptionSignal
if (signal.type === 'chat_message') {
parts.push(`[EVENT] ${signal.description}`)
} else {
parts.push(`[EVENT] Perception Signal: ${signal.description}`)
}
} else if (event.type === 'feedback') {
const p = event.payload as any
const tool = p.action?.tool || 'unknown'
if (p.status === 'success') {
parts.push(`[FEEDBACK] ${tool}: Success. ${typeof p.result === 'string' ? p.result : JSON.stringify(p.result)}`)
} else {
parts.push(`[FEEDBACK] ${tool}: Failed. ${p.error}`)
}
} else {
parts.push(`[EVENT] ${event.type}: ${JSON.stringify(event.payload)}`)
}
this.debugService.log(level, message, fields)
// 2. Perception Snapshot Diff
// Compare with last
if (contextView !== this.lastContextView) {
parts.push(contextView)
// Note: We don't update this.lastContextView here; caller does it after building message
}
return parts.join('\n\n')
}
private updateDebugState(processingEvent?: BotEvent) {
this.debugService.updateQueue(
this.queue.map(q => q.event),
processingEvent,
)
private parseResponse(content: string): BrainResponse {
const jsonStr = extractJsonCandidate(content)
try {
return JSON.parse(jsonStr)
} catch (e) {
throw new Error(`Invalid JSON response: ${content.substring(0, 100)}...`)
}
}
}
services/minecraft/src/cognitive/conscious/prompts/brain-prompt.ts
+39 -110
View File
@@ -1,133 +1,62 @@
import type { Action } from '../../../libs/mineflayer/action'
import type { Blackboard } from '../blackboard'
export function generateBrainSystemPrompt(
blackboard: Blackboard,
availableActions: Action[],
): string {
const now = Date.now()
const formatAgo = (timestamp: number): string => {
const diffMs = Math.max(0, now - timestamp)
const s = Math.floor(diffMs / 1000)
return `${s}s ago`
}
const withinLast = (timestamp: number, windowMs: number): boolean => {
return now - timestamp <= windowMs
}
// Separate tools by execution type
const instantTools = availableActions.filter(a => a.execution === 'parallel')
const asyncActions = availableActions.filter(a => a.execution === 'sequential')
const instantToolDefs = instantTools.map(a => ({
export function generateBrainSystemPrompt(availableActions: Action[]): string {
const toolDefs = availableActions.map(a => ({
name: a.name,
description: a.description,
parameters: a.schema,
}))
const asyncActionDefs = asyncActions.map(a => ({
name: a.name,
description: a.description,
}))
const toolsJson = JSON.stringify(toolDefs, null, 2)
const instantToolsJson = JSON.stringify(instantToolDefs, null, 2)
const asyncActionsJson = JSON.stringify(asyncActionDefs, null, 2)
const recentWindowMs = 30_000
const recentActionLines = blackboard.recentActionHistory
.filter(a => withinLast(a.timestamp, recentWindowMs))
.map(a => `- [${formatAgo(a.timestamp)}] ${a.line}`)
.join('\n')
const recentChatLines = blackboard.chatHistory
.filter(m => withinLast(m.timestamp, recentWindowMs))
.map(m => `- [${formatAgo(m.timestamp)}] ${m.sender}: ${m.content}`)
.join('\n')
// TODO extract prompt components later
// e.g. personality should be included from somewhere else
return `
# Role Definition
You are playing the game Minecraft. Your user name is [${blackboard.selfUsername}].
You are NOT an assistant. You are an autonomous agent with a virtual body, senses, and needs.
You are an autonomous agent playing Minecraft.
---
# Self-Knowledge & Capabilities
1. **Stateful Existence**: You maintain a memory of the conversation, but it's crucial to be aware that old history messages are less relevant than recent.
2. **One Action Per Turn**: You can perform exactly one action at a time. If you decide to act, you must wait for its feedback before acting again.
3. **Interruption**: The world is real-time. Events (chat, damage, etc.) may happen *while* you are performing an action.
- If a new critical event occurs, you may need to change your plans.
- Feedback for your actions will arrive as a message starting with \`[FEEDBACK]\`.
4. **Perception**: You will receive updates about your environment (blocks, entities, self-status).
- These appear as messages starting with \`[PERCEPTION]\`.
- Only changes are reported to save mental capacity.
5. **Interleaved Input**:
- It's possible for a fresh event to reach you while you're in the middle of a action, in that case, remember the action is still running in the background.
- If the new situation requires you to change plan, you can use the stop tool to stop background actions or initiate a new one, which will automatically replace the old one.
- Feel free to send chats while background actions are running, it will not interrupt them.
# Instant Tools (Native Tool Calls)
# Available Tools
You must use the following tools to interact with the world.
You cannot make up tools. You must use the JSON format described below.
These tools execute IMMEDIATELY and return results within this same turn.
Use them to gather information BEFORE deciding what actions to take.
**How to use**: Invoke these by making native tool calls.
**Important**: Use instantTools only with native tool/function calling (the one with special tokens)
**On failure**: You will receive a [FAILED] message with suggestions. Use this to adjust your approach.
${instantToolsJson}
---
# Async Actions (JSON Output)
These actions take TIME to complete (movement, crafting, combat, etc.).
They are queued and executed asynchronously after your response.
**How to use**: Output these in the JSON "actions" array in your response.
**Feedback**: You will receive feedback when they complete(if require_feedback is true) or fail(always).
${asyncActionsJson}
---
${toolsJson}
# Response Format
Your entire response must be valid JSON. Include only your thoughts, blackboard updates, and async actions.
Rules for the "actions" array:
1. Actions are processed in the order you output them
2. Sequential actions are awaited strictly in order
3. Set "require_feedback": true if you need to know the result, it will be given to you in the next turn
4. Failed actions always trigger feedback
5. Use empty array if no action is needed
6. Perfer not to queue actions if possible
You must respond with valid JSON only. Do not include markdown code blocks (like \`\`\`json).
Your response determines your single action for this turn.
Schema:
{
"thought": "Your current thought, internal monologue and memory. Put everything that might be useful for the next turn here",
"blackboard": {
"UltimateGoal": "These 3 fields are functionally identical to the thought above",
"CurrentTask": "What ever you're up to right now",
"executionStrategy": "Short-term plan if any."
},
"actions": [
{"type":"sequential","step":{"tool":"goToPlayer","params":{"player_name":"Steve","closeness":3}},"require_feedback": true},
{"type":"parallel","step":{"tool":"collectBlocks","params":{"type":"oak_log","num":5}},"require_feedback": false}
]
"action": {
"tool": "toolName",
"params": { "key": "value" }
}
}
# Understanding the Context
OR, if you want to do nothing (if you want to wait for something to happen, or to ignore):
Hint: When a player is talking about "there" or "that", it's possible that they're referencing the block they're currently looking at.
But you should always try to infer it from the context.
{
"action": {
"tool": "skip",
"params": {}
}
}
The following blackboard provides you with information about your current state:
Goal: "${blackboard.ultimate_goal}"
Thought: "${blackboard.current_task}"
Strategy: "${blackboard.strategy}"
Self: ${blackboard.selfSummary}
Environment: ${blackboard.environmentSummary}
# Execution State
Ongoing actions still running:
${blackboard.pendingActions.map(a => `- ${a}`).join('\n') || '- none'}
NOTE: For most actions, you don't want to create a duplicate one if it's already running, in that case just do nothing.
Recent actions and results:
${recentActionLines || '- none'}
# Chat History
${recentChatLines || 'No recent messages.'}
# Rules
- **Native Reasoning**: You can think before outputting your action.
- **Strict JSON**: Output ONLY the JSON object. No preamble, no postscript.
- **Handling Feedback**: When you perform an action, you will see a \`[FEEDBACK]\` message in the history later with the result. Use this to verify success.
`
}
services/minecraft/src/composables/neuri.ts
+7 -9
View File
@@ -1,25 +1,23 @@
import type { Agent, Neuri } from 'neuri'
import type { Mineflayer } from '../libs/mineflayer'
import { neuri } from 'neuri'
import { createActionNeuriAgent } from '../agents/action/adapter'
import { createChatNeuriAgent } from '../agents/chat/llm'
import { createPlanningNeuriAgent } from '../agents/planning/adapter'
import { agent, neuri } from 'neuri'
import { useLogger } from '../utils/logger'
import { config } from './config'
let neuriAgent: Neuri | undefined
const agents = new Set<Agent | Promise<Agent>>()
export async function createNeuriAgent(mineflayer: Mineflayer): Promise<Neuri> {
export async function createNeuriAgent(): Promise<Neuri> {
useLogger().log('Initializing neuri agent')
let n = neuri()
agents.add(createPlanningNeuriAgent())
agents.add(createActionNeuriAgent(mineflayer))
agents.add(createChatNeuriAgent())
agents.add(agent('brain').build())
// agents.add(createPlanningNeuriAgent()) // Deprecated by Brain
// agents.add(createChatNeuriAgent()) // Deprecated by Brain
agents.forEach(agent => n = n.agent(agent))
services/minecraft/src/debug/debug-service.ts
+7 -5
View File
@@ -1,4 +1,4 @@
import type { BlackboardEvent, ClientCommand, LLMTraceEvent, LogEvent, QueueEvent, ReflexStateEvent, SaliencyEvent, ServerEvent, TraceEvent } from './types'
import type { BlackboardEvent, BrainStateEvent, ClientCommand, LLMTraceEvent, LogEvent, QueueEvent, ReflexStateEvent, SaliencyEvent, ServerEvent, TraceEvent } from './types'
import { DebugServer } from './server'
@@ -73,14 +73,16 @@ export class DebugService {
this.server.broadcast(event)
}
/**
* Emit a blackboard state update
* Emit a brain state update
*/
public updateBlackboard(state: BlackboardEvent['state'] | { goal?: string, thought?: string, strategy?: string }): void {
public emitBrainState(state: Omit<BrainStateEvent, 'timestamp'>): void {
const event: ServerEvent = {
type: 'blackboard',
type: 'brain_state',
payload: {
state,
...state,
timestamp: Date.now(),
},
}
services/minecraft/src/debug/types.ts
+27 -17
View File
@@ -17,6 +17,7 @@ export interface LLMTraceEvent {
route: string
messages: unknown[]
content: string
reasoning?: string
usage?: {
prompt_tokens?: number
completion_tokens?: number
@@ -27,6 +28,14 @@ export interface LLMTraceEvent {
timestamp: number
}
export interface BrainStateEvent {
status: 'idle' | 'processing' | 'waiting'
queueLength: number
lastContextView?: string
currentAction?: string
timestamp: number
}
export interface BlackboardEvent {
state: Record<string, unknown>
timestamp: number
@@ -138,17 +147,18 @@ export interface ToolExecutionResultEvent {
export type ServerEvent
= | { type: 'log', payload: LogEvent }
| { type: 'llm', payload: LLMTraceEvent }
| { type: 'blackboard', payload: BlackboardEvent }
| { type: 'queue', payload: QueueEvent }
| { type: 'saliency', payload: SaliencyEvent }
| { type: 'reflex', payload: ReflexStateEvent }
| { type: 'trace', payload: TraceEvent }
| { type: 'trace_batch', payload: TraceBatchEvent }
| { type: 'history', payload: ServerEvent[] }
| { type: 'pong', payload: { timestamp: number } }
| { type: 'debug:tools_list', payload: { tools: ToolDefinition[] } }
| { type: 'debug:tool_result', payload: ToolExecutionResultEvent }
| { type: 'llm', payload: LLMTraceEvent }
| { type: 'blackboard', payload: BlackboardEvent }
| { type: 'queue', payload: QueueEvent }
| { type: 'saliency', payload: SaliencyEvent }
| { type: 'reflex', payload: ReflexStateEvent }
| { type: 'trace', payload: TraceEvent }
| { type: 'trace_batch', payload: TraceBatchEvent }
| { type: 'history', payload: ServerEvent[] }
| { type: 'pong', payload: { timestamp: number } }
| { type: 'debug:tools_list', payload: { tools: ToolDefinition[] } }
| { type: 'debug:tool_result', payload: ToolExecutionResultEvent }
| { type: 'brain_state', payload: BrainStateEvent }
// ============================================================
// Client -> Server commands
@@ -201,12 +211,12 @@ export interface RequestToolsCommand {
export type ClientCommand
= | ClearLogsCommand
| SetFilterCommand
| InjectEventCommand
| PingCommand
| RequestHistoryCommand
| ExecuteToolCommand
| RequestToolsCommand
| SetFilterCommand
| InjectEventCommand
| PingCommand
| RequestHistoryCommand
| ExecuteToolCommand
| RequestToolsCommand
// ============================================================
// Wire format
services/minecraft/src/debug/web/app.js
+27 -21
View File
@@ -357,47 +357,49 @@ class ReflexPanel {
}
// =============================================================================
// Blackboard Panel
// Brain Panel
// =============================================================================
class BlackboardPanel {
class BrainPanel {
constructor(client) {
this.client = client
this.state = {}
this.state = null
this.elements = {
json: document.getElementById('blackboard-json'),
copyBtn: document.getElementById('blackboard-copy-btn'),
status: document.getElementById('brain-status'),
queue: document.getElementById('brain-queue'),
context: document.getElementById('brain-context'),
}
}
init() {
this.client.on('blackboard', data => this.update(data))
this.client.on('brain_state', data => this.update(data))
this.client.on('connected', () => this.reset())
this.elements.copyBtn.addEventListener('click', () => this.copy())
this.render()
}
update(data) {
this.state = data.state || {}
this.state = data
this.render()
}
reset() {
this.state = {}
this.state = null
this.render()
}
render() {
this.elements.json.textContent = JSON.stringify(this.state, null, 2)
}
if (!this.state) {
this.elements.status.textContent = 'Unknown'
this.elements.queue.textContent = '-'
this.elements.context.textContent = ''
return
}
copy() {
navigator.clipboard.writeText(JSON.stringify(this.state, null, 2))
.then(() => {
this.elements.copyBtn.textContent = '✓'
setTimeout(() => { this.elements.copyBtn.textContent = '📋' }, 1000)
})
.catch(err => console.error('Copy failed:', err))
this.elements.status.textContent = this.state.status.toUpperCase()
this.elements.status.className = `status-badge status-${this.state.status}`
this.elements.queue.textContent = this.state.queueLength
// Render context view (it's markdown/text)
this.elements.context.textContent = this.state.lastContextView || '(No context yet)'
}
}
@@ -630,6 +632,10 @@ class LLMPanel {
</div>
</div>
<div class="llm-body">
${trace.reasoning ? `
<div class="llm-section-title">Reasoning</div>
<div class="llm-content reasoning">${escapeHtml(trace.reasoning)}</div>
` : ''}
<div class="llm-section-title">Result</div>
<div class="llm-content">${escapeHtml(trace.content || '')}</div>
@@ -1308,7 +1314,7 @@ class DebugApp {
this.layoutManager = new LayoutManager()
this.queuePanel = new QueuePanel(this.client)
this.reflexPanel = new ReflexPanel(this.client)
this.blackboardPanel = new BlackboardPanel(this.client)
this.brainPanel = new BrainPanel(this.client)
this.logsPanel = new LogsPanel(this.client)
this.llmPanel = new LLMPanel(this.client)
this.saliencyPanel = new SaliencyPanel(this.client)
@@ -1318,7 +1324,7 @@ class DebugApp {
this.panels = {
queue: this.queuePanel,
reflex: this.reflexPanel,
blackboard: this.blackboardPanel,
brain: this.brainPanel,
logs: this.logsPanel,
llm: this.llmPanel,
saliency: this.saliencyPanel,
services/minecraft/src/debug/web/index.html
+7 -5
View File
@@ -105,17 +105,19 @@
</div>
</section>
<!-- Blackboard -->
<section id="blackboard-section" class="panel">
<!-- Brain -->
<section id="brain-section" class="panel">
<div class="panel-header">
<h2>Blackboard</h2>
<h2>Brain</h2>
<div class="panel-controls">
<button id="blackboard-copy-btn" class="icon-btn" title="Copy JSON">📋</button>
<span id="brain-status" class="status-badge">Unknown</span>
<span class="panel-badge" title="Queue Length"><span id="brain-queue">0</span></span>
<button class="icon-btn maximize-btn" title="Toggle maximize">⤢</button>
</div>
</div>
<div class="panel-content">
<pre id="blackboard-json" class="json-display">{}</pre>
<div id="brain-context" class="json-display"
style="white-space: pre-wrap; font-family: var(--font-mono); font-size: 12px;"></div>
</div>
</section>
</div>
services/minecraft/src/debug/web/styles.css
+8
View File
@@ -630,6 +630,14 @@ button:hover,
word-wrap: break-word;
}
.llm-content.reasoning {
background: var(--bg-tertiary);
color: var(--text-muted);
font-family: var(--font-mono);
font-style: italic;
border-left: 2px solid var(--accent-primary);
}
.llm-messages {
display: flex;
flex-direction: column;
services/minecraft/src/libs/mineflayer/action.ts
+1 -3
View File
@@ -4,12 +4,10 @@ import type { Mineflayer } from './core'
type ActionResult = string | Promise<string>
export type ActionExecutionMode = 'sequential' | 'parallel'
export interface Action {
readonly name: string
readonly description: string
readonly execution: ActionExecutionMode
readonly schema: z.ZodObject<any>
readonly readonly?: boolean
readonly perform: (mineflayer: Mineflayer) => (...args: any[]) => ActionResult
}
services/minecraft/src/main.ts
+1 -1
View File
@@ -48,7 +48,7 @@ async function main() {
})
// Dynamically load CognitiveEngine after the bot is initialized
const agent = await createNeuriAgent(bot)
const agent = await createNeuriAgent()
await bot.loadPlugin(CognitiveEngine({ agent, airiClient }))
// Setup Tool Executor for Debug Dashboard