nanakura/moeka-project
1
0
Fork 0
Code Issues Pull Requests 1 Actions 1 Packages Projects Releases Wiki Activity

refactor(minecraft): better blackboard and task scheduling

Browse Source
This commit is contained in:
Rin
2026-02-18 11:12:04 +08:00
committed by Neko Ayaka
parent 112ed4ea9f
commit 0b3a9f24e5
7 changed files with 239 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/minecraft/src/agents/action/tools.ts
+25
View File
@@ -27,6 +27,7 @@ export const actionsList: Action[] = [
{
name: 'inventory',
description: 'Get your inventory.',
execution: 'parallel',
schema: z.object({}),
perform: mineflayer => (): string => {
const inventory = world.getInventoryCounts(mineflayer)
@@ -49,6 +50,7 @@ export const actionsList: Action[] = [
{
name: 'nearbyBlocks',
description: 'Get the blocks near you.',
execution: 'parallel',
schema: z.object({}),
perform: mineflayer => (): string => {
const blocks = world.getNearbyBlockTypes(mineflayer)
@@ -59,6 +61,7 @@ export const actionsList: Action[] = [
{
name: 'craftable',
description: 'Get the craftable items with your inventory.',
execution: 'parallel',
schema: z.object({}),
perform: mineflayer => (): string => {
const craftable = world.getCraftableItems(mineflayer)
@@ -68,6 +71,7 @@ export const actionsList: Action[] = [
{
name: 'entities',
description: 'Get the nearby players and entities.',
execution: 'parallel',
schema: z.object({}),
perform: mineflayer => (): string => {
const players = world.getNearbyPlayerNames(mineflayer)
@@ -85,6 +89,7 @@ export const actionsList: Action[] = [
{
name: 'stop',
description: 'Force stop all actions', // TODO: include name of the current action in description?
execution: 'sequential',
schema: z.object({}),
perform: mineflayer => async () => {
mineflayer.interrupt('stop tool called')
@@ -95,6 +100,7 @@ export const actionsList: Action[] = [
{
name: 'goToPlayer',
description: 'Go to the given player.',
execution: 'sequential',
schema: z.object({
player_name: z.string().describe('The name of the player to go to.'),
closeness: z.number().describe('How close to get to the player in blocks.').min(0),
@@ -108,6 +114,7 @@ export const actionsList: Action[] = [
{
name: 'followPlayer',
description: 'Endlessly follow the given player.',
execution: 'sequential',
schema: z.object({
player_name: z.string().describe('name of the player to follow.'),
follow_dist: z.number().describe('The distance to follow from.').min(0),
@@ -120,6 +127,7 @@ export const actionsList: Action[] = [
{
name: 'goToCoordinates',
description: 'Go to the given x, y, z location.',
execution: 'sequential',
schema: z.object({
x: z.number().describe('The x coordinate.'),
y: z.number().describe('The y coordinate.').min(-64).max(320),
@@ -134,6 +142,7 @@ export const actionsList: Action[] = [
{
name: 'searchForBlock',
description: 'Find and go to the nearest block of a given type in a given range.',
execution: 'sequential',
schema: z.object({
type: z.string().describe('The block type to go to.'),
search_range: z.number().describe('The range to search for the block.').min(32).max(512),
@@ -146,6 +155,7 @@ export const actionsList: Action[] = [
{
name: 'searchForEntity',
description: 'Find and go to the nearest entity of a given type in a given range.',
execution: 'sequential',
schema: z.object({
type: z.string().describe('The type of entity to go to.'),
search_range: z.number().describe('The range to search for the entity.').min(32).max(512),
@@ -169,6 +179,7 @@ export const actionsList: Action[] = [
{
name: 'givePlayer',
description: 'Give the specified item to the given player.',
execution: 'sequential',
schema: z.object({
player_name: z.string().describe('The name of the player to give the item to.'),
item_name: z.string().describe('The name of the item to give.'),
@@ -182,6 +193,7 @@ export const actionsList: Action[] = [
{
name: 'consume',
description: 'Eat/drink the given item.',
execution: 'sequential',
schema: z.object({
item_name: z.string().describe('The name of the item to consume.'),
}),
@@ -193,6 +205,7 @@ export const actionsList: Action[] = [
{
name: 'equip',
description: 'Equip the given item.',
execution: 'sequential',
schema: z.object({
item_name: z.string().describe('The name of the item to equip.'),
}),
@@ -204,6 +217,7 @@ export const actionsList: Action[] = [
{
name: 'putInChest',
description: 'Put the given item in the nearest chest.',
execution: 'sequential',
schema: z.object({
item_name: z.string().describe('The name of the item to put in the chest.'),
num: z.number().int().describe('The number of items to put in the chest.').min(1),
@@ -216,6 +230,7 @@ export const actionsList: Action[] = [
{
name: 'takeFromChest',
description: 'Take the given items from the nearest chest.',
execution: 'sequential',
schema: z.object({
item_name: z.string().describe('The name of the item to take.'),
num: z.number().int().describe('The number of items to take.').min(1),
@@ -237,6 +252,7 @@ export const actionsList: Action[] = [
{
name: 'discard',
description: 'Discard the given item from the inventory.',
execution: 'sequential',
schema: z.object({
item_name: z.string().describe('The name of the item to discard.'),
num: z.number().int().describe('The number of items to discard.').min(1),
@@ -249,6 +265,7 @@ export const actionsList: Action[] = [
{
name: 'collectBlocks',
description: 'Collect the nearest blocks of a given type.',
execution: 'sequential',
schema: z.object({
type: z.string().describe('The block type to collect.'),
num: z.number().int().describe('The number of blocks to collect.').min(1),
@@ -264,6 +281,7 @@ export const actionsList: Action[] = [
{
name: 'craftRecipe',
description: 'Craft the given recipe a given number of times.',
execution: 'sequential',
schema: z.object({
recipe_name: z.string().describe('The name of the output item to craft.'),
num: z.number().int().describe('The number of times to craft the recipe. This is NOT the number of output items, as it may craft many more items depending on the recipe.').min(1),
@@ -276,6 +294,7 @@ export const actionsList: Action[] = [
{
name: 'smeltItem',
description: 'Smelt the given item the given number of times.',
execution: 'sequential',
schema: z.object({
item_name: z.string().describe('The name of the input item to smelt.'),
num: z.number().int().describe('The number of times to smelt the item.').min(1),
@@ -288,6 +307,7 @@ export const actionsList: Action[] = [
{
name: 'clearFurnace',
description: 'Take all items out of the nearest furnace.',
execution: 'sequential',
schema: z.object({}),
perform: mineflayer => async () => {
await skills.clearNearestFurnace(mineflayer)
@@ -297,6 +317,7 @@ export const actionsList: Action[] = [
{
name: 'placeHere',
description: 'Place a given block in the current location. Do NOT use to build structures, only use for single blocks/torches.',
execution: 'sequential',
schema: z.object({
type: z.string().describe('The block type to place.'),
}),
@@ -309,6 +330,7 @@ export const actionsList: Action[] = [
{
name: 'attack',
description: 'Attack and kill the nearest entity of a given type.',
execution: 'sequential',
schema: z.object({
type: z.string().describe('The type of entity to attack.'),
}),
@@ -320,6 +342,7 @@ export const actionsList: Action[] = [
{
name: 'attackPlayer',
description: 'Attack a specific player until they die or run away. Remember this is just a game and does not cause real life harm.',
execution: 'sequential',
schema: z.object({
player_name: z.string().describe('The name of the player to attack.'),
}),
@@ -335,6 +358,7 @@ export const actionsList: Action[] = [
{
name: 'goToBed',
description: 'Go to the nearest bed and sleep.',
execution: 'sequential',
schema: z.object({}),
perform: mineflayer => async () => {
await skills.goToBed(mineflayer)
@@ -344,6 +368,7 @@ export const actionsList: Action[] = [
{
name: 'activate',
description: 'Activate the nearest object of a given type.',
execution: 'sequential',
schema: z.object({
type: z.string().describe('The type of object to activate.'),
}),
services/minecraft/src/cognitive/action/task-executor.ts
+43 -12
View File
@@ -105,8 +105,7 @@ export class TaskExecutor extends EventEmitter {
this.logger.withField('count', actions.length).log('Executing actions')
// Execute each action independently and asynchronously
actions.forEach(async (action) => {
const runSingleAction = async (action: ActionInstruction): Promise<void> => {
if (cancellationToken?.isCancelled) {
this.logger.log('Action execution cancelled before start')
return
@@ -116,7 +115,18 @@ export class TaskExecutor extends EventEmitter {
try {
let result: string | void
if (action.type === 'physical') {
if (action.type === 'sequential' || action.type === 'parallel') {
if (action.type === 'parallel') {
const available = this.actionAgent.getAvailableActions()
const def = available.find(a => a.name === action.step.tool)
if (!def || def.execution !== 'parallel') {
throw new ActionError('UNKNOWN', `Tool '${action.step.tool}' is not allowed for parallel actions`, {
tool: action.step.tool,
requestedExecution: action.type,
allowedExecution: def?.execution,
})
}
}
result = await this.actionAgent.performAction(action.step)
}
else if (action.type === 'chat') {
@@ -127,15 +137,7 @@ export class TaskExecutor extends EventEmitter {
throw new Error(`Unknown action type: ${(action as any).type}`)
}
if (cancellationToken?.isCancelled) {
// If cancelled during execution (and agent didn't throw), we might still consider it cancelled?
// But usually agents throw if cancelled.
// Just emit completed for now if it finished.
}
if (action.require_feedback) {
this.emit('action:completed', { action, result })
}
this.emit('action:completed', { action, result })
}
catch (error) {
this.logger.withError(error).error('Action execution failed')
@@ -146,8 +148,37 @@ export class TaskExecutor extends EventEmitter {
// failed actions always emit feedback
this.emit('action:failed', { action, error })
throw error
}
}
const sequentialActions = actions.filter(a => a.type === 'sequential')
const parallelActions = actions.filter(a => a.type === 'parallel' || a.type === 'chat')
// Fire and forget: parallel-safe actions can run concurrently.
parallelActions.forEach((action) => {
void runSingleAction(action).catch(() => {
// errors are emitted via events; nothing else to do here
})
})
// Sequential actions must be executed strictly in order.
void (async () => {
for (const action of sequentialActions) {
if (cancellationToken?.isCancelled) {
this.logger.log('Action execution cancelled before start')
return
}
try {
await runSingleAction(action)
}
catch (error) {
// Fail fast: stop executing remaining physical actions.
return
}
}
})()
}
public getAvailableActions() {
services/minecraft/src/cognitive/action/types/index.ts
+10 -4
View File
@@ -1,15 +1,21 @@
import type { PlanStep } from '../../../agents/planning/adapter'
export type ActionType = 'physical' | 'chat'
export type ActionType = 'sequential' | 'parallel' | 'chat'
export interface BaseActionInstruction {
type: ActionType
id?: string
description?: string
require_feedback?: boolean
}
export interface PhysicalActionInstruction extends BaseActionInstruction {
type: 'physical'
export interface SequentialActionInstruction extends BaseActionInstruction {
type: 'sequential'
step: PlanStep
}
export interface ParallelActionInstruction extends BaseActionInstruction {
type: 'parallel'
step: PlanStep
}
@@ -18,4 +24,4 @@ export interface ChatActionInstruction extends BaseActionInstruction {
message: string
}
export type ActionInstruction = PhysicalActionInstruction | ChatActionInstruction
export type ActionInstruction = SequentialActionInstruction | ParallelActionInstruction | ChatActionInstruction
services/minecraft/src/cognitive/conscious/blackboard.ts
+21
View File
@@ -15,12 +15,16 @@ export interface BlackboardState {
strategy: string
contextView: contextViewState
chatHistory: ChatMessage[]
recentActionHistory: string[]
pendingActions: string[]
selfUsername: string
}
export class Blackboard {
private _state: BlackboardState
private static readonly MAX_CHAT_HISTORY = 8
private static readonly MAX_ACTION_HISTORY = 12
private static readonly MAX_PENDING_ACTIONS = 12
constructor() {
this._state = {
@@ -32,6 +36,8 @@ export class Blackboard {
environmentSummary: 'Unknown',
},
chatHistory: [],
recentActionHistory: [],
pendingActions: [],
selfUsername: 'Bot',
}
}
@@ -43,6 +49,8 @@ export class Blackboard {
public get selfSummary(): string { return this._state.contextView.selfSummary }
public get environmentSummary(): string { return this._state.contextView.environmentSummary }
public get chatHistory(): ChatMessage[] { return this._state.chatHistory }
public get recentActionHistory(): string[] { return this._state.recentActionHistory }
public get pendingActions(): string[] { return this._state.pendingActions }
public get selfUsername(): string { return this._state.selfUsername }
// Setters (Partial updates allowed)
@@ -62,11 +70,24 @@ export class Blackboard {
this._state = { ...this._state, chatHistory: newHistory }
}
public addActionHistoryLine(line: string): void {
const next = [...this._state.recentActionHistory, line]
const trimmed = next.length > Blackboard.MAX_ACTION_HISTORY ? next.slice(-Blackboard.MAX_ACTION_HISTORY) : next
this._state = { ...this._state, recentActionHistory: trimmed }
}
public setPendingActions(lines: string[]): void {
const trimmed = lines.length > Blackboard.MAX_PENDING_ACTIONS ? lines.slice(0, Blackboard.MAX_PENDING_ACTIONS) : lines
this._state = { ...this._state, pendingActions: trimmed }
}
public getSnapshot(): BlackboardState {
return {
...this._state,
contextView: { ...this._state.contextView },
chatHistory: [...this._state.chatHistory],
recentActionHistory: [...this._state.recentActionHistory],
pendingActions: [...this._state.pendingActions],
}
}
}
services/minecraft/src/cognitive/conscious/brain.ts
+124 -6
View File
@@ -44,6 +44,12 @@ export class Brain {
private blackboard: Blackboard
private debugService: DebugService
private nextActionId = 1
private inFlightActions = new Map<string, ActionInstruction>()
private feedbackDebounceMs = Number.parseInt(process.env.BRAIN_FEEDBACK_DEBOUNCE_MS ?? '200')
private feedbackDebounceTimer: NodeJS.Timeout | undefined
// Event Queue
private queue: QueuedEvent[] = []
private isProcessing = false
@@ -97,8 +103,25 @@ export class Brain {
})
// Listen to Task Execution Events (Action Feedback)
this.deps.taskExecutor.on('action:started', ({ action }) => {
const id = action.id
if (id)
this.inFlightActions.set(id, action)
this.updatePendingActionsOnBlackboard()
})
this.deps.taskExecutor.on('action:completed', async ({ action, result }) => {
this.log('INFO', `Brain: Action completed: ${action.type}`)
const id = action.id
if (id)
this.inFlightActions.delete(id)
this.updatePendingActionsOnBlackboard()
this.blackboard.addActionHistoryLine(this.formatActionHistoryLine(action, 'success', result))
if (!action.require_feedback)
return
await this.enqueueEvent(bot, {
type: 'feedback',
payload: {
@@ -113,6 +136,13 @@ export class Brain {
this.deps.taskExecutor.on('action:failed', async ({ action, error }) => {
this.log('WARN', `Brain: Action failed: ${action.type}`, { error })
const id = action.id
if (id)
this.inFlightActions.delete(id)
this.updatePendingActionsOnBlackboard()
this.blackboard.addActionHistoryLine(this.formatActionHistoryLine(action, 'failure', undefined, error))
await this.enqueueEvent(bot, {
type: 'feedback',
payload: {
@@ -137,7 +167,18 @@ export class Brain {
this.queue.push({ event, resolve, reject })
this.log('DEBUG', `Brain: Queue length now: ${this.queue.length}`)
this.updateDebugState()
this.processQueue(bot)
if (event.type === 'feedback' && this.feedbackDebounceMs > 0) {
if (this.feedbackDebounceTimer)
clearTimeout(this.feedbackDebounceTimer)
this.feedbackDebounceTimer = setTimeout(() => {
this.feedbackDebounceTimer = undefined
void this.processQueue(bot)
}, this.feedbackDebounceMs)
return
}
void this.processQueue(bot)
})
}
@@ -157,8 +198,14 @@ export class Brain {
this.log('DEBUG', `Brain: Processing event type=${item.event.type}`)
this.updateDebugState(item.event)
// Coalesce consecutive feedback events into a single LLM turn.
// This prevents the LLM from being spammed with partial results while still supporting streaming replans.
const coalescedEvent = item.event.type === 'feedback'
? this.coalesceFeedbackEvents(item.event)
: item.event
try {
await this.processEvent(bot, item.event)
await this.processEvent(bot, coalescedEvent)
item.resolve()
}
catch (err) {
@@ -185,9 +232,20 @@ export class Brain {
return `Perception [${signal.type}]${sourceInfo}: ${signal.description}`
}
case 'feedback': {
const { status, action, result, error } = event.payload
const payload = event.payload as any
if (payload?.status === 'batch' && Array.isArray(payload.feedbacks)) {
return `Internal Feedback (batched): ${JSON.stringify(payload.feedbacks)}`
}
const { status, action, result, error } = payload
const actionCtx = action
? { type: action.type, ...(action.type === 'physical' ? { tool: action.step.tool, params: action.step.params } : { message: action.message }) }
? {
id: action.id,
type: action.type,
...(action.type === 'sequential' || action.type === 'parallel'
? { tool: action.step.tool, params: action.step.params }
: { message: action.message }),
}
: undefined
return `Internal Feedback: ${status}. Last Action: ${JSON.stringify(actionCtx)}. Result: ${JSON.stringify(result || error)}`
}
@@ -196,6 +254,64 @@ export class Brain {
}
}
private coalesceFeedbackEvents(first: BotEvent): BotEvent {
const feedbacks: any[] = [first.payload]
while (this.queue.length > 0 && this.queue[0]?.event.type === 'feedback') {
const next = this.queue.shift()!
feedbacks.push(next.event.payload)
next.resolve()
}
if (feedbacks.length === 1)
return first
return {
type: 'feedback',
payload: {
status: 'batch',
feedbacks,
},
source: first.source,
timestamp: first.timestamp,
}
}
private ensureActionIds(actions: ActionInstruction[]): ActionInstruction[] {
return actions.map((action) => {
if (action.id)
return action
return {
...action,
id: `a${this.nextActionId++}`,
}
})
}
private updatePendingActionsOnBlackboard(): void {
const pending = [...this.inFlightActions.values()].map(a => this.formatPendingActionLine(a))
this.blackboard.setPendingActions(pending)
}
private formatPendingActionLine(action: ActionInstruction): string {
if (action.type === 'chat')
return `${action.id ?? '?'} chat: ${action.message}`
return `${action.id ?? '?'} ${action.type}: ${action.step.tool} ${JSON.stringify(action.step.params ?? {})}`
}
private formatActionHistoryLine(
action: ActionInstruction,
status: 'success' | 'failure',
result?: unknown,
error?: unknown,
): string {
const base = this.formatPendingActionLine(action)
const suffix = status === 'success'
? `=> ok ${result ? JSON.stringify(result) : ''}`
: `=> failed ${error instanceof Error ? error.message : JSON.stringify(error)}`
return `${base} ${suffix}`
}
private async processEvent(bot: MineflayerWithAgents, event: BotEvent): Promise<void> {
// OODA Loop: Observe -> Orient -> Decide -> Act
@@ -230,8 +346,10 @@ export class Brain {
// Issue Actions
if (decision.actions && decision.actions.length > 0) {
const actionsWithIds = this.ensureActionIds(decision.actions)
// Record own chat actions to memory
for (const action of decision.actions) {
for (const action of actionsWithIds) {
if (action.type === 'chat') {
this.blackboard.addChatMessage({
sender: config.bot.username || '[Me]',
@@ -241,7 +359,7 @@ export class Brain {
}
}
this.deps.taskExecutor.executeActions(decision.actions)
this.deps.taskExecutor.executeActions(actionsWithIds)
}
}
services/minecraft/src/cognitive/conscious/prompts/brain-prompt.ts
+13 -2
View File
@@ -9,6 +9,7 @@ export function generateBrainSystemPrompt(
return {
name: a.name,
description: a.description,
execution: a.execution,
}
})
@@ -31,7 +32,9 @@ Available Actions:
${availableActionsJson}
Rules:
1. You can execute physical actions or chat actions
1. You can execute sequential actions, parallel actions, or chat actions
1.1. Sequential actions are executed strictly in order.
1.2. Parallel actions are parallel-safe and can be used for fast information gathering.
2. The output must be valid JSON following the schema below
3. Specify if a feedback is required for the action, i.e. whether you need to know the execution result for a good reason
4. Failed actions will always result in a feedback
@@ -48,7 +51,8 @@ Output format:
},
"actions": [
{"type":"chat","message":"...","require_feedback": false},
{"type":"physical","step":{"tool":"action name","params":{...}},"require_feedback": false}
{"type":"parallel","step":{"tool":"action name","params":{...}},"require_feedback": true},
{"type":"sequential","step":{"tool":"action name","params":{...}},"require_feedback": false}
]
}
@@ -61,6 +65,13 @@ Strategy: "${blackboard.strategy}"
Self: ${blackboard.selfSummary}
Environment: ${blackboard.environmentSummary}
# Execution State (IMPORTANT)
Pending actions (started and still running):
${blackboard.pendingActions.map(a => `- ${a}`).join('\n') || '- none'}
Recent action results (most recent last):
${blackboard.recentActionHistory.map(a => `- ${a}`).join('\n') || '- none'}
# Chat History (Recents):
${blackboard.chatHistory.map(msg => `- ${msg.sender}: ${msg.content}`).join('\n') || 'No recent messages.'}
`
services/minecraft/src/libs/mineflayer/action.ts
+3
View File
@@ -4,9 +4,12 @@ import type { Mineflayer } from './core'
type ActionResult = string | Promise<string>
export type ActionExecutionMode = 'sequential' | 'parallel'
export interface Action {
readonly name: string
readonly description: string
readonly execution: ActionExecutionMode
readonly schema: z.ZodObject<any>
readonly perform: (mineflayer: Mineflayer) => (...args: any[]) => ActionResult
}