diff --git a/services/minecraft/src/cognitive/conscious/brain.test.ts b/services/minecraft/src/cognitive/conscious/brain.test.ts index f2e59e9db..92c067013 100644 --- a/services/minecraft/src/cognitive/conscious/brain.test.ts +++ b/services/minecraft/src/cognitive/conscious/brain.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it, vi } from 'vitest' +import { z } from 'zod' import { Brain } from './brain' @@ -71,6 +72,30 @@ function createPerceptionEvent() { } as any } +function createAsyncControlAction(name: string = 'goToPlayer') { + return { + name, + description: `${name} action`, + execution: 'async', + schema: z.object({ + player_name: z.string(), + closeness: z.number(), + }), + perform: () => async () => 'ok', + } as any +} + +function createReadonlyAction(name: string = 'querySnapshot') { + return { + name, + description: `${name} action`, + execution: 'sync', + readonly: true, + schema: z.object({}), + perform: () => () => 'ok', + } as any +} + describe('brain no-action follow-up', () => { it('forgets conversation only', () => { const brain: any = new Brain(createDeps('await skip()')) @@ -294,3 +319,43 @@ describe('brain queue coalescing', () => { expect(brain.queue[2].event.type).toBe('feedback') }) }) + +describe('brain control action queue', () => { + it('does not block turn completion while control action executes in worker', async () => { + const deps: any = createDeps('await goToPlayer({ player_name: "Alex", closeness: 2 })') + const deferred = new Promise(() => {}) + deps.taskExecutor.getAvailableActions = vi.fn(() => [createAsyncControlAction('goToPlayer')]) + deps.taskExecutor.executeActionWithResult = vi.fn(async (action: any) => { + if (action.tool === 'goToPlayer') + return deferred + return 'ok' + }) + + const brain: any = new Brain(deps) + const outcome = await Promise.race([ + brain.processEvent({} as any, createPerceptionEvent()).then(() => 'done'), + new Promise(resolve => setTimeout(() => resolve('timeout'), 80)), + ]) + + expect(outcome).toBe('done') + const snapshot = brain.getDebugSnapshot() + expect(snapshot.actionQueue.counts.total).toBe(1) + expect(snapshot.actionQueue.executing?.tool ?? snapshot.actionQueue.pending[0]?.tool).toBe('goToPlayer') + }) + + it('executes readonly tools immediately without consuming control queue', async () => { + const deps: any = createDeps('await querySnapshot()') + deps.taskExecutor.getAvailableActions = vi.fn(() => [createReadonlyAction('querySnapshot')]) + deps.taskExecutor.executeActionWithResult = vi.fn(async () => 'snapshot-ok') + + const brain: any = new Brain(deps) + await brain.processEvent({} as any, createPerceptionEvent()) + + const snapshot = brain.getDebugSnapshot() + expect(snapshot.actionQueue.counts.total).toBe(0) + expect(deps.taskExecutor.executeActionWithResult).toHaveBeenCalledWith({ + tool: 'querySnapshot', + params: {}, + }) + }) +}) diff --git a/services/minecraft/src/cognitive/conscious/brain.ts b/services/minecraft/src/cognitive/conscious/brain.ts index 2bc2fe576..33c694cd2 100644 --- a/services/minecraft/src/cognitive/conscious/brain.ts +++ b/services/minecraft/src/cognitive/conscious/brain.ts @@ -1,6 +1,7 @@ import type { Logg } from '@guiiai/logg' import type { Message } from '@xsai/shared-chat' +import type { Action } from '../../libs/mineflayer/action' import type { TaskExecutor } from '../action/task-executor' import type { ActionInstruction } from '../action/types' import type { EventBus, TracedEvent } from '../os' @@ -123,6 +124,50 @@ interface RuntimeInputEnvelope { } } +type ActionQueueEntryState = 'pending' | 'executing' | 'succeeded' | 'failed' | 'cancelled' + +interface ActionQueueEntryView { + id: number + tool: string + params: Record + state: ActionQueueEntryState + enqueuedAt: number + sourceTurnId: number + startedAt?: number + finishedAt?: number + result?: unknown + error?: string +} + +interface ActionQueueSnapshot { + executing: ActionQueueEntryView | null + pending: ActionQueueEntryView[] + recent: ActionQueueEntryView[] + capacity: { + total: number + executing: number + pending: number + } + counts: { + total: number + executing: number + pending: number + } + updatedAt: number +} + +interface ControlActionQueueEntry { + id: number + action: ActionInstruction + sourceTurnId: number + state: ActionQueueEntryState + enqueuedAt: number + startedAt?: number + finishedAt?: number + result?: unknown + error?: string +} + function truncateForPrompt(value: string, maxLength = 220): string { return value.length <= maxLength ? value : `${value.slice(0, maxLength - 1)}...` } @@ -148,6 +193,9 @@ const EVENT_PRIORITY_PLAYER_CHAT = 0 const EVENT_PRIORITY_PERCEPTION = 1 const EVENT_PRIORITY_FEEDBACK = 2 const EVENT_PRIORITY_NO_ACTION_FOLLOWUP = 3 +const MAX_QUEUED_CONTROL_ACTIONS = 5 +const MAX_PENDING_CONTROL_ACTIONS = 4 +const ACTION_QUEUE_RECENT_HISTORY_LIMIT = 20 function getEventPriority(event: BotEvent): number { if (event.type === 'perception') { @@ -187,6 +235,13 @@ export class Brain { private turnCounter = 0 private currentInputEnvelope: RuntimeInputEnvelope | null = null private readonly llmLogRuntime = createLlmLogRuntime(() => this.llmLogEntries) + private nextControlActionId = 0 + private pendingControlActions: ControlActionQueueEntry[] = [] + private activeControlAction: ControlActionQueueEntry | null = null + private recentControlActions: ControlActionQueueEntry[] = [] + private actionQueueUpdatedAt = Date.now() + private isActionWorkerRunning = false + private completedControlActionsSinceLastFeedback = 0 constructor(private readonly deps: BrainDeps) { this.debugService = DebugService.getInstance() @@ -206,7 +261,7 @@ export class Brain { }).catch(err => this.deps.logger.withError(err).error('Brain: Failed to process perception event')) }) - // Action Feedback Handler + // Action telemetry logger this.deps.taskExecutor.on('action:completed', async ({ action, result }) => { this.deps.logger.log('INFO', `Brain: Action completed: ${action.tool}`) this.appendLlmLog({ @@ -234,12 +289,14 @@ export class Brain { this.giveUpReason = typeof action.params?.reason === 'string' ? action.params.reason : undefined } - this.enqueueEvent(bot, { - type: 'feedback', - payload: { status: 'success', action, result }, - source: { type: 'system', id: 'executor' }, - timestamp: Date.now(), - }).catch(err => this.deps.logger.withError(err).error('Brain: Failed to process success feedback')) + if (action.tool === 'chat' && action.params?.feedback === true) { + this.enqueueEvent(bot, { + type: 'feedback', + payload: { status: 'success', action, result }, + source: { type: 'system', id: 'executor' }, + timestamp: Date.now(), + }).catch(err => this.deps.logger.withError(err).error('Brain: Failed to process chat feedback')) + } }) this.deps.taskExecutor.on('action:failed', async ({ action, error }) => { @@ -256,12 +313,6 @@ export class Brain { params: action.params, }, }) - this.enqueueEvent(bot, { - type: 'feedback', - payload: { status: 'failure', action, error: error.message || error }, - source: { type: 'system', id: 'executor' }, - timestamp: Date.now(), - }).catch(err => this.deps.logger.withError(err).error('Brain: Failed to process failure feedback')) }) this.deps.logger.log('INFO', 'Brain: Online.') @@ -269,6 +320,9 @@ export class Brain { public destroy(): void { this.currentCancellationToken?.cancel() + this.clearPendingControlActions('cancelled') + this.activeControlAction = null + this.touchActionQueue() this.runtimeMineflayer = null } @@ -296,6 +350,7 @@ export class Brain { public getDebugSnapshot(): { isProcessing: boolean queueLength: number + actionQueue: ActionQueueSnapshot turnCounter: number giveUpUntil: number paused: boolean @@ -306,6 +361,7 @@ export class Brain { return { isProcessing: this.isProcessing, queueLength: this.queue.length, + actionQueue: this.getActionQueueSnapshot(), turnCounter: this.turnCounter, giveUpUntil: this.giveUpUntil, paused: this.paused, @@ -526,6 +582,7 @@ export class Brain { llmInput: this.lastLlmInputSnapshot, currentInput: this.currentInputEnvelope, llmLog: this.llmLogRuntime, + actionQueue: this.getActionQueueSnapshot(), forgetConversation: () => this.forgetConversation(), } } @@ -559,6 +616,316 @@ export class Brain { } } + private touchActionQueue(): void { + this.actionQueueUpdatedAt = Date.now() + } + + private cloneActionParams(params: Record): Record { + return JSON.parse(JSON.stringify(params)) as Record + } + + private toActionQueueEntryView(entry: ControlActionQueueEntry): ActionQueueEntryView { + return { + id: entry.id, + tool: entry.action.tool, + params: this.cloneActionParams(entry.action.params), + state: entry.state, + enqueuedAt: entry.enqueuedAt, + sourceTurnId: entry.sourceTurnId, + startedAt: entry.startedAt, + finishedAt: entry.finishedAt, + result: entry.result, + error: entry.error, + } + } + + private pushRecentControlAction(entry: ControlActionQueueEntry): void { + this.recentControlActions.push({ + ...entry, + action: { + tool: entry.action.tool, + params: this.cloneActionParams(entry.action.params), + }, + }) + if (this.recentControlActions.length > ACTION_QUEUE_RECENT_HISTORY_LIMIT) { + this.recentControlActions.shift() + } + } + + private getActionQueueSnapshot(): ActionQueueSnapshot { + const executing = this.activeControlAction ? this.toActionQueueEntryView(this.activeControlAction) : null + const pending = this.pendingControlActions.map(entry => this.toActionQueueEntryView(entry)) + const recent = this.recentControlActions.map(entry => this.toActionQueueEntryView(entry)) + const executingCount = executing ? 1 : 0 + const pendingCount = pending.length + + return { + executing, + pending, + recent, + capacity: { + total: MAX_QUEUED_CONTROL_ACTIONS, + executing: 1, + pending: MAX_PENDING_CONTROL_ACTIONS, + }, + counts: { + total: executingCount + pendingCount, + executing: executingCount, + pending: pendingCount, + }, + updatedAt: this.actionQueueUpdatedAt, + } + } + + private isQueueConsumingControlAction(action: ActionInstruction, actionDef: Action | undefined): boolean { + if (action.tool === 'chat' || action.tool === 'skip' || action.tool === 'stop') + return false + + if (!actionDef) + return false + + if (actionDef?.readonly) + return false + + return actionDef.execution === 'async' + } + + private clearPendingControlActions(state: Extract): number { + if (this.pendingControlActions.length === 0) + return 0 + + const clearedAt = Date.now() + const cleared = this.pendingControlActions.splice(0, this.pendingControlActions.length) + for (const entry of cleared) { + entry.state = state + entry.finishedAt = clearedAt + entry.error = state === 'failed' ? entry.error : entry.error ?? 'Cleared from action queue' + this.pushRecentControlAction(entry) + } + this.touchActionQueue() + return cleared.length + } + + private async enqueueControlAction( + bot: MineflayerWithAgents, + action: ActionInstruction, + sourceTurnId: number, + ): Promise { + const queueSize = this.pendingControlActions.length + (this.activeControlAction ? 1 : 0) + if (queueSize >= MAX_QUEUED_CONTROL_ACTIONS) { + throw new Error(`Action queue full (${queueSize}/${MAX_QUEUED_CONTROL_ACTIONS}). Use stop() or wait for completion.`) + } + + const entry: ControlActionQueueEntry = { + id: ++this.nextControlActionId, + action: { + tool: action.tool, + params: this.cloneActionParams(action.params), + }, + sourceTurnId, + state: 'pending', + enqueuedAt: Date.now(), + } + this.pendingControlActions.push(entry) + this.touchActionQueue() + + this.appendLlmLog({ + turnId: sourceTurnId, + kind: 'scheduler', + eventType: 'system_alert', + sourceType: 'system', + sourceId: 'brain:action_queue', + tags: ['scheduler', 'action_queue', 'enqueued'], + text: `Queued control action #${entry.id}: ${entry.action.tool}`, + metadata: { + actionId: entry.id, + pendingCount: this.pendingControlActions.length, + }, + }) + + this.startControlActionWorker(bot) + return { + queued: true, + actionId: entry.id, + state: entry.state, + pendingAhead: Math.max(0, this.pendingControlActions.length - 1), + queue: this.getActionQueueSnapshot().counts, + } + } + + private startControlActionWorker(bot: MineflayerWithAgents): void { + if (this.isActionWorkerRunning) + return + + this.isActionWorkerRunning = true + setImmediate(() => { + void this.runControlActionWorker(bot) + }) + } + + private async runControlActionWorker(bot: MineflayerWithAgents): Promise { + try { + while (this.pendingControlActions.length > 0) { + const entry = this.pendingControlActions.shift()! + entry.state = 'executing' + entry.startedAt = Date.now() + this.activeControlAction = entry + this.touchActionQueue() + + this.appendLlmLog({ + turnId: entry.sourceTurnId, + kind: 'scheduler', + eventType: 'system_alert', + sourceType: 'system', + sourceId: 'brain:action_queue', + tags: ['scheduler', 'action_queue', 'executing'], + text: `Executing control action #${entry.id}: ${entry.action.tool}`, + metadata: { + actionId: entry.id, + }, + }) + + const actionDef = this.deps.taskExecutor.getAvailableActions().find(item => item.name === entry.action.tool) + if (actionDef?.followControl === 'detach') + this.deps.reflexManager.clearFollowTarget() + + const cancellationToken = createCancellationToken() + this.currentCancellationToken = cancellationToken + + try { + const result = await this.deps.taskExecutor.executeActionWithResult(entry.action, cancellationToken) + entry.state = 'succeeded' + entry.result = result + entry.finishedAt = Date.now() + this.pushRecentControlAction(entry) + this.completedControlActionsSinceLastFeedback++ + + this.appendLlmLog({ + turnId: entry.sourceTurnId, + kind: 'scheduler', + eventType: 'feedback', + sourceType: 'system', + sourceId: 'brain:action_queue', + tags: ['scheduler', 'action_queue', 'success', entry.action.tool], + text: `Control action #${entry.id} succeeded: ${entry.action.tool}`, + }) + + this.activeControlAction = null + this.touchActionQueue() + + if (this.pendingControlActions.length === 0) { + const completedCount = this.completedControlActionsSinceLastFeedback + this.completedControlActionsSinceLastFeedback = 0 + await this.enqueueEvent(bot, { + type: 'feedback', + payload: { + status: 'success', + action: entry.action, + result: entry.result, + summary: { + queueDrained: true, + completedCount, + }, + }, + source: { type: 'system', id: 'executor' }, + timestamp: Date.now(), + }) + } + } + catch (err) { + const errorMessage = toErrorMessage(err) + entry.state = 'failed' + entry.error = errorMessage + entry.finishedAt = Date.now() + this.pushRecentControlAction(entry) + + const clearedCount = this.clearPendingControlActions('cancelled') + this.completedControlActionsSinceLastFeedback = 0 + this.activeControlAction = null + this.touchActionQueue() + + this.appendLlmLog({ + turnId: entry.sourceTurnId, + kind: 'scheduler', + eventType: 'feedback', + sourceType: 'system', + sourceId: 'brain:action_queue', + tags: ['scheduler', 'action_queue', 'failure', entry.action.tool], + text: `Control action #${entry.id} failed: ${entry.action.tool}`, + metadata: { + actionId: entry.id, + clearedPendingCount: clearedCount, + error: errorMessage, + }, + }) + + await this.enqueueEvent(bot, { + type: 'feedback', + payload: { + status: 'failure', + action: entry.action, + error: errorMessage, + summary: { + failedActionId: entry.id, + clearedPendingCount: clearedCount, + }, + }, + source: { type: 'system', id: 'executor' }, + timestamp: Date.now(), + }) + break + } + finally { + if (this.currentCancellationToken === cancellationToken) { + this.currentCancellationToken = undefined + } + } + } + } + finally { + this.isActionWorkerRunning = false + if (this.pendingControlActions.length > 0 && this.runtimeMineflayer) { + this.startControlActionWorker(this.runtimeMineflayer) + } + } + } + + private async executeStopAction(bot: MineflayerWithAgents, sourceTurnId: number): Promise { + const clearedCount = this.clearPendingControlActions('cancelled') + this.currentCancellationToken?.cancel() + + this.appendLlmLog({ + turnId: sourceTurnId, + kind: 'scheduler', + eventType: 'system_alert', + sourceType: 'system', + sourceId: 'brain:action_queue', + tags: ['scheduler', 'action_queue', 'stop'], + text: `Stop requested. Cleared pending control actions: ${clearedCount}`, + }) + + const result = await this.deps.taskExecutor.executeActionWithResult({ tool: 'stop', params: {} }) + void this.enqueueEvent(bot, { + type: 'feedback', + payload: { + status: 'success', + action: { tool: 'stop', params: {} }, + result, + summary: { + clearedPendingCount: clearedCount, + }, + }, + source: { type: 'system', id: 'executor' }, + timestamp: Date.now(), + }).catch(err => this.deps.logger.withError(err).error('Brain: Failed to enqueue stop feedback')) + + return { + ok: true, + stopped: true, + clearedPendingCount: clearedCount, + } + } + private queueNoActionFollowup( bot: MineflayerWithAgents, triggeringEvent: BotEvent, @@ -933,7 +1300,6 @@ export class Brain { } as Message) const actionDefs = new Map(this.deps.taskExecutor.getAvailableActions().map(action => [action.name, action])) - let turnCancellationToken: CancellationToken | undefined const normalizedLlmCode = this.normalizeReplCode(result) const codeToEvaluate = this.repl.canEvaluateAsExpression(normalizedLlmCode) @@ -946,20 +1312,17 @@ export class Brain { this.createRuntimeGlobals(event, snapshot as unknown as Record, bot), async (action: ActionInstruction) => { const actionDef = actionDefs.get(action.tool) + if (action.tool === 'stop') { + return this.executeStopAction(bot, turnId) + } + + const isControlAction = this.isQueueConsumingControlAction(action, actionDef) + if (isControlAction) + return this.enqueueControlAction(bot, action, turnId) + if (actionDef?.followControl === 'detach') this.deps.reflexManager.clearFollowTarget() - const isPhysicalAction = action.tool !== 'skip' && !actionDef?.readonly - - if (isPhysicalAction) { - if (!turnCancellationToken) { - this.currentCancellationToken?.cancel() - this.currentCancellationToken = createCancellationToken() - turnCancellationToken = this.currentCancellationToken - } - return this.deps.taskExecutor.executeActionWithResult(action, turnCancellationToken) - } - return this.deps.taskExecutor.executeActionWithResult(action) }, ) @@ -1116,7 +1479,13 @@ export class Brain { parts.push(`[SCRIPT] Last eval ${ageMs}ms ago: return=${returnValue}; actions=${this.lastReplOutcome.actionCount} (ok=${this.lastReplOutcome.okCount}, err=${this.lastReplOutcome.errorCount}); logs=${logs}`) } - parts.push('[RUNTIME] Globals are refreshed every turn: snapshot, self, environment, social, threat, attention, autonomy, event, now, query, bot, mineflayer, currentInput, llmLog, mem, lastRun, prevRun, lastAction. Player gaze is available in environment.nearbyPlayersGaze when needed.') + const queueSnapshot = this.getActionQueueSnapshot() + const runningLabel = queueSnapshot.executing + ? `${queueSnapshot.executing.tool}#${queueSnapshot.executing.id}` + : 'none' + parts.push(`[ACTION_QUEUE] executing=${runningLabel}; pending=${queueSnapshot.counts.pending}; total=${queueSnapshot.counts.total}/${queueSnapshot.capacity.total}`) + + parts.push('[RUNTIME] Globals are refreshed every turn: snapshot, self, environment, social, threat, attention, autonomy, event, now, query, bot, mineflayer, currentInput, llmLog, actionQueue, mem, lastRun, prevRun, lastAction. Player gaze is available in environment.nearbyPlayersGaze when needed.') return parts.join('\n\n') } diff --git a/services/minecraft/src/cognitive/conscious/js-planner.test.ts b/services/minecraft/src/cognitive/conscious/js-planner.test.ts index e23063598..726c907e6 100644 --- a/services/minecraft/src/cognitive/conscious/js-planner.test.ts +++ b/services/minecraft/src/cognitive/conscious/js-planner.test.ts @@ -46,6 +46,14 @@ describe('javaScriptPlanner', () => { updatedAt: Date.now(), attempt: 1, }, + actionQueue: { + executing: null, + pending: [], + recent: [], + capacity: { total: 5, executing: 1, pending: 4 }, + counts: { total: 0, executing: 0, pending: 0 }, + updatedAt: Date.now(), + }, forgetConversation: () => ({ ok: true, cleared: ['conversationHistory', 'lastLlmInputSnapshot'] }), } as any @@ -188,12 +196,21 @@ describe('javaScriptPlanner', () => { expect(names).toContain('mineflayer') expect(names).toContain('currentInput') expect(names).toContain('llmLog') + expect(names).toContain('actionQueue') expect(names).toContain('forget_conversation') const mem = descriptors.find(d => d.name === 'mem') expect(mem?.readonly).toBe(false) }) + it('exposes actionQueue runtime global to scripts', async () => { + const planner = new JavaScriptPlanner() + const executeAction = vi.fn(async action => `ok:${action.tool}`) + const planned = await planner.evaluate('return actionQueue.capacity.total', actions, globals, executeAction) + expect(planned.returnValue).toBe('5') + expect(planned.actions).toHaveLength(0) + }) + it('exposes llm input globals to scripts', async () => { const planner = new JavaScriptPlanner() const executeAction = vi.fn(async action => `ok:${action.tool}`) diff --git a/services/minecraft/src/cognitive/conscious/js-planner.ts b/services/minecraft/src/cognitive/conscious/js-planner.ts index f6872f4a7..e4402c75d 100644 --- a/services/minecraft/src/cognitive/conscious/js-planner.ts +++ b/services/minecraft/src/cognitive/conscious/js-planner.ts @@ -67,6 +67,7 @@ export interface RuntimeGlobals { snapshot: Record mineflayer?: Mineflayer | null bot?: unknown + actionQueue?: unknown currentInput?: unknown llmLog?: unknown forgetConversation?: () => { ok: true, cleared: string[] } @@ -212,6 +213,7 @@ export class JavaScriptPlanner { { name: 'llmInput', kind: 'object', readonly: true }, { name: 'currentInput', kind: 'object', readonly: true }, { name: 'llmLog', kind: 'object', readonly: true }, + { name: 'actionQueue', kind: 'object', readonly: true }, { name: 'forget_conversation', kind: 'function', readonly: true }, { name: 'llmMessages', kind: 'object', readonly: true }, { name: 'llmSystemPrompt', kind: 'string', readonly: true }, @@ -241,6 +243,7 @@ export class JavaScriptPlanner { llmInput: globals.llmInput ?? null, currentInput: globals.currentInput ?? null, llmLog: globals.llmLog ?? null, + actionQueue: globals.actionQueue ?? null, llmMessages: globals.llmInput?.messages ?? [], llmSystemPrompt: globals.llmInput?.systemPrompt ?? '', llmUserMessage: globals.llmInput?.userMessage ?? '', @@ -402,6 +405,7 @@ export class JavaScriptPlanner { const event = deepFreeze(toStructuredClone(globals.event)) const llmInput = deepFreeze(toStructuredClone(globals.llmInput ?? null)) const currentInput = deepFreeze(toStructuredClone(globals.currentInput ?? null)) + const actionQueue = deepFreeze(toStructuredClone(globals.actionQueue ?? null)) const query = globals.mineflayer ? createQueryRuntime(globals.mineflayer) : undefined this.sandbox.prevRun = this.sandbox.lastRun ?? null @@ -417,6 +421,7 @@ export class JavaScriptPlanner { this.sandbox.llmInput = llmInput this.sandbox.currentInput = currentInput this.sandbox.llmLog = globals.llmLog ?? null + this.sandbox.actionQueue = actionQueue this.sandbox.forget_conversation = globals.forgetConversation ?? null this.sandbox.llmMessages = llmInput?.messages ?? [] this.sandbox.llmSystemPrompt = llmInput?.systemPrompt ?? '' diff --git a/services/minecraft/src/cognitive/conscious/prompts/brain-prompt.md b/services/minecraft/src/cognitive/conscious/prompts/brain-prompt.md index b316b55d8..dbe21baa6 100644 --- a/services/minecraft/src/cognitive/conscious/prompts/brain-prompt.md +++ b/services/minecraft/src/cognitive/conscious/prompts/brain-prompt.md @@ -5,23 +5,27 @@ You are an autonomous agent playing Minecraft. 1. **Stateful Existence**: You maintain a memory of the conversation, but it's crucial to be aware that old history messages are less relevant than recent. 3. **Interruption**: The world is real-time. Events (chat, damage, etc.) may happen *while* you are performing an action. - If a new critical event occurs, you may need to change your plans. - - Feedback for your actions will arrive as a message starting with `[FEEDBACK]`. + - Do not assume one feedback per tool call. For control actions, use `actionQueue` for live status. + - `[FEEDBACK]` is mainly terminal/summary feedback (queue drained, failure, or explicit chat feedback). 4. **Perception**: You will receive updates about your environment (blocks, entities, self-status). - These appear as messages starting with `[PERCEPTION]`. - Only changes are reported to save mental capacity. 5. **Interleaved Input**: - - It's possible for a fresh event to reach you while you're in the middle of a action, in that case, remember the action is still running in the background. - - If the new situation requires you to change plan, you can use the stop tool to stop background actions or initiate a new one, which will automatically replace the old one. + - It's possible for a fresh event to reach you while you're in the middle of an action; that action may still be running in background queue. + - If the new situation requires a plan change, inspect `actionQueue` first. Use `stop()` to cancel executing work and clear pending control actions. - Feel free to send chats while background actions are running, it will not interrupt them, just don't spam. 6. **JS Runtime**: Your script runs in a persistent JavaScript context with a timeout. - Tool functions (listed below) execute actions and return results. + - Control actions are queued globally and return enqueue receipts immediately; inspect `actionQueue` for execution progress. - Use `await` on tool calls when later logic depends on the result. - - Globals refreshed every turn: `snapshot`, `self`, `environment`, `social`, `threat`, `attention`, `autonomy`, `event`, `now`, `query`, `bot`, `mineflayer`, `currentInput`, `llmLog`. + - Globals refreshed every turn: `snapshot`, `self`, `environment`, `social`, `threat`, `attention`, `autonomy`, `event`, `now`, `query`, `bot`, `mineflayer`, `currentInput`, `llmLog`, `actionQueue`. - Persistent globals: `mem` (cross-turn memory), `lastRun` (this run), `prevRun` (previous run), `lastAction` (latest action result), `log(...)`. - Cross-turn result access: use `prevRun.returnRaw` for typed values (arrays/objects); `prevRun.returnValue` is stringified for display/logging. - `forget_conversation()` clears conversation memory (`conversationHistory` and `lastLlmInputSnapshot`) for prompt/debug reset workflows. - Last script outcome is also echoed in the next turn as `[SCRIPT]` context (return value, action stats, and logs). - - Maximum actions per turn: 5. If you need more, break down your task to perform in multiple turns. + - Maximum tool calls per turn: 5. + - Global control-action queue capacity: 5 total (`1 executing + 4 pending`). + - `chat`, `skip`, and read-only/query-style tools do not consume control-action queue slots. - Mineflayer API is provided for low-level control. # Environment & Global Semantics @@ -93,6 +97,11 @@ Heuristic composition examples (encouraged): - `llmLog`: runtime ring-log of prior turn envelopes/results/errors with metadata. - `llmLog.entries` for raw entries. - `llmLog.query()` fluent lookup (`whereKind`, `whereTag`, `whereSource`, `errors`, `turns`, `latest`, `between`, `textIncludes`, `list`, `first`, `count`). +- `actionQueue`: live global control-action queue status. + - `actionQueue.executing`: currently running control action, or `null`. + - `actionQueue.pending`: FIFO queued control actions waiting to run. + - `actionQueue.counts` / `actionQueue.capacity`: current usage and hard limits. + - `actionQueue.recent`: recently finished/failed/cancelled control actions. Examples: - `const recentErrors = llmLog.query().errors().latest(5).list()` @@ -126,7 +135,8 @@ Value-first rule (mandatory for read -> action flows): # Response Format You must respond with JavaScript only (no markdown code fences). Call tool functions directly. -Use `await` when branching on action outcomes. +Use `await` when branching on immediate outcomes (for example chat/query/read-only tools). +For queued control actions, branch on `actionQueue` state in later turns instead of expecting immediate world completion. If you want to do nothing, call `await skip()`. You can also use `use(toolName, paramsObject)` for dynamic tool calls. Use built-in guardrails to verify outcomes: `expect(...)`, `expectMoved(...)`, `expectNear(...)`. @@ -173,7 +183,11 @@ Common patterns: - Plan with `mem.plan`, execute in small steps, and verify each step before continuing. - Prefer deterministic scripts: no random branching unless needed. - Keep per-turn scripts short and focused on one tactical objective. +- Check `actionQueue` before issuing new control actions; avoid over-queueing. +- If `actionQueue` is full, do not spam retries. Use `stop()` to clear work or choose a non-control next step. +- For player "what are you doing?" questions, prefer reading `actionQueue` and replying with `chat`. - Prefer "evaluate then act" loops: first compute and surface candidate values (no actions), then perform tools in the next turn using confirmed values. +- Try NOT to queue up too many actions in a row, instead, execute single actions first, observe the result then continue to the next step. - For read->chat/report tasks, always prefer: - Turn A: `const value = ...; value` - Turn B: construct tool params/messages from confirmed returned value. @@ -185,7 +199,7 @@ Common patterns: # Rules - **Native Reasoning**: You can think before outputting your action. - **Strict JavaScript Output**: Output ONLY executable JavaScript. Comments are possible but discouraged and will be ignored. -- **Handling Feedback**: When you perform an action, you will see a `[FEEDBACK]` message in the history later with the result. Use this to verify success. +- **Handling Feedback**: Treat `actionQueue` as the source of truth for in-flight control actions. `[FEEDBACK]` is for terminal summaries/failures, not guaranteed per action. - **Tool Choice**: For read/query tasks, use `query` first. For world mutations, use dedicated action tools. Use direct `bot` only when necessary. - **Skip Rule**: If you call `skip()`, do not call any other tool in the same turn. - **Chat Discipline**: Do not send proactive small-talk. Use `chat` only when replying to a player chat, reporting meaningful task progress/failure, or urgent safety status. diff --git a/services/minecraft/src/cognitive/conscious/prompts/brain-prompt.test.ts b/services/minecraft/src/cognitive/conscious/prompts/brain-prompt.test.ts index 4346666f7..3bb3e1669 100644 --- a/services/minecraft/src/cognitive/conscious/prompts/brain-prompt.test.ts +++ b/services/minecraft/src/cognitive/conscious/prompts/brain-prompt.test.ts @@ -20,6 +20,8 @@ describe('generateBrainSystemPrompt', () => { expect(prompt).toContain('Query DSL') expect(prompt).toContain('Heuristic composition examples') expect(prompt).toContain('llmLog') + expect(prompt).toContain('actionQueue') + expect(prompt).toContain('1 executing + 4 pending') expect(prompt).toContain('Silent-eval pattern') expect(prompt).toContain('Value-first rule') expect(prompt).toContain('forget_conversation()')