From 74383610082e4318e98caa6b7ebd36af5c318ac9 Mon Sep 17 00:00:00 2001 From: Rin Date: Wed, 4 Feb 2026 01:52:36 +0800 Subject: [PATCH] refactor(minecraft): replace blocking followPlayer with reflex auto-follow system Replace async followPlayer skill with sync reflex-based auto-follow that runs during idle mode, add clearFollowTarget action to disable auto-follow before independent movement, add autonomy state to ReflexContext tracking followPlayer/followDistance/followActive/followLastError, implement reconcileAutoFollow in ReflexRuntime using pathfinder GoalFollow that pauses during work/wander/alert modes, add followControl: --- .../src/cognitive/action/llm-actions.ts | 31 +++++- .../src/cognitive/conscious/brain.ts | 5 +- .../src/cognitive/conscious/context-view.ts | 20 +--- .../src/cognitive/conscious/js-planner.ts | 1 + .../conscious/prompts/brain-prompt.ts | 26 ++++- .../minecraft/src/cognitive/reflex/context.ts | 19 ++++ .../src/cognitive/reflex/reflex-manager.ts | 10 ++ .../minecraft/src/cognitive/reflex/runtime.ts | 102 +++++++++++++++++- .../minecraft/src/libs/mineflayer/action.ts | 1 + 9 files changed, 191 insertions(+), 24 deletions(-) diff --git a/services/minecraft/src/cognitive/action/llm-actions.ts b/services/minecraft/src/cognitive/action/llm-actions.ts index d73307870..295f896ad 100644 --- a/services/minecraft/src/cognitive/action/llm-actions.ts +++ b/services/minecraft/src/cognitive/action/llm-actions.ts @@ -154,21 +154,42 @@ export const actionsList: Action[] = [ }, { name: 'followPlayer', - description: 'Endlessly follow the given player.', - execution: 'async', + description: 'Set idle auto-follow target handled by reflex runtime. While idle, the bot will keep following this player until cleared.', + execution: 'sync', + readonly: true, schema: z.object({ player_name: z.string().describe('name of the player to follow.'), follow_dist: z.number().describe('The distance to follow from.').min(0), }), - perform: mineflayer => async (player_name: string, follow_dist: number) => { - await skills.followPlayer(mineflayer, player_name, follow_dist) - return `Following player [${player_name}]` + perform: mineflayer => (player_name: string, follow_dist: number) => { + const reflexManager = (mineflayer as any).reflexManager + if (!reflexManager || typeof reflexManager.setFollowTarget !== 'function') + throw new Error('Reflex follow manager is unavailable') + + reflexManager.setFollowTarget(player_name, follow_dist) + return `Auto-follow enabled for player [${player_name}] at distance ${follow_dist}` + }, + }, + { + name: 'clearFollowTarget', + description: 'Disable idle auto-follow. Use this before independent exploration or when you no longer want to shadow a player.', + execution: 'sync', + readonly: true, + schema: z.object({}), + perform: mineflayer => () => { + const reflexManager = (mineflayer as any).reflexManager + if (!reflexManager || typeof reflexManager.clearFollowTarget !== 'function') + throw new Error('Reflex follow manager is unavailable') + + reflexManager.clearFollowTarget() + return 'Auto-follow disabled' }, }, { name: 'goToCoordinate', description: 'Go to the given x, y, z location.', execution: 'async', + followControl: 'detach', schema: z.object({ x: z.number().describe('The x coordinate.'), y: z.number().describe('The y coordinate.').min(-64).max(320), diff --git a/services/minecraft/src/cognitive/conscious/brain.ts b/services/minecraft/src/cognitive/conscious/brain.ts index e9efc6c1b..6faaeee5a 100644 --- a/services/minecraft/src/cognitive/conscious/brain.ts +++ b/services/minecraft/src/cognitive/conscious/brain.ts @@ -267,6 +267,9 @@ export class Brain { } const actionDef = actionDefs.get(action.tool) + if (actionDef?.followControl === 'detach') + this.deps.reflexManager.clearFollowTarget() + const isPhysicalAction = action.tool !== 'skip' && !actionDef?.readonly if (isPhysicalAction) { @@ -344,7 +347,7 @@ export class Brain { parts.push(`[STATE] giveUp active (${remainingSec}s left). reason=${this.giveUpReason ?? 'unknown'}`) } - parts.push('[RUNTIME] Globals are refreshed every turn: snapshot, self, environment, social, threat, attention, event, now, mem, lastRun, lastAction.') + parts.push('[RUNTIME] Globals are refreshed every turn: snapshot, self, environment, social, threat, attention, autonomy, event, now, mem, lastRun, lastAction. Player gaze is available in environment.nearbyPlayersGaze when needed.') return parts.join('\n\n') } diff --git a/services/minecraft/src/cognitive/conscious/context-view.ts b/services/minecraft/src/cognitive/conscious/context-view.ts index 0b13b6003..d6fe4c454 100644 --- a/services/minecraft/src/cognitive/conscious/context-view.ts +++ b/services/minecraft/src/cognitive/conscious/context-view.ts @@ -8,27 +8,17 @@ export interface ConsciousContextView { export function buildConsciousContextView(ctx: ReflexContextState): ConsciousContextView { const pos = ctx.self.location const roundedPos = `(${Math.round(pos.x)}, ${Math.round(pos.y)}, ${Math.round(pos.z)})` - const selfSummary = `Position ${roundedPos} Health ${ctx.self.health}/20 Food ${ctx.self.food}/20 and I'm holding ${ctx.self.holding ?? 'nothing'}` + const followState = ctx.autonomy.followPlayer + ? ` Auto-follow target ${ctx.autonomy.followPlayer} (${ctx.autonomy.followActive ? 'active' : 'paused'})` + : ' Auto-follow disabled' + const selfSummary = `Position ${roundedPos} Health ${ctx.self.health}/20 Food ${ctx.self.food}/20 and I'm holding ${ctx.self.holding ?? 'nothing'}.${followState}` const players = ctx.environment.nearbyPlayers .map(p => (p.holding ? `${p.name} is holding (${p.holding})` : p.name)) .join(',') const entities = ctx.environment.nearbyEntities.map(e => e.name).join(',') - const gaze = ctx.environment.nearbyPlayersGaze - .map((g) => { - if (g.hitBlock) { - const block = `${g.hitBlock.name} at (${Math.round(g.hitBlock.pos.x)}, ${Math.round(g.hitBlock.pos.y)}, ${Math.round(g.hitBlock.pos.z)})` - return `${g.name} is looking at ${block}` - } - - const lp = `(${Math.round(g.lookPoint.x)}, ${Math.round(g.lookPoint.y)}, ${Math.round(g.lookPoint.z)})` - return `${g.name} is staring into the air around ${lp}` - }) - .join('\n') - const gazeSummary = ctx.environment.nearbyPlayersGaze.length > 0 ? `\nNearby player gaze:\n${gaze}` : '' - - const environmentSummary = `${ctx.environment.time} ${ctx.environment.weather} Nearby players [${players}] Nearby entities [${entities}] Light ${ctx.environment.lightLevel}${gazeSummary}` + const environmentSummary = `${ctx.environment.time} ${ctx.environment.weather} Nearby players [${players}] Nearby entities [${entities}] Light ${ctx.environment.lightLevel}` return { selfSummary, diff --git a/services/minecraft/src/cognitive/conscious/js-planner.ts b/services/minecraft/src/cognitive/conscious/js-planner.ts index db06b09eb..d9af1d175 100644 --- a/services/minecraft/src/cognitive/conscious/js-planner.ts +++ b/services/minecraft/src/cognitive/conscious/js-planner.ts @@ -168,6 +168,7 @@ export class JavaScriptPlanner { this.sandbox.social = snapshot.social this.sandbox.threat = snapshot.threat this.sandbox.attention = snapshot.attention + this.sandbox.autonomy = snapshot.autonomy this.sandbox.lastRun = { actions: run.executed, logs: run.logs, diff --git a/services/minecraft/src/cognitive/conscious/prompts/brain-prompt.ts b/services/minecraft/src/cognitive/conscious/prompts/brain-prompt.ts index 3285ede55..adf637a57 100644 --- a/services/minecraft/src/cognitive/conscious/prompts/brain-prompt.ts +++ b/services/minecraft/src/cognitive/conscious/prompts/brain-prompt.ts @@ -106,10 +106,30 @@ You are an autonomous agent playing Minecraft. 6. **Planner Runtime**: Your script runs in a persistent JavaScript context with a timeout. - Tool functions (listed below) execute actions and return results. - Use \`await\` on tool calls when later logic depends on the result. - - Globals refreshed every turn: \`snapshot\`, \`self\`, \`environment\`, \`social\`, \`threat\`, \`attention\`, \`event\`, \`now\`. + - Globals refreshed every turn: \`snapshot\`, \`self\`, \`environment\`, \`social\`, \`threat\`, \`attention\`, \`autonomy\`, \`event\`, \`now\`. - Persistent globals: \`mem\` (cross-turn memory), \`lastRun\` (this run), \`prevRun\` (previous run), \`lastAction\` (latest action result), \`log(...)\`. - Maximum actions per turn: 5. +# Environment & Global Semantics +- \`self\`: your current body state (position, health, food, held item). +- \`environment.nearbyPlayers\`: nearby players and rough distance/held item. +- \`environment.nearbyPlayersGaze\`: where nearby players appear to be looking. + - Each entry may include: + - \`name\` + - \`distanceToSelf\` + - \`lookPoint\` (estimated point in world) + - optional \`hitBlock\` with block \`name\` and \`pos\` + - This is heuristic perception, not a guaranteed command or exact target. +- \`social\`: latest speaker/message signals remembered by reflex layer. +- \`threat\`: coarse danger score; higher means more urgent survival behavior. +- \`attention\`: most recent perception signal type/source/time. +- \`autonomy\`: reflex-side autonomous state (including auto-follow target/activity). + +# Limitations You Must Respect +- Perception can be stale/noisy; verify important assumptions before committing long tasks. +- Action execution can fail silently or partially; check results and adapt step by step. +- Player gaze alone is not intent; only treat it as intent when combined with explicit instruction context. + # Available Tools You must use the following tools to interact with the world. You cannot make up tools. @@ -136,6 +156,9 @@ Examples: - Prefer deterministic scripts: no random branching unless needed. - Keep per-turn scripts short and focused on one tactical objective. - If you hit repeated failures with no progress, call \`await giveUp({ reason, cooldown_seconds })\` once instead of retry-spamming. +- Treat \`environment.nearbyPlayersGaze\` as a weak hint, not a command. Never move solely because someone looked somewhere unless they also gave a clear instruction. +- Use \`followPlayer\` to set idle auto-follow and \`clearFollowTarget\` before independent exploration. +- Some relocation actions (for example \`goToCoordinate\`) automatically detach auto-follow so exploration does not keep snapping back. # Rules - **Native Reasoning**: You can think before outputting your action. @@ -147,5 +170,6 @@ Examples: - **No Harness Replies**: Never treat \`[PERCEPTION]\`, \`[FEEDBACK]\`, or other system wrappers as players. Only reply with \`chat\` to actual player \`chat_message\` events. - **No Self Replies**: Never reply to your own previous bot messages. - **Chat Feedback**: \`chat\` feedback is optional; keep \`feedback: false\` for normal conversation. Use \`feedback: true\` only when your next step explicitly needs the chat acknowledgement in history. +- **Follow Mode**: If \`autonomy.followPlayer\` is set, reflex will follow that player while idle. Only clear it when the current mission needs independent movement. ` } diff --git a/services/minecraft/src/cognitive/reflex/context.ts b/services/minecraft/src/cognitive/reflex/context.ts index be9c01757..5c43cee0e 100644 --- a/services/minecraft/src/cognitive/reflex/context.ts +++ b/services/minecraft/src/cognitive/reflex/context.ts @@ -42,6 +42,13 @@ export interface ReflexAttentionState { lastSignalAt: number | null } +export interface ReflexAutonomyState { + followPlayer: string | null + followDistance: number + followActive: boolean + followLastError: string | null +} + export interface ReflexContextState { now: number self: ReflexSelfState @@ -49,6 +56,7 @@ export interface ReflexContextState { social: ReflexSocialState threat: ReflexThreatState attention: ReflexAttentionState + autonomy: ReflexAutonomyState } export class ReflexContext { @@ -89,6 +97,12 @@ export class ReflexContext { lastSignalSourceId: null, lastSignalAt: null, }, + autonomy: { + followPlayer: null, + followDistance: 2, + followActive: false, + followLastError: null, + }, } } @@ -117,6 +131,7 @@ export class ReflexContext { }, threat: { ...this.state.threat }, attention: { ...this.state.attention }, + autonomy: { ...this.state.autonomy }, } } @@ -143,4 +158,8 @@ export class ReflexContext { public updateAttention(patch: Partial): void { this.state.attention = { ...this.state.attention, ...patch } } + + public updateAutonomy(patch: Partial): void { + this.state.autonomy = { ...this.state.autonomy, ...patch } + } } diff --git a/services/minecraft/src/cognitive/reflex/reflex-manager.ts b/services/minecraft/src/cognitive/reflex/reflex-manager.ts index 81d425649..6e998edc3 100644 --- a/services/minecraft/src/cognitive/reflex/reflex-manager.ts +++ b/services/minecraft/src/cognitive/reflex/reflex-manager.ts @@ -101,6 +101,16 @@ export class ReflexManager { this.runtime.getContext().updateEnvironment(patch) } + public setFollowTarget(playerName: string, followDistance = 2): void { + this.runtime.setAutoFollowTarget(playerName, followDistance) + this.emitReflexState() + } + + public clearFollowTarget(): void { + this.runtime.clearAutoFollowTarget(this.bot) + this.emitReflexState() + } + private onSignal(event: TracedEvent): void { const bot = this.bot if (!bot) diff --git a/services/minecraft/src/cognitive/reflex/runtime.ts b/services/minecraft/src/cognitive/reflex/runtime.ts index bc76f7206..59599c9af 100644 --- a/services/minecraft/src/cognitive/reflex/runtime.ts +++ b/services/minecraft/src/cognitive/reflex/runtime.ts @@ -5,10 +5,13 @@ import type { MineflayerWithAgents } from '../types' import type { ReflexModeId } from './modes' import type { ReflexBehavior } from './types/behavior' +import pathfinderModel from 'mineflayer-pathfinder' + import { ReflexContext } from './context' import { selectMode } from './modes' export class ReflexRuntime { + private readonly followMovementsByBot = new WeakMap>() private readonly context = new ReflexContext() private readonly behaviors: ReflexBehavior[] = [] private readonly runHistory = new Map() @@ -16,6 +19,7 @@ export class ReflexRuntime { private mode: ReflexModeId = 'idle' private activeBehaviorId: string | null = null private activeBehaviorUntil: number | null = null + private activeAutoFollowPlayer: string | null = null public constructor( private readonly deps: { @@ -33,6 +37,24 @@ export class ReflexRuntime { return this.mode } + public setAutoFollowTarget(playerName: string, followDistance = 2): void { + this.context.updateAutonomy({ + followPlayer: playerName, + followDistance: Math.max(0, followDistance), + followLastError: null, + }) + } + + public clearAutoFollowTarget(bot: MineflayerWithAgents | null): void { + this.stopAutoFollow(bot) + this.context.updateAutonomy({ + followPlayer: null, + followDistance: 2, + followActive: false, + followLastError: null, + }) + } + /** * Single entrypoint for mode changes. Runs onExit/onEnter side effects and notifies onModeChange * only when the mode actually changes. Pass bot when available so mode handlers can perform @@ -51,8 +73,8 @@ export class ReflexRuntime { } private onEnterMode(mode: ReflexModeId, _bot: MineflayerWithAgents | null): void { - if (mode !== 'social') - return + if (mode === 'work' || mode === 'wander' || mode === 'alert') + this.stopAutoFollow(_bot) } private onExitMode(mode: ReflexModeId, _bot: MineflayerWithAgents | null): void { @@ -136,6 +158,8 @@ export class ReflexRuntime { this.transitionMode(nextMode, bot) } + this.reconcileAutoFollow(bot) + if (this.activeBehaviorUntil && now < this.activeBehaviorUntil) return null @@ -199,4 +223,78 @@ export class ReflexRuntime { return null } } + + private reconcileAutoFollow(bot: MineflayerWithAgents): void { + const { goals, Movements } = pathfinderModel + const snapshot = this.context.getSnapshot() + const followPlayer = snapshot.autonomy.followPlayer + const followDistance = snapshot.autonomy.followDistance + + if (!followPlayer) { + this.stopAutoFollow(bot) + if (snapshot.autonomy.followActive || snapshot.autonomy.followLastError) { + this.context.updateAutonomy({ + followActive: false, + followLastError: null, + }) + } + return + } + + // Work-like modes always take priority over idle follow. + if (this.mode === 'work' || this.mode === 'wander' || this.mode === 'alert') { + if (snapshot.autonomy.followActive) + this.context.updateAutonomy({ followActive: false }) + this.stopAutoFollow(bot) + return + } + + const target = bot.bot.players[followPlayer]?.entity + if (!target) { + this.stopAutoFollow(bot) + this.context.updateAutonomy({ + followActive: false, + followLastError: `Player [${followPlayer}] is not currently visible`, + }) + return + } + + if (this.activeAutoFollowPlayer === followPlayer && snapshot.autonomy.followActive) + return + + try { + const movements = this.followMovementsByBot.get(bot.bot) + ?? new Movements(bot.bot) + if (!this.followMovementsByBot.has(bot.bot)) + this.followMovementsByBot.set(bot.bot, movements) + + bot.bot.pathfinder.setMovements(movements) + bot.bot.pathfinder.setGoal(new goals.GoalFollow(target, followDistance), true) + this.activeAutoFollowPlayer = followPlayer + this.context.updateAutonomy({ + followActive: true, + followLastError: null, + }) + } + catch (error) { + this.stopAutoFollow(bot) + this.context.updateAutonomy({ + followActive: false, + followLastError: error instanceof Error ? error.message : String(error), + }) + } + } + + private stopAutoFollow(bot: MineflayerWithAgents | null): void { + if (!this.activeAutoFollowPlayer) + return + + this.activeAutoFollowPlayer = null + try { + bot?.bot.pathfinder.stop() + } + catch { + // Ignore cleanup errors from transient pathfinder state. + } + } } diff --git a/services/minecraft/src/libs/mineflayer/action.ts b/services/minecraft/src/libs/mineflayer/action.ts index 033de9930..08d0a0f92 100644 --- a/services/minecraft/src/libs/mineflayer/action.ts +++ b/services/minecraft/src/libs/mineflayer/action.ts @@ -9,6 +9,7 @@ export interface Action { readonly description: string readonly schema: z.ZodObject readonly readonly?: boolean + readonly followControl?: 'pause' | 'detach' readonly execution?: 'sync' | 'async' readonly perform: (mineflayer: Mineflayer) => (...args: any[]) => ActionResult }